├── docs ├── .gitignore ├── .python-version ├── requirements.txt ├── README.md └── GANDA_TOUR.ipynb ├── .gitignore ├── Makefile ├── main.go ├── request.schema.json ├── cli ├── worker_flag.go ├── cli_flags_test.go ├── test_helper_test.go ├── cli_echoserver_test.go ├── cli_test.go ├── cli_response_output_test.go └── cli.go ├── go.mod ├── logger └── logger.go ├── .goreleaser.yaml ├── config └── config.go ├── execcontext └── execcontext.go ├── echoserver ├── echoserver.go └── echoserver_test.go ├── requests └── requests.go ├── go.sum ├── parser ├── parser.go └── parser_test.go ├── README.md ├── responses ├── responses_test.go └── responses.go └── LICENSE /docs/.gitignore: -------------------------------------------------------------------------------- 1 | scratch 2 | -------------------------------------------------------------------------------- /docs/.python-version: -------------------------------------------------------------------------------- 1 | gandadocs-3.12.2 2 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | ipykernel 2 | ipython 3 | jupyter_client 4 | jupyter_core 5 | bash_kernel -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ganda 2 | ganda.iml 3 | ganda.ipr 4 | ganda.iws 5 | .DS_Store 6 | out 7 | .idea 8 | TODO.md 9 | dist 10 | ganda-amd64 11 | 12 | dist/ 13 | node_modules 14 | package*.json 15 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: lint test build 2 | 3 | lint: 4 | go fmt 5 | gofmt -s -w . 6 | go vet ./... 7 | 8 | build: 9 | go build -o ganda -v 10 | 11 | test: 12 | go test -v ./... 13 | 14 | install: lint test build 15 | go install 16 | 17 | clean: 18 | go clean 19 | rm -f ganda 20 | rm -f ganda-amd64 21 | 22 | build-linux: 23 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ganda-amd64 -v -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | ctx "context" 5 | "github.com/tednaleid/ganda/cli" 6 | "os" 7 | ) 8 | 9 | // overridden at build time with `-ldflags`, ex: 10 | // go build -ldflags "-X main.version=0.2.0 -X main.commit=123abc -X main.date=2023-12-20" 11 | var ( 12 | version = "dev" 13 | commit = "none" 14 | date = "unknown" 15 | ) 16 | 17 | func main() { 18 | command := cli.SetupCommand( 19 | cli.BuildInfo{Version: version, Commit: commit, Date: date}, 20 | os.Stdin, 21 | os.Stderr, 22 | os.Stdout, 23 | ) 24 | 25 | err := command.Run(ctx.Background(), os.Args) 26 | 27 | if err != nil { 28 | os.Exit(1) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /request.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "url": { 5 | "type": "string" 6 | }, 7 | "method": { 8 | "type": "string", 9 | "enum": ["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS", "TRACE", "CONNECT"] 10 | }, 11 | "headers": { 12 | "type": "object", 13 | "additionalProperties": { 14 | "type": "string" 15 | } 16 | }, 17 | "context": { 18 | "type": ["string", "number", "boolean", "object", "array", "null"] 19 | }, 20 | "body": { 21 | "type": ["string", "number", "boolean", "object", "array", "null"] 22 | }, 23 | "bodyType": { 24 | "type": "string", 25 | "enum": ["escaped", "base64", "json", ""] 26 | } 27 | }, 28 | "required": ["url"], 29 | "additionalProperties": false 30 | } -------------------------------------------------------------------------------- /cli/worker_flag.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | ) 7 | import "github.com/urfave/cli/v3" 8 | 9 | // the urfave/cli package only supports int64 flags, we only want realistic (>0, not too big) values 10 | type WorkerFlag = cli.FlagBase[int, cli.IntegerConfig, intValue] 11 | 12 | type intValue struct { 13 | val *int 14 | base int 15 | } 16 | 17 | func (i intValue) Create(val int, p *int, c cli.IntegerConfig) cli.Value { 18 | *p = val 19 | return &intValue{ 20 | val: p, 21 | base: c.Base, 22 | } 23 | } 24 | 25 | func (i intValue) ToString(b int) string { 26 | return strconv.Itoa(b) 27 | } 28 | 29 | func (i *intValue) Set(s string) error { 30 | v, err := strconv.Atoi(s) 31 | if err != nil { 32 | return err 33 | } 34 | if v < 1 || v > 1<<20 { 35 | return fmt.Errorf("value out of range: %v is not between 1 and 2^20", v) 36 | } 37 | *i.val = v 38 | return err 39 | } 40 | 41 | func (i *intValue) Get() any { return *i.val } 42 | 43 | func (i *intValue) String() string { return strconv.Itoa(*i.val) } 44 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tednaleid/ganda 2 | 3 | go 1.22.5 4 | 5 | require ( 6 | github.com/labstack/echo/v4 v4.11.4 7 | github.com/stretchr/testify v1.9.0 8 | github.com/urfave/cli/v3 v3.0.0-alpha9 9 | golang.org/x/net v0.27.0 10 | ) 11 | 12 | require ( 13 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 14 | github.com/golang-jwt/jwt v3.2.2+incompatible // indirect 15 | github.com/kr/pretty v0.3.1 // indirect 16 | github.com/labstack/gommon v0.4.2 // indirect 17 | github.com/mattn/go-colorable v0.1.13 // indirect 18 | github.com/mattn/go-isatty v0.0.20 // indirect 19 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 20 | github.com/rogpeppe/go-internal v1.12.0 // indirect 21 | github.com/valyala/bytebufferpool v1.0.0 // indirect 22 | github.com/valyala/fasttemplate v1.2.2 // indirect 23 | github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect 24 | golang.org/x/crypto v0.25.0 // indirect 25 | golang.org/x/sys v0.22.0 // indirect 26 | golang.org/x/text v0.16.0 // indirect 27 | golang.org/x/time v0.5.0 // indirect 28 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 29 | gopkg.in/yaml.v3 v3.0.1 // indirect 30 | ) 31 | -------------------------------------------------------------------------------- /logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import "log" 4 | 5 | type LeveledLogger struct { 6 | showColor bool 7 | silent bool 8 | logger *log.Logger 9 | } 10 | 11 | func NewSilentLogger() *LeveledLogger { 12 | return &LeveledLogger{ 13 | silent: true, 14 | showColor: false, 15 | } 16 | } 17 | 18 | func NewPlainLeveledLogger(logger *log.Logger) *LeveledLogger { 19 | return &LeveledLogger{ 20 | silent: false, 21 | showColor: false, 22 | logger: logger, 23 | } 24 | } 25 | 26 | func NewLeveledLogger(logger *log.Logger) *LeveledLogger { 27 | return &LeveledLogger{ 28 | silent: false, 29 | showColor: true, 30 | logger: logger, 31 | } 32 | } 33 | 34 | func (l *LeveledLogger) Info(format string, args ...interface{}) { 35 | if !l.silent { 36 | l.logger.Printf(format, args...) 37 | } 38 | } 39 | 40 | func (l *LeveledLogger) Warn(format string, args ...interface{}) { 41 | if l.showColor { 42 | l.logger.Printf("\033[31m"+format+"\033[0m", args...) 43 | } else if !l.silent { 44 | l.logger.Printf(format, args...) 45 | } 46 | } 47 | 48 | func (l *LeveledLogger) Success(format string, args ...interface{}) { 49 | if l.showColor { 50 | l.logger.Printf("\033[32m"+format+"\033[0m", args...) 51 | } else if !l.silent { 52 | l.logger.Printf(format, args...) 53 | } 54 | } 55 | 56 | func (l *LeveledLogger) LogResponse(statusCode int, message string) { 57 | if statusCode < 400 { 58 | l.Success("Response: %d %s", statusCode, message) 59 | } else { 60 | l.Warn("Response: %d %s", statusCode, message) 61 | } 62 | } 63 | 64 | func (l *LeveledLogger) LogError(err error, message string) { 65 | l.Warn("%s Error: %s", message, err) 66 | } 67 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | # This is an example .goreleaser.yml file with some sensible defaults. 2 | # Make sure to check the documentation at https://goreleaser.com 3 | 4 | # The lines below are called `modelines`. See `:help modeline` 5 | # Feel free to remove those if you don't want/need to use them. 6 | # yaml-language-server: $schema=https://goreleaser.com/static/schema.json 7 | # vim: set ts=2 sw=2 tw=0 fo=cnqoj 8 | 9 | version: 2 10 | 11 | before: 12 | hooks: 13 | # You may remove this if you don't use go modules. 14 | - go mod tidy 15 | # you may remove this if you don't need go generate 16 | - go generate ./... 17 | 18 | builds: 19 | - env: 20 | - CGO_ENABLED=0 21 | goos: 22 | - linux 23 | - windows 24 | - darwin 25 | 26 | archives: 27 | - format: tar.gz 28 | # this name template makes the OS and Arch compatible with the results of `uname`. 29 | name_template: >- 30 | {{ .ProjectName }}_ 31 | {{- title .Os }}_ 32 | {{- if eq .Arch "amd64" }}x86_64 33 | {{- else if eq .Arch "386" }}i386 34 | {{- else }}{{ .Arch }}{{ end }} 35 | {{- if .Arm }}v{{ .Arm }}{{ end }} 36 | # use zip for windows archives 37 | format_overrides: 38 | - goos: windows 39 | format: zip 40 | 41 | changelog: 42 | sort: asc 43 | filters: 44 | exclude: 45 | - "^docs:" 46 | - "^test:" 47 | 48 | brews: 49 | - name: ganda 50 | homepage: "https://github.com/tednaleid/ganda" 51 | description: "fast cmd-line app that quickly request millions of urls and save/echo the results" 52 | repository: 53 | owner: tednaleid 54 | name: homebrew-ganda 55 | commit_author: 56 | name: tednaleid 57 | email: contact@naleid.com 58 | directory: Formula -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "math" 6 | "strings" 7 | ) 8 | 9 | type Config struct { 10 | BaseDirectory string 11 | BaseRetryDelayMillis int64 12 | Color bool 13 | ConnectTimeoutMillis int64 14 | Insecure bool 15 | JsonEnvelope bool 16 | RequestFilename string 17 | RequestHeaders []RequestHeader 18 | RequestMethod string 19 | RequestWorkers int 20 | ResponseWorkers int 21 | ResponseBody ResponseBodyType 22 | Retries int64 23 | Silent bool 24 | SubdirLength int64 25 | ThrottlePerSecond int64 26 | } 27 | 28 | func New() *Config { 29 | return &Config{ 30 | BaseRetryDelayMillis: 1_000, 31 | Color: false, 32 | ConnectTimeoutMillis: 10_000, 33 | Insecure: false, 34 | JsonEnvelope: false, 35 | RequestMethod: "GET", 36 | RequestWorkers: 1, 37 | ResponseBody: Raw, 38 | Retries: 0, 39 | Silent: false, 40 | SubdirLength: 0, 41 | ThrottlePerSecond: math.MaxInt32, 42 | } 43 | } 44 | 45 | type RequestHeader struct { 46 | Key string 47 | Value string 48 | } 49 | 50 | func NewRequestHeader(headerString string) (RequestHeader, error) { 51 | if strings.Contains(headerString, ":") { 52 | parts := strings.SplitN(headerString, ":", 2) 53 | return RequestHeader{Key: strings.TrimSpace(parts[0]), Value: strings.TrimSpace(parts[1])}, nil 54 | } 55 | 56 | return RequestHeader{}, errors.New("Header should be in the format 'Key: value', missing ':' -> " + headerString) 57 | } 58 | 59 | func ConvertRequestHeaders(stringHeaders []string) ([]RequestHeader, error) { 60 | var requestHeaders []RequestHeader 61 | 62 | for _, header := range stringHeaders { 63 | var requestHeader RequestHeader 64 | requestHeader, err := NewRequestHeader(header) 65 | 66 | if err != nil { 67 | return requestHeaders, err 68 | } 69 | 70 | requestHeaders = append(requestHeaders, requestHeader) 71 | } 72 | 73 | return requestHeaders, nil 74 | } 75 | 76 | type ResponseBodyType string 77 | 78 | const ( 79 | Base64 ResponseBodyType = "base64" 80 | Discard ResponseBodyType = "discard" 81 | Escaped ResponseBodyType = "escaped" // escaped to a valid JSON string 82 | Sha256 ResponseBodyType = "sha256" 83 | Raw ResponseBodyType = "raw" 84 | ) 85 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Setup/Installation 2 | 3 | I'm using VSCode's python and jupyter plugins to run the [GANDA_TOUR.ipynb](GANDA_TOUR.ipynb) notebook: 4 | 5 | * https://marketplace.visualstudio.com/items?itemName=ms-python.python 6 | * https://marketplace.visualstudio.com/items?itemName=ms-toolsai.jupyter 7 | 8 | 9 | ## Install pyenv-virtualenv 10 | 11 | I use [pyenv](https://github.com/pyenv/pyenv) and [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv) for managing python and installing virtual environments. On the Mac, they can be installed with homebrew using: 12 | 13 | ``` 14 | brew update 15 | brew install pyenv pyenv-virtualenv 16 | ``` 17 | 18 | and you can add this to your `.zshrc` to get it to automatically load the shims in your terminal with: 19 | 20 | ``` 21 | if command -v pyenv >/dev/null 2>&1; then 22 | export PYENV_ROOT="$HOME/.pyenv" 23 | export PATH="$PYENV_ROOT/bin:$PATH" 24 | eval "$(pyenv init -)" 25 | eval "$(pyenv virtualenv-init -)" 26 | else 27 | echo "missing pyenv, install with:" 28 | echo "brew install pyenv" 29 | echo "pyenv install 3.12.2" 30 | fi 31 | ``` 32 | 33 | ## Create a Python Environment and Install Dependencies 34 | 35 | Install python and create a new virtual environment for this notebook with: 36 | 37 | ``` 38 | pyenv install 3.12.2 39 | 40 | # make it the global python if desired: 41 | pyenv global 3.12.2 42 | 43 | # create the virtual environment used in .python-version: 44 | pyenv virtualenv 3.12.2 gandadocs-3.12.2 45 | ``` 46 | 47 | Now, when you're in this directory in your shell, you should see this as the active virtualenv: 48 | 49 | ``` 50 | which python 51 | /Users//.pyenv/shims/python 52 | 53 | python -V 54 | Python 3.12.2 55 | 56 | pyenv versions 57 | system 58 | 3.12.2 59 | 3.12.2/envs/gandadocs-3.12.2 60 | * gandadocs-3.12.2 --> /Users//.pyenv/versions/3.12.2/envs/gandadocs-3.12.2 (set by /Users///ganda/docs/.python-version) 61 | ``` 62 | 63 | Python (pip) typically stores dependencies in `requirements.txt` (or other modern replacements), install them with: 64 | 65 | ``` 66 | pip install -r requirements.txt 67 | ``` 68 | 69 | ## Install the bash kernel 70 | 71 | ``` 72 | python -m bash_kernel.install 73 | ``` 74 | 75 | ## Choose the Bash kernel in VSCode 76 | 77 | When you run the first bash cell, VSCode will prompt you for the kernel to use. You should be able to pick the `gandadocs-3.12.2` kernel. 78 | 79 | ## Notebooks 80 | 81 | - GANDA_TOUR.ipynb - An interactive bash jupyter notebook that demonstrates how to use `ganda` to make parallel http requests and process the responses in a pipeline. -------------------------------------------------------------------------------- /execcontext/execcontext.go: -------------------------------------------------------------------------------- 1 | package execcontext 2 | 3 | import ( 4 | "fmt" 5 | "github.com/tednaleid/ganda/config" 6 | "github.com/tednaleid/ganda/logger" 7 | "io" 8 | "log" 9 | "math" 10 | "os" 11 | "time" 12 | ) 13 | 14 | type Context struct { 15 | BaseDirectory string 16 | BaseRetryDelayDuration time.Duration 17 | ConnectTimeoutDuration time.Duration 18 | In io.Reader 19 | Insecure bool 20 | JsonEnvelope bool 21 | Logger *logger.LeveledLogger 22 | Out io.Writer 23 | RequestHeaders []config.RequestHeader 24 | RequestMethod string 25 | RequestWorkers int 26 | ResponseBody config.ResponseBodyType 27 | ResponseWorkers int 28 | Retries int64 29 | SubdirLength int64 30 | ThrottlePerSecond int64 31 | WriteFiles bool 32 | } 33 | 34 | func New(conf *config.Config, in io.Reader, stderr io.Writer, stdout io.Writer) (*Context, error) { 35 | var err error 36 | 37 | context := Context{ 38 | BaseDirectory: conf.BaseDirectory, 39 | BaseRetryDelayDuration: time.Duration(conf.BaseRetryDelayMillis) * time.Millisecond, 40 | ConnectTimeoutDuration: time.Duration(conf.ConnectTimeoutMillis) * time.Millisecond, 41 | In: in, 42 | Insecure: conf.Insecure, 43 | JsonEnvelope: conf.JsonEnvelope, 44 | Logger: createLeveledLogger(conf, stderr), 45 | Out: stdout, 46 | RequestMethod: conf.RequestMethod, 47 | RequestWorkers: conf.RequestWorkers, 48 | RequestHeaders: conf.RequestHeaders, 49 | ResponseBody: conf.ResponseBody, 50 | Retries: conf.Retries, 51 | SubdirLength: conf.SubdirLength, 52 | ThrottlePerSecond: math.MaxInt32, 53 | } 54 | 55 | if conf.ThrottlePerSecond > 0 { 56 | context.ThrottlePerSecond = conf.ThrottlePerSecond 57 | } 58 | 59 | if context.RequestWorkers <= 0 { 60 | context.RequestWorkers = 1 61 | } 62 | 63 | // updating to a single response worker for now, need to fix a bug where they aren't sharing stdout properly 64 | context.ResponseWorkers = 1 65 | 66 | if len(conf.RequestFilename) > 0 { 67 | // replace stdin with the file 68 | context.In, err = requestFileReader(conf.RequestFilename) 69 | } 70 | 71 | if len(conf.BaseDirectory) > 0 { 72 | context.WriteFiles = true 73 | } else { 74 | context.WriteFiles = false 75 | } 76 | 77 | return &context, err 78 | } 79 | 80 | func createLeveledLogger(conf *config.Config, stderr io.Writer) *logger.LeveledLogger { 81 | 82 | if conf.Silent { 83 | return logger.NewSilentLogger() 84 | } 85 | 86 | stdErrLogger := log.New(stderr, "", 0) 87 | 88 | if conf.Color { 89 | return logger.NewLeveledLogger(stdErrLogger) 90 | } 91 | 92 | return logger.NewPlainLeveledLogger(stdErrLogger) 93 | } 94 | 95 | func requestFileReader(requestFilename string) (io.Reader, error) { 96 | if _, err := os.Stat(requestFilename); os.IsNotExist(err) { 97 | return nil, fmt.Errorf("Unable to open specified file: %s", requestFilename) 98 | } 99 | 100 | return os.Open(requestFilename) 101 | } 102 | -------------------------------------------------------------------------------- /cli/cli_flags_test.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/tednaleid/ganda/config" 6 | "math" 7 | "strconv" 8 | "testing" 9 | ) 10 | 11 | func TestHelp(t *testing.T) { 12 | results, _ := ParseGandaArgs([]string{"ganda", "-h"}) 13 | assert.NotNil(t, results) 14 | assert.Nil(t, results.GetContext()) // context isn't set up when help is called 15 | assert.Equal(t, "", results.stderr) // help is not written to stderr when explicitly called 16 | assert.Contains(t, results.stdout, "NAME:\n ganda") 17 | } 18 | 19 | func TestVersion(t *testing.T) { 20 | results, _ := ParseGandaArgs([]string{"ganda", "-v"}) 21 | assert.NotNil(t, results) 22 | assert.Nil(t, results.GetContext()) // context isn't set up when version is called 23 | assert.Equal(t, "", results.stderr) 24 | assert.Equal(t, "ganda version "+testBuildInfo.ToString()+"\n", results.stdout) 25 | } 26 | 27 | func TestWorkers(t *testing.T) { 28 | results, _ := ParseGandaArgs([]string{"ganda", "-W", "10"}) 29 | assert.NotNil(t, results) 30 | assert.Equal(t, 10, results.GetContext().RequestWorkers) 31 | } 32 | 33 | func TestRetries(t *testing.T) { 34 | results, _ := ParseGandaArgs([]string{"ganda"}) 35 | assert.NotNil(t, results) 36 | assert.Equal(t, int64(0), results.GetContext().Retries) 37 | 38 | separateResults, _ := ParseGandaArgs([]string{"ganda", "--retry", "5"}) 39 | assert.NotNil(t, separateResults) 40 | assert.Equal(t, int64(5), separateResults.GetContext().Retries) 41 | } 42 | 43 | func TestInvalidWorkers(t *testing.T) { 44 | testCases := []struct { 45 | input string 46 | error string 47 | }{ 48 | {strconv.FormatInt(int64(math.MaxInt32)+1, 10), "value out of range"}, 49 | {"foobar", "invalid value \"foobar\" for flag -W"}, 50 | } 51 | 52 | for _, tc := range testCases { 53 | results, _ := ParseGandaArgs([]string{"ganda", "-W", tc.input}) 54 | assert.NotNil(t, results) 55 | assert.Nil(t, results.GetContext()) 56 | assert.Contains(t, results.stderr, tc.error) 57 | } 58 | } 59 | 60 | func TestResponseBodyFlags(t *testing.T) { 61 | results, _ := ParseGandaArgs([]string{"ganda"}) 62 | assert.NotNil(t, results) 63 | assert.NotNil(t, results.GetContext()) 64 | assert.Equal(t, config.Raw, results.GetContext().ResponseBody) 65 | 66 | testCases := []struct { 67 | input string 68 | expected config.ResponseBodyType 69 | }{ 70 | {"", config.Raw}, 71 | {"base64", config.Base64}, 72 | {"discard", config.Discard}, 73 | {"escaped", config.Escaped}, 74 | {"raw", config.Raw}, 75 | {"sha256", config.Sha256}, 76 | } 77 | 78 | for _, tc := range testCases { 79 | if tc.input == "" { 80 | results, _ := ParseGandaArgs([]string{"ganda"}) 81 | assert.NotNil(t, results) 82 | assert.NotNil(t, results.GetContext()) 83 | assert.Equal(t, tc.expected, results.GetContext().ResponseBody) 84 | } else { 85 | shortResults, _ := ParseGandaArgs([]string{"ganda", "-B", tc.input}) 86 | assert.NotNil(t, shortResults) 87 | assert.NotNil(t, shortResults.GetContext()) 88 | assert.Equal(t, tc.expected, shortResults.GetContext().ResponseBody) 89 | 90 | longResults, _ := ParseGandaArgs([]string{"ganda", "--response-body", tc.input}) 91 | assert.NotNil(t, longResults) 92 | assert.NotNil(t, longResults.GetContext()) 93 | assert.Equal(t, tc.expected, longResults.GetContext().ResponseBody) 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /echoserver/echoserver.go: -------------------------------------------------------------------------------- 1 | package echoserver 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/labstack/echo/v4" 8 | "github.com/labstack/echo/v4/middleware" 9 | "io" 10 | "net/http" 11 | "os" 12 | "os/signal" 13 | "strings" 14 | "syscall" 15 | "time" 16 | ) 17 | 18 | type RequestEcho struct { 19 | Time string `json:"time"` 20 | ID string `json:"id"` 21 | RemoteIP string `json:"remote_ip"` 22 | Host string `json:"host"` 23 | Method string `json:"method"` 24 | URI string `json:"uri"` 25 | UserAgent string `json:"user_agent"` 26 | Status int `json:"status"` 27 | Headers map[string]string `json:"headers"` 28 | RequestBody string `json:"request_body"` 29 | } 30 | 31 | func Echoserver(port int64, delayMillis int64, out io.Writer) (func() error, error) { 32 | e := echo.New() 33 | e.HideBanner = true 34 | e.HidePort = true 35 | 36 | e.Use(middleware.BodyDump(func(c echo.Context, reqBody, resBody []byte) { 37 | logEntryJSON := requestToJSON(c, reqBody) 38 | fmt.Fprintf(out, "%s\n", logEntryJSON) 39 | })) 40 | 41 | e.Use(middleware.Recover()) 42 | e.Use(middleware.GzipWithConfig(middleware.GzipConfig{ 43 | Level: 5, 44 | })) 45 | 46 | echoRequest := func(c echo.Context) error { 47 | // quick and dirty way to simulate latency, sleeps are bad mkay 48 | if delayMillis > 0 { 49 | time.Sleep(time.Duration(delayMillis) * time.Millisecond) 50 | } 51 | reqBody, _ := io.ReadAll(c.Request().Body) 52 | logEntryJSON := requestToJSON(c, reqBody) 53 | return c.JSONBlob(http.StatusOK, logEntryJSON) 54 | } 55 | 56 | e.Any("/*", echoRequest) 57 | 58 | s := &http.Server{ 59 | Addr: fmt.Sprintf(":%d", port), 60 | Handler: e, 61 | ReadTimeout: 5 * time.Minute, 62 | WriteTimeout: 5 * time.Minute, 63 | } 64 | 65 | go func() { 66 | if err := e.StartServer(s); err != nil && err != http.ErrServerClosed { 67 | e.Logger.Fatal(err) 68 | } 69 | }() 70 | 71 | quit := make(chan os.Signal, 1) 72 | signal.Notify(quit, os.Interrupt, syscall.SIGTERM) 73 | 74 | shutdown := func() error { 75 | close(quit) 76 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 77 | defer cancel() 78 | return s.Shutdown(ctx) 79 | } 80 | 81 | return shutdown, nil 82 | } 83 | 84 | func requestToJSON(c echo.Context, reqBody []byte) []byte { 85 | headers := formatHeaders(c.Request().Header) 86 | requestEcho := RequestEcho{ 87 | Time: time.Now().Format(time.RFC3339), 88 | ID: c.Response().Header().Get(echo.HeaderXRequestID), 89 | RemoteIP: c.RealIP(), 90 | Host: c.Request().Host, 91 | Method: c.Request().Method, 92 | URI: c.Request().RequestURI, 93 | UserAgent: c.Request().UserAgent(), 94 | Status: c.Response().Status, 95 | Headers: headers, 96 | RequestBody: string(reqBody), 97 | } 98 | requestEchoJson, _ := json.Marshal(requestEcho) 99 | return requestEchoJson 100 | } 101 | 102 | func formatHeaders(headers http.Header) map[string]string { 103 | formattedHeaders := make(map[string]string) 104 | for key, values := range headers { 105 | formattedHeaders[key] = strings.Join(values, ", ") 106 | } 107 | return formattedHeaders 108 | } 109 | -------------------------------------------------------------------------------- /requests/requests.go: -------------------------------------------------------------------------------- 1 | package requests 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "github.com/tednaleid/ganda/execcontext" 7 | "github.com/tednaleid/ganda/logger" 8 | "github.com/tednaleid/ganda/parser" 9 | "github.com/tednaleid/ganda/responses" 10 | "net/http" 11 | "sync" 12 | "time" 13 | ) 14 | 15 | type HttpClient struct { 16 | MaxRetries int64 17 | Client *http.Client 18 | Logger *logger.LeveledLogger 19 | } 20 | 21 | func NewHttpClient(context *execcontext.Context) *HttpClient { 22 | return &HttpClient{ 23 | MaxRetries: context.Retries, 24 | Logger: context.Logger, 25 | Client: &http.Client{ 26 | Timeout: context.ConnectTimeoutDuration, 27 | Transport: &http.Transport{ 28 | MaxIdleConns: 500, 29 | MaxIdleConnsPerHost: 50, 30 | TLSClientConfig: &tls.Config{ 31 | InsecureSkipVerify: context.Insecure, 32 | }, 33 | }, 34 | }, 35 | } 36 | } 37 | 38 | func StartRequestWorkers( 39 | requestsWithContext <-chan parser.RequestWithContext, 40 | responsesWithContext chan<- *responses.ResponseWithContext, 41 | rateLimitTicker *time.Ticker, 42 | context *execcontext.Context, 43 | ) *sync.WaitGroup { 44 | var requestWaitGroup sync.WaitGroup 45 | requestWaitGroup.Add(context.RequestWorkers) 46 | 47 | for i := 1; i <= context.RequestWorkers; i++ { 48 | go func() { 49 | requestWorker(context, requestsWithContext, responsesWithContext, rateLimitTicker) 50 | requestWaitGroup.Done() 51 | }() 52 | } 53 | 54 | return &requestWaitGroup 55 | } 56 | 57 | func requestWorker( 58 | context *execcontext.Context, 59 | requestsWithContext <-chan parser.RequestWithContext, 60 | responsesWithContext chan<- *responses.ResponseWithContext, 61 | rateLimitTicker *time.Ticker, 62 | ) { 63 | httpClient := NewHttpClient(context) 64 | 65 | for requestWithContext := range requestsWithContext { 66 | if rateLimitTicker != nil { 67 | <-rateLimitTicker.C // wait for the next tick to send the request 68 | } 69 | 70 | finalResponse, err := requestWithRetry(httpClient, requestWithContext, context.BaseRetryDelayDuration) 71 | 72 | if err == nil { 73 | responsesWithContext <- finalResponse 74 | } 75 | } 76 | } 77 | 78 | func requestWithRetry( 79 | httpClient *HttpClient, 80 | requestWithContext parser.RequestWithContext, 81 | baseRetryDelay time.Duration, 82 | ) (*responses.ResponseWithContext, error) { 83 | var response *http.Response 84 | var err error 85 | 86 | for attempts := int64(1); ; attempts++ { 87 | response, err = httpClient.Client.Do(requestWithContext.Request) 88 | 89 | responseWithContext := &responses.ResponseWithContext{ 90 | Response: response, 91 | RequestContext: requestWithContext.RequestContext, 92 | } 93 | 94 | if err == nil && response.StatusCode < 500 { 95 | // return successful response or non-server error, we don't retry those 96 | return responseWithContext, nil 97 | } 98 | 99 | message := requestWithContext.Request.URL.String() 100 | 101 | if err == nil { 102 | httpClient.Logger.LogResponse(response.StatusCode, message) 103 | } else { 104 | httpClient.Logger.LogError(err, message) 105 | } 106 | 107 | if attempts > httpClient.MaxRetries { 108 | return responseWithContext, fmt.Errorf("maximum number of retries (%d) reached for request", httpClient.MaxRetries) 109 | } 110 | 111 | time.Sleep(baseRetryDelay * time.Duration(2^attempts)) 112 | } 113 | 114 | } 115 | -------------------------------------------------------------------------------- /cli/test_helper_test.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "bytes" 5 | ctx "context" 6 | "fmt" 7 | "github.com/stretchr/testify/assert" 8 | "github.com/tednaleid/ganda/execcontext" 9 | "github.com/urfave/cli/v3" 10 | "io" 11 | "net/http" 12 | "net/http/httptest" 13 | "strings" 14 | "testing" 15 | ) 16 | 17 | // test helper structs and functions, no actual tests 18 | 19 | var testBuildInfo = BuildInfo{Version: "testing", Commit: "123abc", Date: "2023-12-20"} 20 | 21 | type GandaResults struct { 22 | stderr string 23 | stdout string 24 | command *cli.Command 25 | } 26 | 27 | func (results *GandaResults) GetContext() *execcontext.Context { 28 | context := results.command.Metadata["context"] 29 | if context == nil { 30 | return nil 31 | } 32 | return context.(*execcontext.Context) 33 | } 34 | 35 | func (results *GandaResults) assert(t *testing.T, expectedStandardOut string, expectedLog string) { 36 | assert.Equal(t, expectedStandardOut, results.stdout, "expected stdout") 37 | assert.Equal(t, expectedLog, results.stderr, "expected logger stderr") 38 | } 39 | 40 | func ParseGandaArgs(args []string) (GandaResults, error) { 41 | // we want to test parsing of arguments, we don't actually want to execute any requests, empty stdin 42 | in := strings.NewReader("") 43 | return RunGanda(args, in) 44 | } 45 | 46 | func RunGanda(args []string, in io.Reader) (GandaResults, error) { 47 | return RunGandaWithContext(args, in, ctx.Background()) 48 | } 49 | 50 | func RunGandaWithContext(args []string, in io.Reader, ctx ctx.Context) (GandaResults, error) { 51 | stderr := new(bytes.Buffer) 52 | stdout := new(bytes.Buffer) 53 | 54 | command := SetupCommand(testBuildInfo, in, stderr, stdout) 55 | 56 | err := command.Run(ctx, args) 57 | 58 | return GandaResults{stderr.String(), stdout.String(), command}, err 59 | } 60 | 61 | type HttpServerStub struct { 62 | *httptest.Server 63 | } 64 | 65 | // The passed in handler function can verify the request and write a response given that input 66 | func NewHttpServerStub(handler http.Handler) *HttpServerStub { 67 | return &HttpServerStub{httptest.NewServer(handler)} 68 | } 69 | 70 | // append the fragment to the end of the server base url 71 | func (server *HttpServerStub) urlFor(fragment string) string { 72 | return fmt.Sprintf("%s/%s", server.URL, fragment) 73 | } 74 | 75 | func (server *HttpServerStub) urlsFor(fragments []string) []string { 76 | urls := make([]string, len(fragments)) 77 | for i, path := range fragments { 78 | urls[i] = server.urlFor(path) 79 | } 80 | return urls 81 | } 82 | 83 | // stub stdin for the path fragment to create an url for this server 84 | func (server *HttpServerStub) stubStdinUrl(fragment string) io.Reader { 85 | return server.stubStdinUrls([]string{fragment}) 86 | } 87 | 88 | // given an array of paths, we will create a stub of stdin that has one url per line for our server stub 89 | func (server *HttpServerStub) stubStdinUrls(fragments []string) io.Reader { 90 | urls := server.urlsFor(fragments) 91 | urlsString := strings.Join(urls, "\n") 92 | return strings.NewReader(urlsString) 93 | } 94 | 95 | func trimmedInputReader(s string) io.Reader { 96 | return strings.NewReader(trimIndent(s)) 97 | } 98 | 99 | func trimIndentKeepTrailingNewline(s string) string { 100 | return trimIndent(s) + "\n" 101 | } 102 | 103 | func trimIndent(s string) string { 104 | lines := strings.Split(s, "\n") 105 | var trimmedLines []string 106 | 107 | for _, line := range lines { 108 | trimmedLine := strings.TrimSpace(line) 109 | if len(trimmedLine) > 0 { 110 | trimmedLines = append(trimmedLines, trimmedLine) 111 | } 112 | } 113 | return strings.Join(trimmedLines, "\n") 114 | } 115 | -------------------------------------------------------------------------------- /echoserver/echoserver_test.go: -------------------------------------------------------------------------------- 1 | package echoserver 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "net" 7 | "net/http" 8 | "strconv" 9 | "strings" 10 | "testing" 11 | ) 12 | 13 | func getAvailablePort() (int, error) { 14 | addr, err := net.ResolveTCPAddr("tcp", "localhost:0") 15 | if err != nil { 16 | return 0, err 17 | } 18 | 19 | l, err := net.ListenTCP("tcp", addr) 20 | if err != nil { 21 | return 0, err 22 | } 23 | defer l.Close() 24 | return l.Addr().(*net.TCPAddr).Port, nil 25 | } 26 | 27 | func withEchoserver(t *testing.T, test func(port int)) { 28 | port, err := getAvailablePort() 29 | delayMillis := 0 30 | if err != nil { 31 | t.Fatal(err) 32 | } 33 | 34 | shutdown, err := Echoserver(int64(port), int64(delayMillis), io.Discard) 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | defer func() { 39 | if err := shutdown(); err != nil { 40 | t.Fatal(err) 41 | } 42 | }() 43 | 44 | test(port) 45 | } 46 | 47 | func TestEchoserverGET(t *testing.T) { 48 | withEchoserver(t, func(port int) { 49 | resp, err := http.Get("http://localhost:" + strconv.Itoa(port) + "/foobar") 50 | if err != nil { 51 | t.Fatal(err) 52 | } 53 | defer resp.Body.Close() 54 | 55 | if resp.StatusCode != http.StatusOK { 56 | t.Errorf("expected status 200, got %d", resp.StatusCode) 57 | } 58 | 59 | body, err := io.ReadAll(resp.Body) 60 | if err != nil { 61 | t.Fatal(err) 62 | } 63 | 64 | var logEntry RequestEcho 65 | if err := json.Unmarshal(body, &logEntry); err != nil { 66 | t.Fatalf("failed to unmarshal response body: %v", err) 67 | } 68 | 69 | if logEntry.URI != "/foobar" { 70 | t.Errorf("expected uri '/foobar', got '%s'", logEntry.URI) 71 | } 72 | 73 | if logEntry.Method != "GET" { 74 | t.Errorf("expected method 'GET', got '%s'", logEntry.Method) 75 | } 76 | 77 | if logEntry.Status != 200 { 78 | t.Errorf("expected status 200, got %d", logEntry.Status) 79 | } 80 | 81 | if logEntry.RequestBody != "" { 82 | t.Errorf("expected request_body to be empty, got '%s'", logEntry.RequestBody) 83 | } 84 | 85 | if logEntry.Headers["User-Agent"] != "Go-http-client/1.1" { 86 | t.Errorf("expected User-Agent header to be set") 87 | } 88 | }) 89 | } 90 | 91 | func TestEchoserverPOST(t *testing.T) { 92 | withEchoserver(t, func(port int) { 93 | jsonBody := `{"foo":"bar", "baz":[1, 2, 3]}` 94 | reader := strings.NewReader(jsonBody) 95 | 96 | resp, err := http.Post("http://localhost:"+strconv.Itoa(port)+"/foobar", "application/json", reader) 97 | if err != nil { 98 | t.Fatal(err) 99 | } 100 | defer resp.Body.Close() 101 | 102 | if resp.StatusCode != http.StatusOK { 103 | t.Errorf("expected status 200, got %d", resp.StatusCode) 104 | } 105 | 106 | body, err := io.ReadAll(resp.Body) 107 | if err != nil { 108 | t.Fatal(err) 109 | } 110 | 111 | var logEntry RequestEcho 112 | if err := json.Unmarshal(body, &logEntry); err != nil { 113 | t.Fatalf("failed to unmarshal response body: %v", err) 114 | } 115 | 116 | if logEntry.URI != "/foobar" { 117 | t.Errorf("expected uri '/foobar', got '%s'", logEntry.URI) 118 | } 119 | 120 | if logEntry.Method != "POST" { 121 | t.Errorf("expected method 'GET', got '%s'", logEntry.Method) 122 | } 123 | 124 | if logEntry.Status != 200 { 125 | t.Errorf("expected status 200, got %d", logEntry.Status) 126 | } 127 | 128 | if logEntry.RequestBody != jsonBody { 129 | t.Errorf("expected request_body to be empty, got '%s'", logEntry.RequestBody) 130 | } 131 | 132 | if logEntry.Headers["User-Agent"] != "Go-http-client/1.1" { 133 | t.Errorf("expected User-Agent header to be set") 134 | } 135 | }) 136 | } 137 | -------------------------------------------------------------------------------- /cli/cli_echoserver_test.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/tednaleid/ganda/echoserver" 8 | "github.com/urfave/cli/v3" 9 | "golang.org/x/net/context" 10 | "io" 11 | "net" 12 | "strconv" 13 | "strings" 14 | "testing" 15 | ) 16 | 17 | func TestEchoserverDefaultPort(t *testing.T) { 18 | shutdownFunc := RunGandaAsync([]string{"ganda", "echoserver"}, nil) 19 | 20 | // asserts that the port is open 21 | waitForPort(8080) 22 | 23 | // we aren't doing anything with the server, just wanted it to start up 24 | results := shutdownFunc() 25 | assert.NotNil(t, results) 26 | subcommand := FindSubcommand(results.command, "echoserver") 27 | assert.NotNil(t, subcommand) 28 | assert.Equal(t, subcommand.Name, "echoserver") 29 | assert.Equal(t, subcommand.Int("port"), int64(8080)) 30 | } 31 | 32 | func TestEchoserverOverridePort(t *testing.T) { 33 | port := 9090 34 | shutdownFunc := RunGandaAsync([]string{"ganda", "echoserver", "--port", strconv.Itoa(port)}, nil) 35 | 36 | // asserts that the port is open 37 | waitForPort(port) 38 | 39 | // we aren't doing anything with the server, just wanted it to start up 40 | results := shutdownFunc() 41 | assert.NotNil(t, results) 42 | subcommand := FindSubcommand(results.command, "echoserver") 43 | assert.NotNil(t, subcommand) 44 | assert.Equal(t, subcommand.Name, "echoserver") 45 | assert.Equal(t, subcommand.Int("port"), int64(port)) 46 | } 47 | 48 | // Runs the Echoserver and then runs ganda against it 49 | func TestAllTogetherNow(t *testing.T) { 50 | port := 9090 51 | shutdownFunc := RunGandaAsync([]string{"ganda", "echoserver", "--port", strconv.Itoa(port)}, nil) 52 | 53 | waitForPort(port) 54 | 55 | url := fmt.Sprintf("http://localhost:%d/hello/world", port) 56 | 57 | runResults, _ := RunGanda([]string{"ganda"}, strings.NewReader(url+"\n")) 58 | 59 | assert.Equal(t, "Response: 200 "+url+"\n", runResults.stderr, "expected logger stderr") 60 | 61 | var logEntry echoserver.RequestEcho 62 | if err := json.Unmarshal([]byte(runResults.stdout), &logEntry); err != nil { 63 | t.Fatalf("failed to unmarshal response body: %v", err) 64 | } 65 | 66 | assert.Equal(t, "GET", logEntry.Method, "expected method") 67 | assert.Equal(t, "/hello/world", logEntry.URI, "expected URI") 68 | assert.Equal(t, "Go-http-client/1.1", logEntry.UserAgent, "expected user agent") 69 | assert.Equal(t, 200, logEntry.Status, "expected status") 70 | assert.Contains(t, logEntry.Headers, "Accept-Encoding", "expected headers to contain Accept-Encoding") 71 | assert.Contains(t, logEntry.Headers, "User-Agent", "expected headers to contain User-Agent") 72 | 73 | shutdownFunc() 74 | } 75 | 76 | // RunGandaAsync will run ganda in a separate goroutine and return a function that can 77 | // be called to cancel the ganda run and return the results 78 | func RunGandaAsync(args []string, in io.Reader) func() GandaResults { 79 | resultsChan := make(chan GandaResults, 1) 80 | ctx, cancelFunc := context.WithCancel(context.Background()) 81 | 82 | go func() { 83 | results, err := RunGandaWithContext(args, in, ctx) 84 | if err != nil { 85 | results.stderr = fmt.Sprintf("RunGandaWithContext failed: %v", err) 86 | } 87 | resultsChan <- results 88 | close(resultsChan) 89 | }() 90 | 91 | return func() GandaResults { 92 | cancelFunc() 93 | result := <-resultsChan 94 | return result 95 | } 96 | } 97 | 98 | // func to check if an int port argument is open in a spin loop and will return when it is 99 | func waitForPort(port int) { 100 | for { 101 | conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", port)) 102 | if err == nil { 103 | conn.Close() 104 | break 105 | } 106 | } 107 | } 108 | 109 | func FindSubcommand(c *cli.Command, name string) *cli.Command { 110 | for _, cmd := range c.Commands { 111 | if cmd.Name == name { 112 | return cmd 113 | } 114 | } 115 | return nil 116 | } 117 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 2 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 3 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= 5 | github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= 6 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 7 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 8 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 9 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 10 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 11 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 12 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 13 | github.com/labstack/echo/v4 v4.11.4 h1:vDZmA+qNeh1pd/cCkEicDMrjtrnMGQ1QFI9gWN1zGq8= 14 | github.com/labstack/echo/v4 v4.11.4/go.mod h1:noh7EvLwqDsmh/X/HWKPUl1AjzJrhyptRyEbQJfxen8= 15 | github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= 16 | github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= 17 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 18 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 19 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 20 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 21 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 22 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 23 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 24 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 25 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 26 | github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= 27 | github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= 28 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 29 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 30 | github.com/urfave/cli/v3 v3.0.0-alpha9 h1:P0RMy5fQm1AslQS+XCmy9UknDXctOmG/q/FZkUFnJSo= 31 | github.com/urfave/cli/v3 v3.0.0-alpha9/go.mod h1:0kK/RUFHyh+yIKSfWxwheGndfnrvYSmYFVeKCh03ZUc= 32 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 33 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 34 | github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= 35 | github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= 36 | github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI= 37 | github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= 38 | golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= 39 | golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= 40 | golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= 41 | golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= 42 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 43 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 44 | golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= 45 | golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 46 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 47 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 48 | golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= 49 | golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 50 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 51 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 52 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 53 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 54 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 55 | -------------------------------------------------------------------------------- /cli/cli_test.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "net/http" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestRequestHappyPathHasDefaultHeaders(t *testing.T) { 12 | t.Parallel() 13 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 14 | // default headers added by http client 15 | assert.Equal(t, r.Header["User-Agent"][0], "Go-http-client/1.1", "User-Agent header") 16 | assert.Equal(t, r.Header["Connection"][0], "keep-alive", "Connection header") 17 | assert.Equal(t, r.Header["Accept-Encoding"][0], "gzip", "Accept-Encoding header") 18 | fmt.Fprint(w, "Hello ", r.URL.Path) 19 | })) 20 | defer server.Close() 21 | 22 | runResults, _ := RunGanda([]string{"ganda"}, server.stubStdinUrl("foo/1")) 23 | 24 | runResults.assert( 25 | t, 26 | "Hello /foo/1\n", 27 | "Response: 200 "+server.urlFor("foo/1")+"\n", 28 | ) 29 | } 30 | 31 | func TestTimeout(t *testing.T) { 32 | t.Parallel() 33 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 34 | time.Sleep(10 * time.Millisecond) 35 | fmt.Fprint(w, "Should not get this, should time out first") 36 | })) 37 | defer server.Server.Close() 38 | 39 | runResults, _ := RunGanda([]string{"ganda", "--connect-timeout-millis", "1"}, server.stubStdinUrl("bar")) 40 | 41 | url := server.urlFor("bar") 42 | 43 | runResults.assert( 44 | t, 45 | "", 46 | url+" Error: Get \""+url+"\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\n", 47 | ) 48 | } 49 | 50 | func TestRetryEnabledShouldRetry5XX(t *testing.T) { 51 | t.Parallel() 52 | requests := 0 53 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 54 | requests++ 55 | if requests == 1 { 56 | w.WriteHeader(500) 57 | } else { 58 | fmt.Fprint(w, "Retried request") 59 | } 60 | })) 61 | defer server.Server.Close() 62 | 63 | runResults, _ := RunGanda([]string{"ganda", "--retry", "1", "--base-retry-millis", "1"}, server.stubStdinUrl("bar")) 64 | 65 | url := server.urlFor("bar") 66 | 67 | assert.Equal(t, 2, requests, "expected a failed request followed by a successful one") 68 | runResults.assert( 69 | t, 70 | "Retried request\n", 71 | "Response: 500 "+url+"\nResponse: 200 "+url+"\n", 72 | ) 73 | } 74 | 75 | func TestRunningOutOfRetriesShouldStopProcessing(t *testing.T) { 76 | t.Parallel() 77 | requests := 0 78 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 79 | requests++ 80 | w.WriteHeader(500) 81 | })) 82 | defer server.Server.Close() 83 | 84 | runResults, _ := RunGanda([]string{"ganda", "--retry", "2", "--base-retry-millis", "1"}, server.stubStdinUrl("bar")) 85 | 86 | url := server.urlFor("bar") 87 | 88 | assert.Equal(t, 3, requests, "3 total requests (original and 2 retries), all failed so expecting error") 89 | runResults.assert( 90 | t, 91 | "", 92 | "Response: 500 "+url+"\nResponse: 500 "+url+"\nResponse: 500 "+url+"\n", 93 | ) 94 | } 95 | 96 | func TestRetryEnabledShouldNotRetry4XX(t *testing.T) { 97 | t.Parallel() 98 | requestCount := 0 99 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 100 | requestCount++ 101 | w.WriteHeader(400) 102 | })) 103 | defer server.Server.Close() 104 | 105 | runResults, _ := RunGanda([]string{"ganda", "--retry", "1", "--base-retry-millis", "1"}, server.stubStdinUrl("bar")) 106 | 107 | url := server.urlFor("bar") 108 | 109 | assert.Equal(t, 1, requestCount, "we shouldn't retry 4xx errors") 110 | runResults.assert(t, 111 | "", 112 | "Response: 400 "+url+"\n") 113 | } 114 | 115 | func TestRetryEnabledShouldRetryTimeout(t *testing.T) { 116 | t.Parallel() 117 | requestCount := 0 118 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 119 | requestCount++ 120 | if requestCount == 1 { 121 | // for the first request, we sleep longer than it takes to timeout 122 | time.Sleep(20 * time.Millisecond) 123 | } 124 | fmt.Fprint(w, "Request ", requestCount) 125 | })) 126 | defer server.Server.Close() 127 | 128 | runResults, _ := RunGanda([]string{"ganda", "--connect-timeout-millis", "10", "--retry", "1", "--base-retry-millis", "1"}, server.stubStdinUrl("bar")) 129 | url := server.urlFor("bar") 130 | 131 | //assert.Equal(t, 2, requestCount, "expected a second request") 132 | runResults.assert(t, 133 | "Request 2\n", 134 | url+" Error: Get \""+url+"\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\nResponse: 200 "+url+"\n") 135 | } 136 | 137 | func TestAddHeadersToRequestCreatesCanonicalKeys(t *testing.T) { 138 | t.Parallel() 139 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 140 | // turns to uppercase versions for header key when transmitted 141 | assert.Equal(t, r.Header["Foo"][0], "bar", "foo header") 142 | assert.Equal(t, r.Header["X-Baz"][0], "qux", "baz header") 143 | fmt.Fprint(w, "Hello ", r.URL.Path) 144 | })) 145 | defer server.Server.Close() 146 | 147 | runResults, _ := RunGanda([]string{"ganda", "-H", "foo: bar", "-H", "x-baz: qux"}, server.stubStdinUrl("bar")) 148 | url := server.urlFor("bar") 149 | 150 | runResults.assert(t, 151 | "Hello /bar\n", 152 | "Response: 200 "+url+"\n") 153 | } 154 | -------------------------------------------------------------------------------- /parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/base64" 7 | "encoding/csv" 8 | "encoding/json" 9 | "fmt" 10 | "github.com/tednaleid/ganda/config" 11 | "io" 12 | "net/http" 13 | "strconv" 14 | "strings" 15 | ) 16 | 17 | type InputType int 18 | 19 | const ( 20 | Unknown InputType = iota 21 | Urls 22 | JsonLines 23 | ) 24 | 25 | type RequestWithContext struct { 26 | Request *http.Request 27 | RequestContext interface{} 28 | } 29 | 30 | func SendRequests( 31 | requestsWithContext chan<- RequestWithContext, 32 | in io.Reader, 33 | requestMethod string, 34 | staticHeaders []config.RequestHeader, 35 | ) error { 36 | reader := bufio.NewReader(in) 37 | inputType, _ := determineInputType(reader) 38 | 39 | if inputType == JsonLines { 40 | return SendJsonLinesRequests(requestsWithContext, reader, requestMethod, staticHeaders) 41 | } 42 | 43 | return SendUrlsRequests(requestsWithContext, reader, requestMethod, staticHeaders) 44 | } 45 | 46 | // Each line is an URL and optionally some TSV context that can be passed through 47 | // an emitted along with the response output 48 | func SendUrlsRequests( 49 | requestsWithContext chan<- RequestWithContext, 50 | reader *bufio.Reader, 51 | requestMethod string, 52 | staticHeaders []config.RequestHeader, 53 | ) error { 54 | tsvReader := csv.NewReader(reader) 55 | tsvReader.Comma = '\t' 56 | tsvReader.FieldsPerRecord = -1 57 | 58 | for { 59 | record, err := tsvReader.Read() 60 | if err == io.EOF { 61 | break 62 | } else if err != nil { 63 | return err 64 | } 65 | 66 | if len(record) > 0 { 67 | url := record[0] 68 | request := createRequest(url, nil, requestMethod, staticHeaders) 69 | recordContext := record[1:] 70 | 71 | if len(recordContext) == 0 { 72 | recordContext = nil 73 | } 74 | 75 | requestsWithContext <- RequestWithContext{Request: request, RequestContext: recordContext} 76 | } 77 | } 78 | return nil 79 | } 80 | 81 | type JsonLine struct { 82 | URL string `json:"url"` 83 | Method string `json:"method"` 84 | Context interface{} `json:"context"` 85 | Headers map[string]string `json:"headers"` 86 | Body json.RawMessage `json:"body"` 87 | BodyType string `json:"bodyType"` 88 | } 89 | 90 | func SendJsonLinesRequests( 91 | requestsWithContext chan<- RequestWithContext, 92 | reader *bufio.Reader, 93 | requestMethod string, 94 | staticHeaders []config.RequestHeader, 95 | ) error { 96 | scanner := bufio.NewScanner(reader) 97 | 98 | for scanner.Scan() { 99 | line := scanner.Text() 100 | var jsonLine JsonLine 101 | 102 | err := json.Unmarshal([]byte(line), &jsonLine) 103 | if err != nil { 104 | return fmt.Errorf("%s: %s", err.Error(), line) 105 | } else if jsonLine.URL == "" { 106 | return fmt.Errorf("missing url property: %s", line) 107 | } 108 | 109 | body, err := parseBody(jsonLine.BodyType, jsonLine.Body) 110 | if err != nil { 111 | return fmt.Errorf("failed to parse body: %s", err) 112 | } 113 | 114 | // allow overriding of the request method per JSON line, but otherwise use the default 115 | method := requestMethod 116 | if jsonLine.Method != "" { 117 | method = jsonLine.Method 118 | } 119 | 120 | mergedHeaders := mergeHeaders(staticHeaders, jsonLine.Headers) 121 | 122 | request := createRequest(jsonLine.URL, body, method, mergedHeaders) 123 | requestsWithContext <- RequestWithContext{Request: request, RequestContext: jsonLine.Context} 124 | } 125 | 126 | if err := scanner.Err(); err != nil { 127 | return err 128 | } 129 | 130 | return nil 131 | } 132 | 133 | func mergeHeaders(staticHeaders []config.RequestHeader, jsonLineHeaders map[string]string) []config.RequestHeader { 134 | if len(jsonLineHeaders) == 0 { 135 | return staticHeaders 136 | } 137 | 138 | headersMap := make(map[string]string) 139 | for _, header := range staticHeaders { 140 | headersMap[header.Key] = header.Value 141 | } 142 | 143 | for key, value := range jsonLineHeaders { 144 | headersMap[key] = value 145 | } 146 | 147 | mergedHeaders := make([]config.RequestHeader, 0, len(headersMap)) 148 | for key, value := range headersMap { 149 | mergedHeaders = append(mergedHeaders, config.RequestHeader{Key: key, Value: value}) 150 | } 151 | 152 | return mergedHeaders 153 | } 154 | 155 | func parseBody(bodyType string, body json.RawMessage) (io.ReadCloser, error) { 156 | switch bodyType { 157 | case "escaped": 158 | str, err := strconv.Unquote(string(body)) 159 | if err != nil { 160 | return nil, err 161 | } 162 | return io.NopCloser(strings.NewReader(str)), nil 163 | case "base64": 164 | unquoted, err := strconv.Unquote(string(body)) 165 | data, err := base64.StdEncoding.DecodeString(unquoted) 166 | if err != nil { 167 | return nil, err 168 | } 169 | return io.NopCloser(bytes.NewReader(data)), nil 170 | case "json", "": 171 | // Use the JSON as is 172 | return io.NopCloser(bytes.NewReader(body)), nil 173 | default: 174 | return nil, fmt.Errorf("unsupported body type: %s, valid values: \"json\", \"base64\", \"escaped\"", bodyType) 175 | } 176 | } 177 | 178 | // current assumption is that the first character is '{' for a stream of json lines, 179 | // otherwise it's a stream of urls 180 | func determineInputType(bufferedReader *bufio.Reader) (InputType, error) { 181 | initialByte, err := bufferedReader.Peek(1) 182 | 183 | if err != nil { 184 | return Unknown, err 185 | } 186 | 187 | if initialByte[0] == '{' { 188 | return JsonLines, nil 189 | } 190 | 191 | return Urls, nil 192 | } 193 | 194 | func createRequest(url string, body io.Reader, requestMethod string, requestHeaders []config.RequestHeader) *http.Request { 195 | request, err := http.NewRequest(requestMethod, url, body) 196 | 197 | if err != nil { 198 | panic(err) 199 | } 200 | 201 | request.Header.Add("connection", "keep-alive") 202 | 203 | for _, header := range requestHeaders { 204 | request.Header.Add(header.Key, header.Value) 205 | } 206 | 207 | return request 208 | } 209 | -------------------------------------------------------------------------------- /cli/cli_response_output_test.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "fmt" 5 | "github.com/tednaleid/ganda/config" 6 | "net/http" 7 | "testing" 8 | ) 9 | 10 | func TestRequestColorOutput(t *testing.T) { 11 | t.Parallel() 12 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 13 | fmt.Fprint(w, "Hello ", r.URL.Path) 14 | })) 15 | defer server.Close() 16 | 17 | runResults, _ := RunGanda([]string{"ganda", "--color"}, server.stubStdinUrl("foo/1")) 18 | 19 | runResults.assert( 20 | t, 21 | "Hello /foo/1\n", 22 | "\x1b[32mResponse: 200 "+server.urlFor("foo/1")+"\x1b[0m\n", 23 | ) 24 | } 25 | 26 | func TestSilentOutput(t *testing.T) { 27 | t.Parallel() 28 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 29 | fmt.Fprint(w, "Hello ", r.URL.Path) 30 | })) 31 | defer server.Close() 32 | 33 | runResults, _ := RunGanda([]string{"ganda", "-s"}, server.stubStdinUrl("foo/1")) 34 | 35 | runResults.assert( 36 | t, 37 | "Hello /foo/1\n", 38 | "", 39 | ) 40 | } 41 | 42 | func TestResponseBody(t *testing.T) { 43 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 44 | fmt.Fprint(w, "Hello ", r.URL.Path) 45 | })) 46 | defer server.Server.Close() 47 | 48 | testCases := []struct { 49 | name string 50 | responseBody config.ResponseBodyType 51 | expected string 52 | }{ 53 | {"raw", config.Raw, "Hello /bar\n"}, 54 | {"discard", config.Discard, ""}, 55 | {"escaped", config.Escaped, "\"Hello /bar\"\n"}, 56 | {"base64", config.Base64, "SGVsbG8gL2Jhcg==\n"}, 57 | {"sha256", config.Sha256, "13a05f3ce0f3edc94bdeee3783c969dfb27c234b6dd98ce7fd004ffc69a45ece\n"}, 58 | } 59 | 60 | for _, tc := range testCases { 61 | t.Run(tc.name, func(t *testing.T) { 62 | runResults, _ := RunGanda([]string{"ganda", "-B", tc.name}, server.stubStdinUrl("bar")) 63 | url := server.urlFor("bar") 64 | 65 | runResults.assert(t, tc.expected, "Response: 200 "+url+"\n") 66 | }) 67 | } 68 | } 69 | 70 | func TestResponseBodyWithJsonEnvelope(t *testing.T) { 71 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 72 | fmt.Fprint(w, "{ \"foo\": \"", r.URL.Path+"\" }") 73 | })) 74 | defer server.Server.Close() 75 | 76 | testCases := []struct { 77 | name string 78 | responseBody config.ResponseBodyType 79 | expected string 80 | }{ 81 | {"raw", config.Raw, "{ \"url\": \"" + server.urlFor("bar") + "\", \"code\": 200, \"body\": { \"foo\": \"/bar\" } }\n"}, 82 | {"discard", config.Discard, "{ \"url\": \"" + server.urlFor("bar") + "\", \"code\": 200, \"body\": null }\n"}, 83 | {"escaped", config.Escaped, "{ \"url\": \"" + server.urlFor("bar") + "\", \"code\": 200, \"body\": \"{ \\\"foo\\\": \\\"/bar\\\" }\" }\n"}, 84 | {"base64", config.Base64, "{ \"url\": \"" + server.urlFor("bar") + "\", \"code\": 200, \"body\": \"eyAiZm9vIjogIi9iYXIiIH0=\" }\n"}, 85 | {"sha256", config.Sha256, "{ \"url\": \"" + server.urlFor("bar") + "\", \"code\": 200, \"body\": \"f660cd1420c6acd9408932b9983909c26ab6cb21ffb40525670a7b7aa67092ec\" }\n"}, 86 | } 87 | 88 | for _, tc := range testCases { 89 | t.Run(tc.name, func(t *testing.T) { 90 | runResults, _ := RunGanda([]string{"ganda", "-J", "-B", tc.name}, server.stubStdinUrl("bar")) 91 | url := server.urlFor("bar") 92 | 93 | runResults.assert(t, tc.expected, "Response: 200 "+url+"\n") 94 | }) 95 | } 96 | } 97 | 98 | func TestErrorWithJsonEnvelope(t *testing.T) { 99 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 100 | w.WriteHeader(404) 101 | })) 102 | defer server.Server.Close() 103 | 104 | testCases := []struct { 105 | name string 106 | responseBody config.ResponseBodyType 107 | expected string 108 | }{ 109 | {"raw", config.Raw, "{ \"url\": \"" + server.urlFor("bar") + "\", \"code\": 404, \"body\": null }\n"}, 110 | {"discard", config.Discard, "{ \"url\": \"" + server.urlFor("bar") + "\", \"code\": 404, \"body\": null }\n"}, 111 | {"escaped", config.Escaped, "{ \"url\": \"" + server.urlFor("bar") + "\", \"code\": 404, \"body\": null }\n"}, 112 | {"base64", config.Base64, "{ \"url\": \"" + server.urlFor("bar") + "\", \"code\": 404, \"body\": null }\n"}, 113 | {"sha256", config.Sha256, "{ \"url\": \"" + server.urlFor("bar") + "\", \"code\": 404, \"body\": null }\n"}, 114 | } 115 | 116 | for _, tc := range testCases { 117 | t.Run(tc.name, func(t *testing.T) { 118 | runResults, _ := RunGanda([]string{"ganda", "-J", "-B", tc.name}, server.stubStdinUrl("bar")) 119 | url := server.urlFor("bar") 120 | 121 | runResults.assert(t, tc.expected, "Response: 404 "+url+"\n") 122 | }) 123 | } 124 | } 125 | 126 | func TestJsonLinesContextWithJsonEnvelope(t *testing.T) { 127 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 128 | fmt.Fprint(w, "") 129 | })) 130 | defer server.Server.Close() 131 | 132 | url := server.urlFor("bar") 133 | 134 | inputLines := ` 135 | { "url": "` + url + `", "context": ["foo", "quoted content"] } 136 | { "url": "` + url + `", "method": "POST", "context": { "quux": " \"quoted with whitespace\" ", "corge": 456 } } 137 | { "url": "` + url + `", "method": "DELETE", "context": "baz" } 138 | ` 139 | 140 | runResults, _ := RunGanda([]string{"ganda", "-J"}, trimmedInputReader(inputLines)) 141 | 142 | expectedOutput := trimIndentKeepTrailingNewline(` 143 | { "url": "` + url + `", "code": 200, "body": null, "context": ["foo","quoted content"] } 144 | { "url": "` + url + `", "code": 200, "body": null, "context": {"corge":456,"quux":" \"quoted with whitespace\" "} } 145 | { "url": "` + url + `", "code": 200, "body": null, "context": "baz" } 146 | `) 147 | 148 | expectedLog := trimIndentKeepTrailingNewline(` 149 | Response: 200 ` + url + ` 150 | Response: 200 ` + url + ` 151 | Response: 200 ` + url + ` 152 | `) 153 | 154 | runResults.assert(t, expectedOutput, expectedLog) 155 | } 156 | 157 | func TestErrorResponse(t *testing.T) { 158 | t.Parallel() 159 | server := NewHttpServerStub(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 160 | w.WriteHeader(404) 161 | })) 162 | defer server.Close() 163 | 164 | runResults, _ := RunGanda([]string{"ganda", "-J"}, server.stubStdinUrl("bar")) 165 | 166 | runResults.assert( 167 | t, 168 | "{ \"url\": \""+server.urlFor("bar")+"\", \"code\": 404, \"body\": null }\n", 169 | "Response: 404 "+server.urlFor("bar")+"\n", 170 | ) 171 | } 172 | 173 | // TODO test the file saving version of this 174 | -------------------------------------------------------------------------------- /cli/cli.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | ctx "context" 5 | "fmt" 6 | "github.com/tednaleid/ganda/config" 7 | "github.com/tednaleid/ganda/echoserver" 8 | "github.com/tednaleid/ganda/execcontext" 9 | "github.com/tednaleid/ganda/parser" 10 | "github.com/tednaleid/ganda/requests" 11 | "github.com/tednaleid/ganda/responses" 12 | "github.com/urfave/cli/v3" 13 | "io" 14 | "math" 15 | "os" 16 | "os/signal" 17 | "syscall" 18 | "time" 19 | ) 20 | 21 | type BuildInfo struct { 22 | Version string 23 | Commit string 24 | Date string 25 | } 26 | 27 | func (buildInfo BuildInfo) ToString() string { 28 | return buildInfo.Version + " " + buildInfo.Commit + " " + buildInfo.Date 29 | } 30 | 31 | // SetupCommand creates the cli.Command so it is wired up with the given in/stdout/stderr 32 | func SetupCommand( 33 | buildInfo BuildInfo, 34 | in io.Reader, 35 | stderr io.Writer, 36 | stdout io.Writer, 37 | ) *cli.Command { 38 | conf := config.New() 39 | 40 | command := cli.Command{ 41 | Name: "ganda", 42 | Usage: "make http requests in parallel", 43 | Authors: []any{ 44 | "Ted Naleid ", 45 | }, 46 | UsageText: " | ganda [options]", 47 | Description: "Pipe urls to ganda over stdout to make http requests to each url in parallel.", 48 | Version: buildInfo.ToString(), 49 | Reader: in, 50 | Writer: stdout, 51 | ErrWriter: stderr, 52 | Flags: []cli.Flag{ 53 | &cli.IntFlag{ 54 | Name: "base-retry-millis", 55 | Usage: "the base number of milliseconds to wait before retrying a request, exponential backoff is used for retries", 56 | Value: conf.BaseRetryDelayMillis, 57 | Destination: &conf.BaseRetryDelayMillis, 58 | }, 59 | &cli.StringFlag{ 60 | Name: "response-body", 61 | Aliases: []string{"B"}, 62 | DefaultText: "raw", 63 | Usage: "transforms the body of the response. Values: 'raw' (unchanged), 'base64', 'discard' (don't emit body), 'escaped' (JSON escaped string), 'sha256'", 64 | // we are slightly abusing the validator as a setter because v3 of urfave/cli doesn't currently support generic flags 65 | Validator: func(s string) error { 66 | switch s { 67 | case "", string(config.Raw): 68 | conf.ResponseBody = config.Raw 69 | case string(config.Base64): 70 | conf.ResponseBody = config.Base64 71 | case string(config.Discard): 72 | conf.ResponseBody = config.Discard 73 | case string(config.Escaped): 74 | conf.ResponseBody = config.Escaped 75 | case string(config.Sha256): 76 | conf.ResponseBody = config.Sha256 77 | return nil 78 | default: 79 | return fmt.Errorf("invalid response-body value: %s", s) 80 | } 81 | return nil 82 | }, 83 | }, 84 | &cli.IntFlag{ 85 | Name: "connect-timeout-millis", 86 | Usage: "number of milliseconds to wait for a connection to be established before timeout", 87 | Value: conf.ConnectTimeoutMillis, 88 | Destination: &conf.ConnectTimeoutMillis, 89 | }, 90 | 91 | &cli.StringSliceFlag{ 92 | Name: "header", 93 | Aliases: []string{"H"}, 94 | Usage: "headers to send with every request, can be used multiple times (gzip and keep-alive are already there)", 95 | }, 96 | &cli.BoolFlag{ 97 | Name: "insecure", 98 | Aliases: []string{"k"}, 99 | Usage: "if flag is present, skip verification of https certificates", 100 | Destination: &conf.Insecure, 101 | }, 102 | &cli.BoolFlag{ 103 | Name: "json-envelope", 104 | Aliases: []string{"J"}, 105 | Usage: "emit result with JSON envelope with url, status, length, and body fields, assumes result is valid json", 106 | Destination: &conf.JsonEnvelope, 107 | }, 108 | &cli.BoolFlag{ 109 | Name: "color", 110 | Usage: "if flag is present, add color to success/warn messages", 111 | Destination: &conf.Color, 112 | }, 113 | &cli.StringFlag{ 114 | Name: "output-directory", 115 | Usage: "if flag is present, save response bodies to files in the specified directory", 116 | Destination: &conf.BaseDirectory, 117 | }, 118 | &cli.StringFlag{ 119 | Name: "request", 120 | Aliases: []string{"X"}, 121 | Value: conf.RequestMethod, 122 | Usage: "HTTP request method to use", 123 | Destination: &conf.RequestMethod, 124 | }, 125 | &cli.IntFlag{ 126 | Name: "retry", 127 | Usage: "max number of retries on transient errors (5XX status codes/timeouts) to attempt", 128 | Value: conf.Retries, 129 | Destination: &conf.Retries, 130 | }, 131 | &cli.BoolFlag{ 132 | Name: "silent", 133 | Aliases: []string{"s"}, 134 | Usage: "if flag is present, omit showing response code for each url only output response bodies", 135 | Destination: &conf.Silent, 136 | }, 137 | &cli.IntFlag{ 138 | Name: "subdir-length", 139 | Usage: "length of hashed subdirectory name to put saved files when using --output-directory; use 2 for > 5k urls, 4 for > 5M urls", 140 | Value: conf.SubdirLength, 141 | Destination: &conf.SubdirLength, 142 | }, 143 | &cli.IntFlag{ 144 | Name: "throttle-per-second", 145 | Usage: "max number of requests to process per second, default is unlimited", 146 | Value: -1, 147 | Destination: &conf.ThrottlePerSecond, 148 | }, 149 | &WorkerFlag{ 150 | Name: "workers", 151 | Aliases: []string{"W"}, 152 | Usage: "number of concurrent workers that will be making requests, increase this for more requests in parallel", 153 | Value: conf.RequestWorkers, 154 | Destination: &conf.RequestWorkers, 155 | }, 156 | }, 157 | Commands: []*cli.Command{ 158 | { 159 | Name: "echoserver", 160 | Usage: "Starts an echo server, --port to override the default port of 8080", 161 | Flags: []cli.Flag{ 162 | &cli.IntFlag{ 163 | Name: "port", 164 | Usage: "Port number to start the echo server on", 165 | Value: 8080, // Default port number 166 | }, 167 | &cli.IntFlag{ 168 | Name: "delay-millis", 169 | Usage: "Number of milliseconds to delay responding", 170 | Value: 0, // Default delay is 0 milliseconds 171 | }, 172 | }, 173 | Action: func(ctx ctx.Context, cmd *cli.Command) error { 174 | port := cmd.Int("port") 175 | delayMillis := cmd.Int("delay-millis") 176 | shutdown, err := echoserver.Echoserver(port, delayMillis, io.Writer(os.Stdout)) 177 | if err != nil { 178 | fmt.Println("Error starting server:", err) 179 | os.Exit(1) 180 | } 181 | 182 | // Wait until an interrupt signal is received, or the context is cancelled 183 | quit := make(chan os.Signal, 1) 184 | signal.Notify(quit, os.Interrupt, syscall.SIGTERM) 185 | 186 | select { 187 | case <-quit: 188 | fmt.Println("Received interrupt signal, shutting down.") 189 | case <-ctx.Done(): 190 | fmt.Println("Context cancelled, shutting down.") 191 | } 192 | 193 | fmt.Println("Shutting echoserver down.") 194 | 195 | return shutdown() 196 | }, 197 | }, 198 | }, 199 | Before: func(_ ctx.Context, cmd *cli.Command) error { 200 | var err error 201 | 202 | if cmd.Args().Present() && cmd.Args().First() != "help" && 203 | cmd.Args().First() != "h" && cmd.Args().First() != "echoserver" { 204 | conf.RequestFilename = cmd.Args().First() 205 | } 206 | 207 | conf.RequestHeaders, err = config.ConvertRequestHeaders(cmd.StringSlice("header")) 208 | 209 | if err != nil { 210 | return err 211 | } 212 | 213 | // convert the conf into a context that has resolved/converted values that we want to 214 | // use when processing. Store in metadata so we can access it in the action 215 | cmd.Metadata["context"], err = execcontext.New(conf, in, stderr, stdout) 216 | 217 | return err 218 | }, 219 | Action: func(_ ctx.Context, cmd *cli.Command) error { 220 | context := cmd.Metadata["context"].(*execcontext.Context) 221 | ProcessRequests(context) 222 | return nil 223 | }, 224 | } 225 | 226 | return &command 227 | } 228 | 229 | // ProcessRequests wires up the request and response workers with channels 230 | // and asks the parser to start sending requests 231 | func ProcessRequests(context *execcontext.Context) { 232 | requestsWithContextChannel := make(chan parser.RequestWithContext) 233 | responsesWithContextChannel := make(chan *responses.ResponseWithContext) 234 | 235 | var rateLimitTicker *time.Ticker 236 | 237 | // don't throttle if we're not limiting the number of requests per second 238 | if context.ThrottlePerSecond != math.MaxInt32 { 239 | rateLimitTicker = time.NewTicker(time.Second / time.Duration(context.ThrottlePerSecond)) 240 | defer rateLimitTicker.Stop() 241 | } 242 | 243 | requestWaitGroup := requests.StartRequestWorkers(requestsWithContextChannel, responsesWithContextChannel, rateLimitTicker, context) 244 | responseWaitGroup := responses.StartResponseWorkers(responsesWithContextChannel, context) 245 | 246 | err := parser.SendRequests(requestsWithContextChannel, context.In, context.RequestMethod, context.RequestHeaders) 247 | 248 | if err != nil { 249 | context.Logger.LogError(err, "error parsing requests") 250 | } 251 | 252 | close(requestsWithContextChannel) 253 | requestWaitGroup.Wait() 254 | 255 | close(responsesWithContextChannel) 256 | responseWaitGroup.Wait() 257 | } 258 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # ganda - High-Performance HTTP Request CLI 3 | 4 | ## Overview 5 | 6 | `ganda` lets you make HTTP/HTTPS requests to hundreds to millions of URLs in just a few minutes. 7 | It's designed with the Unix philosophy of ["do one thing well"](https://en.wikipedia.org/wiki/Unix_philosophy#Do_One_Thing_and_Do_It_Well) and wants to be used in a chain of command line pipes to make its requests in parallel. 8 | By default, it will echo all response bodies to standard out but can optionally save the results of each request in a directory for later analysis. 9 | 10 | ### Key Features 11 | 12 | * **Parallel Request Processing:** Handle thousands of URLs simultaneously with customizable worker counts. 13 | * **Flexible Output Options:** Output responses to stdout, save to a directory, or format as JSON for easy parsing. 14 | * **Integrate with CLI Tools:** Works well with tools like jq, awk, sort, and more for powerful data transformations. 15 | 16 | ### Why use `ganda` over `curl` (or `wget`, `httpie`, `postman-cli`, ...)? 17 | 18 | All existing CLI tools for making HTTP requests are oriented around making a single request at a time. They're great 19 | at starting a pipe of commands (ex: `curl | jq .`) but they're awkward to use beyond a few requests. 20 | 21 | The easiest way to use them is in a bash `for` loop or with something like `xargs`. This is slow and expensive as they open up a new HTTP connection on every request. 22 | 23 | `ganda` makes many requests in parallel and can maintain context between the request and response. It's designed to 24 | be used in a pipeline of commands and can be used to make hundreds of thousands of requests in just a few minutes. 25 | 26 | `ganda` will reuse HTTP connections and can specify how many "worker" threads should be used to tightly control parallelism. 27 | 28 | The closest CLIs I've found to `ganda` are load-testing tools like `vegeta`. They're able to make many requests in 29 | parallel, but they're not designed to only call each URL once, don't maintain context between the request and response, 30 | and don't have the same flexibility in how the response is handled. 31 | 32 | `ganda` isn't for load testing, it's for making lots of requests in parallel and processing the results in a pipeline. 33 | 34 | ## Documentation Links 35 | 36 | * [Installation](#installation) 37 | * [Usage Configuration Options](#usage--configuration-options) 38 | * [Quick Examples](#quick-examples) 39 | * [Advanced Use Cases](#sample-advanced-use-cases) 40 | 41 | # Installation 42 | 43 | One currently has 3 options: 44 | 45 | 1\. On MacOS you can install using [homebrew](https://brew.sh/) 46 | ```bash 47 | brew tap tednaleid/homebrew-ganda 48 | brew install ganda 49 | ``` 50 | 51 | 2\. Download the appropriate binary from the [releases page]((https://github.com/tednaleid/ganda/releases) and put it in your path 52 | 53 | 3\. Compile from source with golang: 54 | 55 | ```bash 56 | go install github.com/tednaleid/ganda@latest 57 | ``` 58 | 59 | or, if you have this repo downloaded locally: 60 | 61 | ```bash 62 | make install 63 | ``` 64 | 65 | to install in your `$GOPATH/bin` (which you want in your `$PATH`) 66 | 67 | # Usage & Configuration Options 68 | 69 | ```bash 70 | ganda help 71 | 72 | NAME: 73 | ganda - make http requests in parallel 74 | 75 | USAGE: 76 | | ganda [options] 77 | 78 | VERSION: 79 | 1.0.2 80 | 81 | DESCRIPTION: 82 | Pipe urls to ganda over stdout to make http requests to each url in parallel. 83 | 84 | AUTHOR: 85 | Ted Naleid 86 | 87 | COMMANDS: 88 | echoserver Starts an echo server, --port to override the default port of 8080 89 | help, h Shows a list of commands or help for one command 90 | 91 | GLOBAL OPTIONS: 92 | --base-retry-millis value the base number of milliseconds to wait before retrying a request, exponential backoff is used for retries (default: 1000) 93 | --response-body value, -B value transforms the body of the response. Values: 'raw' (unchanged), 'base64', 'discard' (don't emit body), 'escaped' (JSON escaped string), 'sha256' (default: raw) 94 | --connect-timeout-millis value number of milliseconds to wait for a connection to be established before timeout (default: 10000) 95 | --header value, -H value [ --header value, -H value ] headers to send with every request, can be used multiple times (gzip and keep-alive are already there) 96 | --insecure, -k if flag is present, skip verification of https certificates (default: false) 97 | --json-envelope, -J emit result with JSON envelope with url, status, length, and body fields, assumes result is valid json (default: false) 98 | --color if flag is present, add color to success/warn messages (default: false) 99 | --output-directory value if flag is present, save response bodies to files in the specified directory 100 | --request value, -X value HTTP request method to use (default: "GET") 101 | --retry value max number of retries on transient errors (5XX status codes/timeouts) to attempt (default: 0) 102 | --silent, -s if flag is present, omit showing response code for each url only output response bodies (default: false) 103 | --subdir-length value length of hashed subdirectory name to put saved files when using --output-directory; use 2 for > 5k urls, 4 for > 5M urls (default: 0) 104 | --throttle-per-second value max number of requests to process per second, default is unlimited (default: -1) 105 | --workers value, -W value number of concurrent workers that will be making requests, increase this for more requests in parallel (default: 1) 106 | --help, -h show help (default: false) 107 | --version, -v print the version (default: false) 108 | ``` 109 | 110 | # Quick Examples 111 | 112 | Here are a few quick examples to show how `ganda` can be used. 113 | 114 | ### Example 1: Basic Request from a List of IDs 115 | 116 | Given a file with a list of IDs in it, you could do something like: 117 | 118 | ```bash 119 | cat id_list.txt | awk '{printf "https://api.example.com/resource/%s?key=foo\n", $1}' | ganda 120 | ``` 121 | and that will pipe a stream of URLs into `ganda` in the format `https://api.example.com/resource/?key=foo`. 122 | 123 | This command: 124 | * Reads IDs from `id_list.txt`. 125 | * Uses `awk` to format each ID as a URL. 126 | * Pipes the generated URLs into `ganda` for parallel requests. 127 | 128 | ### Example 2: Requesting URLs from a File 129 | 130 | If you have a file containing URLs (one per line), you can pass it directly to `ganda`: 131 | 132 | ```bash 133 | ganda my_file_of_urls.txt 134 | ``` 135 | This command sends each URL in `my_file_of_urls.txt` as a request in parallel. You can control the output location by specifying an output directory with `-o `. 136 | 137 | ### Example 3: Save Responses to a Directory 138 | 139 | To save each response in a separate file within a specified directory: 140 | 141 | ```bash 142 | cat urls.txt | ganda -o response_dir 143 | ``` 144 | 145 | To save all responses to a single file, you can use standard output redirection: 146 | 147 | ```bash 148 | cat urls.txt | ganda > results.txt 149 | ``` 150 | 151 | For many more examples, take a look at the [Tour of `ganda`](docs/GANDA_TOUR.ipynb). 152 | 153 | ## Sample Advanced Use Cases 154 | 155 | `ganda` enables powerful workflows that would otherwise require custom scripting. Here are a few advanced examples. 156 | 157 | ### Example 1: Consuming Events from Kafka and Calling an API 158 | 159 | Using `kcat` (https://github.com/edenhill/kcat) (or another Kafka CLI that emits events from Kafka topics), we can consume all the events on a Kafka topic, then use `jq` to pull an identifier out of an event and make an API call for every identifier: 160 | 161 | ```bash 162 | # get all events on the `my-topic` topic 163 | kcat -C -e -q -b broker.example.com:9092 -t my-topic |\ 164 | # parse the identifier out of the JSON event 165 | jq -r '.identifier' |\ 166 | # use awk to turn that identifier into an URL 167 | awk '{ printf "https://api.example.com/item/%s\n", $1}' |\ 168 | # have 5 workers make requests and use a static header with and API key for every request 169 | ganda -s -W 5 -H "X-Api-Key: my-key" |\ 170 | # parse the `value` out of the response and emit it on stdout 171 | jq -r '.value' 172 | ``` 173 | 174 | ### Example 2: Requesting Multiple Pages from an API 175 | 176 | Here, we ask for the first 100 pages from an API. Each returns a JSON list of `status` fields. Pull those `status` fields out and do a unique count on the distribution. 177 | 178 | ```bash 179 | # emit a sequence of the numbers from 1 to 100 180 | seq 100 |\ 181 | # use awk to create an url asking for each of the buckets 182 | awk '{printf "https://example.com/items?type=BUCKET&value=%s\n", $1}' |\ 183 | # use a single ganda worker to ask for each page in sequence 184 | ganda -s -W 1 -H "X-Api-Key: my-key" |\ 185 | # use jq to parse the resulting json and grab the status 186 | jq -r '.items[].status' |\ 187 | sort |\ 188 | # get a unique count of how many times each status appears 189 | uniq -c 190 | 191 | 41128 DELETED 192 | 6491 INITIATED 193 | 34222 PROCESSED 194 | 5032 ERRORED 195 | ``` 196 | ## Contribution Guidelines 197 | 198 | If you like to contribute, please follow these steps: 199 | 1. Fork the repository and create a new branch. 200 | 2. Make your changes and write tests if applicable. 201 | 3. Submit a pull request with a clear description of your changes. 202 | -------------------------------------------------------------------------------- /responses/responses_test.go: -------------------------------------------------------------------------------- 1 | package responses 2 | 3 | import ( 4 | "bytes" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/tednaleid/ganda/config" 7 | "net/http" 8 | "net/url" 9 | "testing" 10 | ) 11 | 12 | func TestRawOutput(t *testing.T) { 13 | responseFn := determineEmitResponseFn(config.Raw) 14 | assert.NotNil(t, responseFn) 15 | 16 | mockResponse := NewMockResponseBodyOnly("hello world") 17 | writeCloser := NewMockWriteCloser() 18 | 19 | responseFn(&ResponseWithContext{Response: mockResponse.Response}, writeCloser) 20 | 21 | assert.True(t, mockResponse.BodyClosed()) 22 | assert.Equal(t, "hello world", writeCloser.ToString()) 23 | } 24 | 25 | func TestRawOutputJSON(t *testing.T) { 26 | responseFn := determineEmitJsonResponseWithContextFn(config.Raw) 27 | assert.NotNil(t, responseFn) 28 | 29 | mockResponse := NewMockResponseBodyOnly("\"hello world\"") 30 | writeCloser := NewMockWriteCloser() 31 | 32 | responseFn(&ResponseWithContext{Response: mockResponse.Response, RequestContext: nil}, writeCloser) 33 | 34 | assert.True(t, mockResponse.BodyClosed()) 35 | assert.Equal(t, "{ \"url\": \"http://example.com\", \"code\": 200, \"body\": \"hello world\" }", writeCloser.ToString()) 36 | } 37 | 38 | func TestEscapedOutput(t *testing.T) { 39 | responseFn := determineEmitResponseFn(config.Escaped) 40 | assert.NotNil(t, responseFn) 41 | 42 | mockResponse := NewMockResponseBodyOnly("hello world") 43 | writeCloser := NewMockWriteCloser() 44 | 45 | responseFn(&ResponseWithContext{Response: mockResponse.Response}, writeCloser) 46 | 47 | assert.True(t, mockResponse.BodyClosed()) 48 | assert.Equal(t, "\"hello world\"", writeCloser.ToString()) 49 | } 50 | 51 | func TestEscapedOutputJSON(t *testing.T) { 52 | responseFn := determineEmitJsonResponseWithContextFn(config.Escaped) 53 | assert.NotNil(t, responseFn) 54 | 55 | mockResponse := NewMockResponseBodyOnly("\"hello world\"") 56 | writeCloser := NewMockWriteCloser() 57 | 58 | responseFn(&ResponseWithContext{Response: mockResponse.Response, RequestContext: nil}, writeCloser) 59 | 60 | assert.True(t, mockResponse.BodyClosed()) 61 | assert.Equal(t, "{ \"url\": \"http://example.com\", \"code\": 200, \"body\": \"\\\"hello world\\\"\" }", writeCloser.ToString()) 62 | } 63 | 64 | func TestDiscardOutput(t *testing.T) { 65 | responseFn := determineEmitResponseFn(config.Discard) 66 | assert.NotNil(t, responseFn) 67 | 68 | mockResponse := NewMockResponseBodyOnly("hello world") 69 | out := NewMockWriteCloser() 70 | 71 | responseFn(&ResponseWithContext{Response: mockResponse.Response}, out) 72 | assert.True(t, mockResponse.BodyClosed()) 73 | assert.Equal(t, "", out.ToString()) 74 | } 75 | 76 | func TestDiscardOutputJSON(t *testing.T) { 77 | responseFn := determineEmitJsonResponseWithContextFn(config.Discard) 78 | assert.NotNil(t, responseFn) 79 | 80 | mockResponse := NewMockResponseBodyOnly("hello world") 81 | out := NewMockWriteCloser() 82 | 83 | responseFn(&ResponseWithContext{Response: mockResponse.Response}, out) 84 | assert.True(t, mockResponse.BodyClosed()) 85 | assert.Equal(t, "{ \"url\": \"http://example.com\", \"code\": 200, \"body\": null }", out.ToString()) 86 | } 87 | 88 | func TestBase64Output(t *testing.T) { 89 | responseFn := determineEmitResponseFn(config.Base64) 90 | assert.NotNil(t, responseFn) 91 | 92 | mockResponse := NewMockResponseBodyOnly("hello world") 93 | out := NewMockWriteCloser() 94 | 95 | responseFn(&ResponseWithContext{Response: mockResponse.Response}, out) 96 | assert.True(t, mockResponse.BodyClosed()) 97 | assert.Equal(t, "aGVsbG8gd29ybGQ=", out.ToString()) 98 | } 99 | 100 | func TestBase64OutputJSON(t *testing.T) { 101 | responseFn := determineEmitJsonResponseWithContextFn(config.Base64) 102 | assert.NotNil(t, responseFn) 103 | 104 | mockResponse := NewMockResponseBodyOnly("hello world") 105 | out := NewMockWriteCloser() 106 | 107 | responseFn(&ResponseWithContext{Response: mockResponse.Response}, out) 108 | assert.True(t, mockResponse.BodyClosed()) 109 | assert.Equal(t, "{ \"url\": \"http://example.com\", \"code\": 200, \"body\": \"aGVsbG8gd29ybGQ=\" }", out.ToString()) 110 | } 111 | 112 | func TestSha256Output(t *testing.T) { 113 | responseFn := determineEmitResponseFn(config.Sha256) 114 | assert.NotNil(t, responseFn) 115 | 116 | mockResponse := NewMockResponseBodyOnly("hello world") 117 | out := NewMockWriteCloser() 118 | 119 | responseFn(&ResponseWithContext{Response: mockResponse.Response}, out) 120 | assert.True(t, mockResponse.BodyClosed()) 121 | 122 | // if testing with "echo" be sure to use the -n flag to not include the newline 123 | // echo -n "hello world" | shasum -a 256 124 | // b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 - 125 | assert.Equal(t, "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", out.ToString()) 126 | 127 | // ensure that when called a second time, we get the same answer and that the hasher can be reused 128 | mockResponse2 := NewMockResponseBodyOnly("hello world") 129 | out2 := NewMockWriteCloser() 130 | 131 | responseFn(&ResponseWithContext{Response: mockResponse2.Response}, out2) 132 | assert.True(t, mockResponse2.BodyClosed()) 133 | assert.Equal(t, "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", out2.ToString()) 134 | } 135 | 136 | func TestSha256OutputJSON(t *testing.T) { 137 | responseFn := determineEmitJsonResponseWithContextFn(config.Sha256) 138 | assert.NotNil(t, responseFn) 139 | 140 | mockResponse := NewMockResponseBodyOnly("hello world") 141 | out := NewMockWriteCloser() 142 | 143 | responseFn(&ResponseWithContext{Response: mockResponse.Response}, out) 144 | assert.True(t, mockResponse.BodyClosed()) 145 | 146 | // if testing with "echo" be sure to use the -n flag to not include the newline 147 | // echo -n "hello world" | shasum -a 256 148 | // b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 - 149 | assert.Equal(t, 150 | "{ \"url\": \"http://example.com\", \"code\": 200, \"body\": \"b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9\" }", 151 | out.ToString()) 152 | 153 | // ensure that when called a second time, we get the same answer and that the hasher can be reused 154 | mockResponse2 := NewMockResponseBodyOnly("hello world") 155 | out2 := NewMockWriteCloser() 156 | 157 | responseFn(&ResponseWithContext{Response: mockResponse2.Response}, out2) 158 | assert.True(t, mockResponse2.BodyClosed()) 159 | assert.Equal(t, 160 | "{ \"url\": \"http://example.com\", \"code\": 200, \"body\": \"b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9\" }", 161 | out2.ToString()) 162 | } 163 | 164 | func TestRawOutputWithRequestContextJSON(t *testing.T) { 165 | responseFn := determineEmitJsonResponseWithContextFn(config.Raw) 166 | assert.NotNil(t, responseFn) 167 | 168 | testCases := []struct { 169 | name string 170 | requestContext interface{} 171 | expectedOutput string 172 | }{ 173 | { 174 | name: "string RequestContext", 175 | requestContext: "a context string", 176 | expectedOutput: "{ \"url\": \"http://example.com\", \"code\": 200, \"body\": \"hello world\", \"context\": \"a context string\" }", 177 | }, 178 | { 179 | name: "list of strings RequestContext", 180 | requestContext: []string{"context1", "context2"}, 181 | expectedOutput: "{ \"url\": \"http://example.com\", \"code\": 200, \"body\": \"hello world\", \"context\": [\"context1\",\"context2\"] }", 182 | }, 183 | { 184 | name: "map RequestContext", 185 | requestContext: map[string]string{"key1": "value1", "key2": "value2"}, 186 | expectedOutput: "{ \"url\": \"http://example.com\", \"code\": 200, \"body\": \"hello world\", \"context\": {\"key1\":\"value1\",\"key2\":\"value2\"} }", 187 | }, 188 | { 189 | name: "object RequestContext", 190 | requestContext: struct { 191 | Field1 string `json:"field1"` 192 | Field2 int `json:"field2"` 193 | }{Field1: "value1", Field2: 2}, 194 | expectedOutput: "{ \"url\": \"http://example.com\", \"code\": 200, \"body\": \"hello world\", \"context\": {\"field1\":\"value1\",\"field2\":2} }", 195 | }, 196 | } 197 | 198 | for _, tc := range testCases { 199 | t.Run(tc.name, func(t *testing.T) { 200 | mockResponse := NewMockResponseBodyOnly("\"hello world\"") 201 | writeCloser := NewMockWriteCloser() 202 | 203 | responseFn(&ResponseWithContext{Response: mockResponse.Response, RequestContext: tc.requestContext}, writeCloser) 204 | 205 | assert.True(t, mockResponse.BodyClosed()) 206 | assert.Equal(t, tc.expectedOutput, writeCloser.ToString()) 207 | }) 208 | } 209 | } 210 | 211 | type MockResponse struct { 212 | *http.Response 213 | mockBody *MockReadCloser 214 | } 215 | 216 | func (mr *MockResponse) BodyClosed() bool { 217 | return mr.mockBody.Closed 218 | } 219 | 220 | func NewMockResponseBodyOnly(body string) *MockResponse { 221 | return NewMockResponse(body, "http://example.com", 200) 222 | } 223 | 224 | func NewMockResponse(body string, fullUrl string, statusCode int) *MockResponse { 225 | parsedURL, _ := url.Parse(fullUrl) 226 | 227 | mockReadCloser := &MockReadCloser{ 228 | Reader: bytes.NewReader([]byte(body)), 229 | Closed: false, 230 | } 231 | return &MockResponse{ 232 | Response: &http.Response{ 233 | Body: mockReadCloser, 234 | StatusCode: statusCode, 235 | Request: &http.Request{ 236 | URL: parsedURL, 237 | }, 238 | }, 239 | mockBody: mockReadCloser, 240 | } 241 | } 242 | 243 | type MockReadCloser struct { 244 | *bytes.Reader 245 | Closed bool 246 | } 247 | 248 | func (mrc *MockReadCloser) Close() error { 249 | mrc.Closed = true 250 | return nil 251 | } 252 | 253 | type MockWriteCloser struct { 254 | Buffer *bytes.Buffer 255 | Closed bool 256 | } 257 | 258 | func (m *MockWriteCloser) Write(p []byte) (n int, err error) { 259 | return m.Buffer.Write(p) 260 | } 261 | 262 | func (m *MockWriteCloser) Close() error { 263 | m.Closed = true 264 | return nil 265 | } 266 | 267 | func (m *MockWriteCloser) ToString() string { 268 | return m.Buffer.String() 269 | } 270 | 271 | func NewMockWriteCloser() *MockWriteCloser { 272 | return &MockWriteCloser{ 273 | Buffer: new(bytes.Buffer), 274 | Closed: false, 275 | } 276 | } 277 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright (c) 2018 Ted Naleid 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /responses/responses.go: -------------------------------------------------------------------------------- 1 | package responses 2 | 3 | import ( 4 | "bytes" 5 | "crypto/md5" 6 | "crypto/sha256" 7 | "encoding/base64" 8 | "encoding/hex" 9 | "encoding/json" 10 | "fmt" 11 | "github.com/tednaleid/ganda/config" 12 | "github.com/tednaleid/ganda/execcontext" 13 | "hash" 14 | "io" 15 | "net/http" 16 | "os" 17 | "regexp" 18 | "sync" 19 | ) 20 | 21 | type ResponseWithContext struct { 22 | Response *http.Response 23 | RequestContext interface{} 24 | } 25 | 26 | func StartResponseWorkers(responsesWithContext <-chan *ResponseWithContext, context *execcontext.Context) *sync.WaitGroup { 27 | var responseWaitGroup sync.WaitGroup 28 | responseWaitGroup.Add(context.ResponseWorkers) 29 | 30 | for i := 1; i <= context.ResponseWorkers; i++ { 31 | go func() { 32 | var emitResponse emitResponseWithContextFn 33 | if context.JsonEnvelope { 34 | emitResponse = determineEmitJsonResponseWithContextFn(context.ResponseBody) 35 | } else { 36 | emitResponse = determineEmitResponseFn(context.ResponseBody) 37 | } 38 | 39 | if context.WriteFiles { 40 | responseSavingWorker(responsesWithContext, context, emitResponse) 41 | } else { 42 | responsePrintingWorker(responsesWithContext, context, emitResponse) 43 | } 44 | responseWaitGroup.Done() 45 | }() 46 | } 47 | 48 | return &responseWaitGroup 49 | } 50 | 51 | // creates a worker that takes responses off the channel and saves each one to a file 52 | // the directory path is based off the md5 hash of the url 53 | // the filename is the url with all non-alphanumeric characters replaced with dashes 54 | func responseSavingWorker( 55 | responsesWithContext <-chan *ResponseWithContext, 56 | context *execcontext.Context, 57 | emitResponseWithContextFn emitResponseWithContextFn, 58 | ) { 59 | specialCharactersRegexp := regexp.MustCompile("[^A-Za-z0-9]+") 60 | 61 | responseWorker(responsesWithContext, func(responseWithContext *ResponseWithContext) { 62 | response := responseWithContext.Response 63 | filename := specialCharactersRegexp.ReplaceAllString(response.Request.URL.String(), "-") 64 | writeableFile := createWritableFile(context.BaseDirectory, context.SubdirLength, filename) 65 | defer writeableFile.WriteCloser.Close() 66 | 67 | _, err := emitResponseWithContextFn(responseWithContext, writeableFile.WriteCloser) 68 | 69 | if err != nil { 70 | context.Logger.LogError(err, response.Request.URL.String()+" -> "+writeableFile.FullPath) 71 | } else { 72 | context.Logger.LogResponse(response.StatusCode, response.Request.URL.String()+" -> "+writeableFile.FullPath) 73 | } 74 | }) 75 | } 76 | 77 | // creates a worker that takes responses off the channel and prints each one to stdout 78 | // if the JsonEnvelope flag is set, it will wrap the response in a JSON envelope 79 | // a newline will be emitted after each non-empty response 80 | func responsePrintingWorker( 81 | responsesWithContext <-chan *ResponseWithContext, 82 | context *execcontext.Context, 83 | emitResponseWithContext emitResponseWithContextFn, 84 | ) { 85 | out := context.Out 86 | newline := []byte("\n") 87 | responseWorker(responsesWithContext, func(responseWithContext *ResponseWithContext) { 88 | response := responseWithContext.Response 89 | bytesWritten, err := emitResponseWithContext(responseWithContext, out) 90 | 91 | if err != nil { 92 | context.Logger.LogError(err, response.Request.URL.String()) 93 | } else { 94 | context.Logger.LogResponse(response.StatusCode, response.Request.URL.String()) 95 | if bytesWritten > 0 { 96 | out.Write(newline) 97 | } 98 | } 99 | }) 100 | } 101 | 102 | // takes a response and writes it to the writer, returns true if it wrote anything 103 | type emitResponseFn func(response *http.Response, out io.Writer) (bytesWritten int64, err error) 104 | type emitResponseWithContextFn func(responseWithContext *ResponseWithContext, out io.Writer) (bytesWritten int64, err error) 105 | 106 | // emits the response without the context, context is only supported in JSON output 107 | func determineEmitResponseFn(responseBody config.ResponseBodyType) emitResponseWithContextFn { 108 | bodyResponseFn := determineEmitBodyResponseFn(responseBody) 109 | 110 | return func(responseWithContext *ResponseWithContext, out io.Writer) (bytesWritten int64, err error) { 111 | return bodyResponseFn(responseWithContext.Response, out) 112 | } 113 | } 114 | 115 | // surrounds the responsesBody with a JSON envelope that includes the context of the request (if any) 116 | func determineEmitJsonResponseWithContextFn(responseBody config.ResponseBodyType) emitResponseWithContextFn { 117 | bodyResponseFn := determineEmitBodyResponseFn(responseBody) 118 | return jsonEnvelopeResponseFn(bodyResponseFn, responseBody) 119 | } 120 | 121 | // returns a function that will emit the JSON envelope around the response body 122 | // the JSON envelope will include the url and http code along with the response body 123 | func jsonEnvelopeResponseFn(bodyResponseFn emitResponseFn, responseBody config.ResponseBodyType) emitResponseWithContextFn { 124 | return func(responseWithContext *ResponseWithContext, out io.Writer) (bytesWritten int64, err error) { 125 | var bodyBytesWritten int64 126 | var contextBytesWritten int64 127 | var closingBytesWritten int64 128 | 129 | response := responseWithContext.Response 130 | 131 | requestContext := responseWithContext.RequestContext 132 | 133 | // everything before emitting the body response 134 | bytesWritten, err = appendString(0, out, fmt.Sprintf( 135 | "{ \"url\": \"%s\", \"code\": %d, \"body\": ", 136 | response.Request.URL.String(), 137 | response.StatusCode, 138 | )) 139 | if err != nil { 140 | return bytesWritten, err 141 | } 142 | 143 | // emit the body response 144 | if responseBody == config.Discard || responseBody == config.Raw || responseBody == config.Escaped { 145 | // no need to wrap either of these in quotes, Raw is assumed to be JSON 146 | bodyBytesWritten, err = bodyResponseFn(response, out) 147 | } else { 148 | // for all other ResponseBody types we want to encapsulate the body in quotes if it exists, 149 | // so we need to use a temp buffer to see if there's anything to quote 150 | tempBuffer := new(bytes.Buffer) 151 | bodyBytesWritten, err = bodyResponseFn(response, tempBuffer) 152 | if err == nil && bodyBytesWritten > 0 { 153 | bytesWritten, err = appendString(bytesWritten, out, "\""+tempBuffer.String()+"\"") 154 | } 155 | } 156 | 157 | bytesWritten += bodyBytesWritten 158 | 159 | if err != nil { 160 | return bytesWritten, err 161 | } 162 | 163 | // if we didn't write anything for the body response, we emit a `null` 164 | if bodyBytesWritten == 0 { 165 | bodyBytesWritten, err = appendString(bytesWritten, out, "null") 166 | bytesWritten += bodyBytesWritten 167 | if err != nil { 168 | return bytesWritten, err 169 | } 170 | } 171 | 172 | // Add requestContext to JSON if it is not nil/null 173 | if requestContext != nil { 174 | requestContextJson, err := json.Marshal(requestContext) 175 | if err != nil { 176 | return bytesWritten, err 177 | } 178 | requestContextString := string(requestContextJson) 179 | if requestContextString != "null" { 180 | contextBytesWritten, err = appendString(bytesWritten, out, fmt.Sprintf(", \"context\": %s", string(requestContextJson))) 181 | bytesWritten += contextBytesWritten 182 | if err != nil { 183 | return bytesWritten, err 184 | } 185 | } 186 | } 187 | 188 | // close out the JSON envelope 189 | closingBytesWritten, err = appendString(bytesWritten, out, " }") 190 | bytesWritten += closingBytesWritten 191 | if err != nil { 192 | return bytesWritten, err 193 | } 194 | 195 | return bytesWritten, err 196 | } 197 | } 198 | 199 | // writes a string to the writer and updates the number of bytes written 200 | func appendString(bytesPreviouslyWritten int64, out io.Writer, s string) (int64, error) { 201 | appendedBytes, err := fmt.Fprint(out, s) 202 | return bytesPreviouslyWritten + int64(appendedBytes), err 203 | } 204 | 205 | func determineEmitBodyResponseFn(responseBody config.ResponseBodyType) emitResponseFn { 206 | switch responseBody { 207 | case config.Raw: 208 | return emitRawBody 209 | case config.Sha256: 210 | return emitSha256BodyFn() 211 | case config.Discard: 212 | return emitNothingBody 213 | case config.Escaped: 214 | return emitEscapedBodyFn() 215 | case config.Base64: 216 | return emitBase64Body 217 | default: 218 | panic(fmt.Sprintf("unknown response body type %s", responseBody)) 219 | } 220 | } 221 | 222 | func emitRawBody(response *http.Response, out io.Writer) (bytesWritten int64, err error) { 223 | defer response.Body.Close() 224 | return io.Copy(out, response.Body) 225 | } 226 | 227 | func emitSha256BodyFn() func(response *http.Response, out io.Writer) (bytesWritten int64, err error) { 228 | hasher := sha256.New() 229 | return func(response *http.Response, out io.Writer) (bytesWritten int64, err error) { 230 | return emitHashedBody(hasher, response, out) 231 | } 232 | } 233 | 234 | func emitHashedBody(hasher hash.Hash, response *http.Response, out io.Writer) (bytesWritten int64, err error) { 235 | defer response.Body.Close() 236 | 237 | hasher.Reset() 238 | 239 | hashedBytesWritten, err := io.Copy(hasher, response.Body) 240 | if err != nil || hashedBytesWritten == 0 { 241 | return 0, err 242 | } 243 | 244 | n, err := fmt.Fprint(out, hex.EncodeToString(hasher.Sum(nil))) 245 | return int64(n), err 246 | } 247 | 248 | func emitBase64Body(response *http.Response, out io.Writer) (bytesWritten int64, err error) { 249 | defer response.Body.Close() 250 | 251 | encoder := base64.NewEncoder(base64.StdEncoding, out) 252 | bytesWritten, err = io.Copy(encoder, response.Body) 253 | if err != nil { 254 | return bytesWritten, err 255 | } 256 | 257 | err = encoder.Close() 258 | return bytesWritten, err 259 | } 260 | 261 | func emitEscapedBodyFn() func(response *http.Response, out io.Writer) (bytesWritten int64, err error) { 262 | buffer := new(bytes.Buffer) 263 | return func(response *http.Response, out io.Writer) (bytesWritten int64, err error) { 264 | return emitEscapedBody(buffer, response, out) 265 | } 266 | } 267 | 268 | func emitEscapedBody(buffer *bytes.Buffer, response *http.Response, out io.Writer) (bytesWritten int64, err error) { 269 | defer response.Body.Close() 270 | 271 | buffer.Reset() 272 | 273 | _, err = io.Copy(buffer, response.Body) 274 | if err != nil { 275 | return 0, err 276 | } 277 | 278 | if buffer.Len() == 0 { 279 | return 0, nil 280 | } 281 | 282 | // Marshal the buffer's contents to JSON 283 | jsonBytes, err := json.Marshal(buffer.String()) 284 | if err != nil { 285 | return 0, err 286 | } 287 | 288 | // Write the JSON bytes to the output writer 289 | n, err := out.Write(jsonBytes) 290 | return int64(n), err 291 | } 292 | 293 | func emitNothingBody(response *http.Response, out io.Writer) (bytesWritten int64, err error) { 294 | response.Body.Close() 295 | return 0, nil 296 | } 297 | 298 | func responseWorker(responsesWithContext <-chan *ResponseWithContext, responseHandler func(*ResponseWithContext)) { 299 | for responseWithContext := range responsesWithContext { 300 | responseHandler(responseWithContext) 301 | } 302 | } 303 | 304 | type WritableFile struct { 305 | FullPath string 306 | WriteCloser io.WriteCloser 307 | } 308 | 309 | func createWritableFile(baseDirectory string, subdirLength int64, filename string) *WritableFile { 310 | directory := directoryForFile(baseDirectory, filename, subdirLength) 311 | fullPath := directory + filename 312 | 313 | file, err := os.Create(fullPath) 314 | if err != nil { 315 | panic(err) 316 | } 317 | 318 | return &WritableFile{FullPath: fullPath, WriteCloser: file} 319 | } 320 | 321 | func directoryForFile(baseDirectory string, filename string, subdirLength int64) string { 322 | var directory string 323 | if subdirLength <= 0 { 324 | directory = fmt.Sprintf("%s/", baseDirectory) 325 | } else { 326 | sliceEnd := 1 327 | 328 | // don't create directories longer than 4 binary hex characters (4^16 = 65k directories) 329 | if subdirLength > 2 { 330 | sliceEnd = 2 331 | } 332 | 333 | md5val := md5.Sum([]byte(filename)) 334 | directory = fmt.Sprintf("%s/%x/", baseDirectory, md5val[0:sliceEnd]) 335 | } 336 | 337 | os.MkdirAll(directory, os.ModePerm) 338 | return directory 339 | } 340 | -------------------------------------------------------------------------------- /parser/parser_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/tednaleid/ganda/config" 7 | "github.com/tednaleid/ganda/parser" 8 | "io" 9 | "strings" 10 | "testing" 11 | ) 12 | 13 | func TestSendGetRequestUrlsHaveDefaultHeaders(t *testing.T) { 14 | requestsWithContext := make(chan parser.RequestWithContext, 2) 15 | defer close(requestsWithContext) 16 | 17 | inputLines := ` 18 | https://example.com/bar 19 | https://example.com/qux 20 | ` 21 | 22 | var in = trimmedInputReader(inputLines) 23 | 24 | err := parser.SendRequests(requestsWithContext, in, "GET", []config.RequestHeader{}) 25 | 26 | assert.Nil(t, err, "expected no error") 27 | 28 | requestWithContext := <-requestsWithContext 29 | request := requestWithContext.Request 30 | requestContext := requestWithContext.RequestContext 31 | 32 | assert.Equal(t, "https://example.com/bar", request.URL.String(), "expected url") 33 | assert.Equal(t, "GET", request.Method, "expected method") 34 | assert.Equal(t, request.Header["Connection"][0], "keep-alive", "Connection header") 35 | assert.Equal(t, []string(nil), requestContext, "expected nil string context") 36 | 37 | secondRequestWithContext := <-requestsWithContext 38 | secondRequest := secondRequestWithContext.Request 39 | secondRequestContext := secondRequestWithContext.RequestContext 40 | 41 | assert.Equal(t, "https://example.com/qux", secondRequest.URL.String(), "expected url") 42 | assert.Equal(t, []string(nil), secondRequestContext, "expected nil string context") 43 | } 44 | 45 | func TestSendGetRequestUrlsAddGivenHeaders(t *testing.T) { 46 | requestsWithContext := make(chan parser.RequestWithContext, 1) 47 | defer close(requestsWithContext) 48 | 49 | inputLines := `https://example.com/bar` 50 | var in = trimmedInputReader(inputLines) 51 | 52 | requestHeaders := []config.RequestHeader{{Key: "X-Test", Value: "foo"}, {Key: "X-Test2", Value: "bar"}} 53 | 54 | err := parser.SendRequests(requestsWithContext, in, "GET", requestHeaders) 55 | 56 | assert.Nil(t, err, "expected no error") 57 | 58 | requestWithContext := <-requestsWithContext 59 | request := requestWithContext.Request 60 | requestContext := requestWithContext.RequestContext 61 | 62 | assert.Equal(t, "https://example.com/bar", request.URL.String(), "expected url") 63 | assert.Equal(t, "GET", request.Method, "expected method") 64 | assert.Equal(t, request.Header["Connection"][0], "keep-alive", "Connection header") 65 | assert.Equal(t, request.Header["X-Test"][0], "foo") 66 | assert.Equal(t, request.Header["X-Test2"][0], "bar") 67 | assert.Equal(t, []string(nil), requestContext, "expected nil string context") 68 | } 69 | 70 | func TestSendRequestsHasRaggedRequestContext(t *testing.T) { 71 | requestsWithContext := make(chan parser.RequestWithContext, 3) 72 | defer close(requestsWithContext) 73 | 74 | // we allow ragged numbers of fields in the TSV input 75 | // we also follow the quoting rules in RFC 4180, so: 76 | // a double quote in a quoted field is escaped with another double quote 77 | // whitespace inside a quoted field is preserved 78 | inputLines := ` 79 | https://ex.com/bar foo "quoted content" 80 | https://ex.com/qux quux " ""quoted with whitespace"" " 456 81 | https://ex.com/123 "baz" 82 | ` 83 | 84 | var in = trimmedInputReader(inputLines) 85 | 86 | parser.SendRequests(requestsWithContext, in, "GET", []config.RequestHeader{}) 87 | 88 | expectedResults := []struct { 89 | url string 90 | context []string 91 | }{ 92 | {"https://ex.com/bar", []string{"foo", "quoted content"}}, 93 | {"https://ex.com/qux", []string{"quux", " \"quoted with whitespace\" ", "456"}}, 94 | {"https://ex.com/123", []string{"baz"}}, 95 | } 96 | 97 | for _, expectedResult := range expectedResults { 98 | requestWithContext := <-requestsWithContext 99 | request := requestWithContext.Request 100 | requestContext := requestWithContext.RequestContext 101 | 102 | assert.Equal(t, expectedResult.url, request.URL.String(), "expected url") 103 | assert.Equal(t, expectedResult.context, requestContext, "expected context") 104 | } 105 | } 106 | 107 | func TestSendRequestsHasMalformedTSV(t *testing.T) { 108 | requestsWithContext := make(chan parser.RequestWithContext, 1) 109 | defer close(requestsWithContext) 110 | 111 | inputLines := `https://ex.com/bar foo "quoted content missing terminating quote` 112 | 113 | var in = trimmedInputReader(inputLines) 114 | 115 | err := parser.SendRequests(requestsWithContext, in, "GET", []config.RequestHeader{}) 116 | 117 | assert.NotNil(t, err, "expected error") 118 | assert.Equal(t, "parse error on line 1, column 65: extraneous or missing \" in quoted-field", err.Error()) 119 | } 120 | 121 | func TestSendJsonLinesRequests(t *testing.T) { 122 | requestsWithContext := make(chan parser.RequestWithContext, 3) 123 | defer close(requestsWithContext) 124 | 125 | inputLines := ` 126 | { "url": "https://ex.com/bar", "context": ["foo", "quoted content"] } 127 | { "url": "https://ex.com/qux", "method": "POST", "context": { "quux": " \"quoted with whitespace\" ", "corge": 456 } } 128 | { "url": "https://ex.com/123", "method": "DELETE", "context": "baz" } 129 | ` 130 | 131 | var in = trimmedInputReader(inputLines) 132 | 133 | err := parser.SendRequests(requestsWithContext, in, "GET", []config.RequestHeader{}) 134 | assert.Nil(t, err, "expected no error") 135 | 136 | expectedResults := []struct { 137 | url string 138 | context interface{} 139 | method string 140 | }{ 141 | {"https://ex.com/bar", []interface{}{"foo", "quoted content"}, "GET"}, 142 | {"https://ex.com/qux", map[string]interface{}{"quux": " \"quoted with whitespace\" ", "corge": float64(456)}, "POST"}, 143 | {"https://ex.com/123", "baz", "DELETE"}, 144 | } 145 | 146 | for _, expectedResult := range expectedResults { 147 | requestWithContext := <-requestsWithContext 148 | request := requestWithContext.Request 149 | requestContext := requestWithContext.RequestContext 150 | 151 | assert.Equal(t, expectedResult.url, request.URL.String(), "expected url") 152 | assert.Equal(t, expectedResult.context, requestContext, "expected context") 153 | assert.Equal(t, expectedResult.method, request.Method, "expected method") 154 | } 155 | } 156 | 157 | func TestSendJsonLinesRequestsMissingUrl(t *testing.T) { 158 | requestsWithContext := make(chan parser.RequestWithContext, 1) 159 | defer close(requestsWithContext) 160 | 161 | // missing `url` field 162 | inputLines := ` { "noturl": "https://ex.com/bar", "context": ["foo", "quoted content"] }` 163 | 164 | var in = trimmedInputReader(inputLines) 165 | 166 | err := parser.SendRequests(requestsWithContext, in, "GET", []config.RequestHeader{}) 167 | 168 | assert.NotNil(t, err, "expected error") 169 | assert.Equal(t, "missing url property: { \"noturl\": \"https://ex.com/bar\", \"context\": [\"foo\", \"quoted content\"] }", err.Error()) 170 | } 171 | 172 | func TestSendJsonLinesRequestsMalformedJson(t *testing.T) { 173 | requestsWithContext := make(chan parser.RequestWithContext, 1) 174 | defer close(requestsWithContext) 175 | 176 | // missing trailing `}` 177 | inputLines := ` { "url": "https://ex.com/bar", "context": ["foo", "quoted content"]` 178 | 179 | var in = trimmedInputReader(inputLines) 180 | 181 | err := parser.SendRequests(requestsWithContext, in, "GET", []config.RequestHeader{}) 182 | 183 | assert.NotNil(t, err, "expected error") 184 | assert.Equal(t, "unexpected end of JSON input: { \"url\": \"https://ex.com/bar\", \"context\": [\"foo\", \"quoted content\"]", err.Error()) 185 | } 186 | 187 | func TestSendJsonLinesAddGivenHeaders(t *testing.T) { 188 | requestsWithContext := make(chan parser.RequestWithContext, 1) 189 | defer close(requestsWithContext) 190 | 191 | inputLines := `{ "url": "https://ex.com/123", "method": "DELETE", "headers": { "X-Bar": "corge" }, "context": "baz" }` 192 | 193 | var in = trimmedInputReader(inputLines) 194 | 195 | staticHeaders := []config.RequestHeader{{Key: "X-Static", Value: "foo"}} 196 | 197 | err := parser.SendRequests(requestsWithContext, in, "GET", staticHeaders) 198 | 199 | assert.Nil(t, err, "expected no error") 200 | 201 | requestWithContext := <-requestsWithContext 202 | request := requestWithContext.Request 203 | requestContext := requestWithContext.RequestContext 204 | 205 | assert.Equal(t, "https://ex.com/123", request.URL.String(), "expected url") 206 | assert.Equal(t, "DELETE", request.Method, "expected method") 207 | assert.Equal(t, request.Header["Connection"][0], "keep-alive", "Connection header") 208 | assert.Equal(t, request.Header["X-Static"][0], "foo") 209 | assert.Equal(t, request.Header["X-Bar"][0], "corge") 210 | assert.Equal(t, "baz", requestContext, "expected context") 211 | } 212 | 213 | func TestSendJsonLinesGivenHeadersOverrideStaticHeaders(t *testing.T) { 214 | requestsWithContext := make(chan parser.RequestWithContext, 1) 215 | defer close(requestsWithContext) 216 | 217 | inputLines := `{ "url": "https://ex.com/123", "method": "DELETE", "headers": { "X-Bar": "corge" }, "context": "baz" }` 218 | 219 | var in = trimmedInputReader(inputLines) 220 | 221 | staticHeaders := []config.RequestHeader{{Key: "X-Bar", Value: "foo"}} 222 | 223 | err := parser.SendRequests(requestsWithContext, in, "GET", staticHeaders) 224 | 225 | assert.Nil(t, err, "expected no error") 226 | 227 | requestWithContext := <-requestsWithContext 228 | request := requestWithContext.Request 229 | requestContext := requestWithContext.RequestContext 230 | 231 | assert.Equal(t, "https://ex.com/123", request.URL.String(), "expected url") 232 | assert.Equal(t, "DELETE", request.Method, "expected method") 233 | assert.Equal(t, request.Header["Connection"][0], "keep-alive", "Connection header") 234 | assert.Equal(t, request.Header["X-Bar"][0], "corge") 235 | assert.Equal(t, "baz", requestContext, "expected context") 236 | } 237 | 238 | func TestSendJsonLinesStaticHeadersAreNotRequiredToAddHeaders(t *testing.T) { 239 | requestsWithContext := make(chan parser.RequestWithContext, 1) 240 | defer close(requestsWithContext) 241 | 242 | inputLines := `{ "url": "https://ex.com/123", "method": "DELETE", "headers": { "X-Bar": "corge" }, "context": "baz" }` 243 | 244 | var in = trimmedInputReader(inputLines) 245 | 246 | err := parser.SendRequests(requestsWithContext, in, "GET", nil) 247 | 248 | assert.Nil(t, err, "expected no error") 249 | 250 | requestWithContext := <-requestsWithContext 251 | request := requestWithContext.Request 252 | requestContext := requestWithContext.RequestContext 253 | 254 | assert.Equal(t, "https://ex.com/123", request.URL.String(), "expected url") 255 | assert.Equal(t, "DELETE", request.Method, "expected method") 256 | assert.Equal(t, request.Header["Connection"][0], "keep-alive", "Connection header") 257 | assert.Equal(t, request.Header["X-Bar"][0], "corge") 258 | assert.Equal(t, "baz", requestContext, "expected context") 259 | } 260 | 261 | func TestSendJsonLinesPassesBody(t *testing.T) { 262 | requestsWithContext := make(chan parser.RequestWithContext, 3) 263 | defer close(requestsWithContext) 264 | 265 | // Define the three types of body inputs 266 | bodyInputs := []struct { 267 | bodyType string 268 | body string 269 | expected string 270 | }{ 271 | {"escaped", `"the \"body\""`, "the \"body\""}, 272 | {"base64", `"dGhlIGJvZHk="`, "the body"}, 273 | {"json", `{"key": "value"}`, `{"key": "value"}`}, 274 | {"", `{"key": "value"}`, `{"key": "value"}`}, 275 | } 276 | 277 | for _, bodyInput := range bodyInputs { 278 | inputLines := fmt.Sprintf(`{ "url": "https://ex.com/123", "body": %s, "bodyType": "%s" }`, bodyInput.body, bodyInput.bodyType) 279 | 280 | var in = strings.NewReader(inputLines) 281 | 282 | err := parser.SendRequests(requestsWithContext, in, "GET", nil) 283 | 284 | assert.Nil(t, err, "expected no error") 285 | 286 | requestWithContext := <-requestsWithContext 287 | request := requestWithContext.Request 288 | 289 | assert.Equal(t, "https://ex.com/123", request.URL.String(), "expected url") 290 | assert.NotNil(t, request.Body, "expected body") 291 | 292 | bodyBytes, _ := io.ReadAll(request.Body) 293 | bodyString := string(bodyBytes) 294 | 295 | assert.Equal(t, bodyInput.expected, bodyString, "expected body") 296 | } 297 | } 298 | 299 | func trimmedInputReader(s string) io.Reader { 300 | lines := strings.Split(s, "\n") 301 | var trimmedLines []string 302 | 303 | for _, line := range lines { 304 | trimmedLine := strings.TrimSpace(line) 305 | if len(trimmedLine) > 0 { 306 | trimmedLines = append(trimmedLines, trimmedLine) 307 | } 308 | } 309 | return strings.NewReader(strings.Join(trimmedLines, "\n")) 310 | } 311 | -------------------------------------------------------------------------------- /docs/GANDA_TOUR.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# A Tour of `ganda`\n", 8 | "\n", 9 | "This user guide is built in an interactive `bash` Jupyter Notebook. If you've got `ganda` [installed](../README.md#installation) and in your `PATH` you can run the same commands." 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 1, 15 | "metadata": { 16 | "vscode": { 17 | "languageId": "shellscript" 18 | } 19 | }, 20 | "outputs": [ 21 | { 22 | "name": "stdout", 23 | "output_type": "stream", 24 | "text": [ 25 | "ganda found in PATH\n" 26 | ] 27 | } 28 | ], 29 | "source": [ 30 | "# ensure that the ganda executable is in your PATH\n", 31 | "which ganda >/dev/null && echo \"ganda found in PATH\" || echo \"ganda not found in PATH\"" 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "metadata": { 37 | "vscode": { 38 | "languageId": "shellscript" 39 | } 40 | }, 41 | "source": [ 42 | "# `ganda` Usage" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 2, 48 | "metadata": { 49 | "vscode": { 50 | "languageId": "shellscript" 51 | } 52 | }, 53 | "outputs": [ 54 | { 55 | "name": "stdout", 56 | "output_type": "stream", 57 | "text": [ 58 | "NAME:\n", 59 | " ganda - make http requests in parallel\n", 60 | "\n", 61 | "USAGE:\n", 62 | " | ganda [options]\n", 63 | "\n", 64 | "VERSION:\n", 65 | " dev none unknown\n", 66 | "\n", 67 | "DESCRIPTION:\n", 68 | " Pipe urls to ganda over stdout to make http requests to each url in parallel.\n", 69 | "\n", 70 | "AUTHOR:\n", 71 | " Ted Naleid \n", 72 | "\n", 73 | "COMMANDS:\n", 74 | " echoserver Starts an echo server, --port to override the default port of 8080\n", 75 | " help, h Shows a list of commands or help for one command\n", 76 | "\n", 77 | "GLOBAL OPTIONS:\n", 78 | " --base-retry-millis value the base number of milliseconds to wait before retrying a request, exponential backoff is used for retries (default: 1000)\n", 79 | " --response-body value, -B value transforms the body of the response. Values: 'raw' (unchanged), 'base64', 'discard' (don't emit body), 'escaped' (JSON escaped string), 'sha256' (default: raw)\n", 80 | " --connect-timeout-millis value number of milliseconds to wait for a connection to be established before timeout (default: 10000)\n", 81 | " --header value, -H value [ --header value, -H value ] headers to send with every request, can be used multiple times (gzip and keep-alive are already there)\n", 82 | " --insecure, -k if flag is present, skip verification of https certificates (default: false)\n", 83 | " --json-envelope, -J emit result with JSON envelope with url, status, length, and body fields, assumes result is valid json (default: false)\n", 84 | " --color if flag is present, add color to success/warn messages (default: false)\n", 85 | " --output-directory value if flag is present, save response bodies to files in the specified directory\n", 86 | " --request value, -X value HTTP request method to use (default: \"GET\")\n", 87 | " --retry value max number of retries on transient errors (5XX status codes/timeouts) to attempt (default: 0)\n", 88 | " --silent, -s if flag is present, omit showing response code for each url only output response bodies (default: false)\n", 89 | " --subdir-length value length of hashed subdirectory name to put saved files when using --output-directory; use 2 for > 5k urls, 4 for > 5M urls (default: 0)\n", 90 | " --throttle-per-second value max number of requests to process per second, default is unlimited (default: -1)\n", 91 | " --workers value, -W value number of concurrent workers that will be making requests, increase this for more requests in parallel (default: 1)\n", 92 | " --help, -h show help (default: false)\n", 93 | " --version, -v print the version (default: false)\n" 94 | ] 95 | } 96 | ], 97 | "source": [ 98 | "ganda help" 99 | ] 100 | }, 101 | { 102 | "cell_type": "markdown", 103 | "metadata": { 104 | "vscode": { 105 | "languageId": "shellscript" 106 | } 107 | }, 108 | "source": [ 109 | "# `ganda` Basics\n", 110 | "\n", 111 | "\n", 112 | "`ganda` makes HTTP requests, similar to `curl`, just pipe it an URL on stdin and it will make a `GET` request and echo the body of the response on stdout. The status code of the URL will be sent to stderr.\n", 113 | "\n", 114 | "We'll use [httpbin.org](http://httpbin.org) for the first few requests. It returns a JSON representation of the request in the body of the response." 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 3, 120 | "metadata": { 121 | "vscode": { 122 | "languageId": "shellscript" 123 | } 124 | }, 125 | "outputs": [ 126 | { 127 | "name": "stdout", 128 | "output_type": "stream", 129 | "text": [ 130 | "{\n", 131 | " \"args\": {}, \n", 132 | " \"data\": \"\", \n", 133 | " \"files\": {}, \n", 134 | " \"form\": {}, \n", 135 | " \"headers\": {\n", 136 | " \"Accept-Encoding\": \"gzip\", \n", 137 | " \"Host\": \"httpbin.org\", \n", 138 | " \"User-Agent\": \"Go-http-client/1.1\", \n", 139 | " \"X-Amzn-Trace-Id\": \"Root=1-66a91ed1-104785e5182ad5d848a0096c\"\n", 140 | " }, \n", 141 | " \"json\": null, \n", 142 | " \"method\": \"GET\", \n", 143 | " \"origin\": \"173.16.32.166\", \n", 144 | " \"url\": \"http://httpbin.org/anything/1\"\n", 145 | "}\n", 146 | "Response: 200 http://httpbin.org/anything/1\n", 147 | "\n" 148 | ] 149 | } 150 | ], 151 | "source": [ 152 | "echo \"http://httpbin.org/anything/1\" | ganda " 153 | ] 154 | }, 155 | { 156 | "cell_type": "markdown", 157 | "metadata": { 158 | "vscode": { 159 | "languageId": "shellscript" 160 | } 161 | }, 162 | "source": [ 163 | "You can pipe multiple URLs to `ganda`. It happily lives in the middle of shell pipes for making requests.\n", 164 | "\n", 165 | "Here we make 3 requests to `/anything/1`, `/anything/2`, and `/anything/3` and pipe them to `jq` where we grab just the `method` and `url` properties from the response.\n", 166 | "\n", 167 | "We've also added the `-s` (silent) flag to `ganda` to suppress the stderr output that shows the url and response codes." 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": 4, 173 | "metadata": { 174 | "vscode": { 175 | "languageId": "shellscript" 176 | } 177 | }, 178 | "outputs": [ 179 | { 180 | "name": "stdout", 181 | "output_type": "stream", 182 | "text": [ 183 | "\u001b[1;39m{\u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m:\u001b[0m\u001b[0;32m\"GET\"\u001b[0m\u001b[1;39m,\u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m:\u001b[0m\u001b[0;32m\"http://httpbin.org/anything/1\"\u001b[0m\u001b[1;39m\u001b[1;39m}\u001b[0m\n", 184 | "\u001b[1;39m{\u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m:\u001b[0m\u001b[0;32m\"GET\"\u001b[0m\u001b[1;39m,\u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m:\u001b[0m\u001b[0;32m\"http://httpbin.org/anything/2\"\u001b[0m\u001b[1;39m\u001b[1;39m}\u001b[0m\n", 185 | "\u001b[1;39m{\u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m:\u001b[0m\u001b[0;32m\"GET\"\u001b[0m\u001b[1;39m,\u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m:\u001b[0m\u001b[0;32m\"http://httpbin.org/anything/3\"\u001b[0m\u001b[1;39m\u001b[1;39m}\u001b[0m\n" 186 | ] 187 | } 188 | ], 189 | "source": [ 190 | "seq 3 |\\\n", 191 | " awk '{printf \"http://httpbin.org/anything/%s\\n\", $1}' |\\\n", 192 | " ganda -s |\\\n", 193 | " jq -c '{method, url}'" 194 | ] 195 | }, 196 | { 197 | "cell_type": "markdown", 198 | "metadata": { 199 | "vscode": { 200 | "languageId": "shellscript" 201 | } 202 | }, 203 | "source": [ 204 | "## JSON Output\n", 205 | "\n", 206 | "`ganda` uses the `-J` flag for JSON output. This emits JSON with the response as the `\"body\"` field and includes other details about the request:" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 5, 212 | "metadata": { 213 | "vscode": { 214 | "languageId": "shellscript" 215 | } 216 | }, 217 | "outputs": [ 218 | { 219 | "name": "stdout", 220 | "output_type": "stream", 221 | "text": [ 222 | "\u001b[1;39m{\n", 223 | " \u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"http://httpbin.org/anything/1\"\u001b[0m\u001b[1;39m,\n", 224 | " \u001b[0m\u001b[34;1m\"code\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m200\u001b[0m\u001b[1;39m,\n", 225 | " \u001b[0m\u001b[34;1m\"body\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 226 | " \u001b[0m\u001b[34;1m\"args\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 227 | " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"\"\u001b[0m\u001b[1;39m,\n", 228 | " \u001b[0m\u001b[34;1m\"files\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 229 | " \u001b[0m\u001b[34;1m\"form\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 230 | " \u001b[0m\u001b[34;1m\"headers\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 231 | " \u001b[0m\u001b[34;1m\"Accept-Encoding\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"gzip\"\u001b[0m\u001b[1;39m,\n", 232 | " \u001b[0m\u001b[34;1m\"Host\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"httpbin.org\"\u001b[0m\u001b[1;39m,\n", 233 | " \u001b[0m\u001b[34;1m\"User-Agent\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Go-http-client/1.1\"\u001b[0m\u001b[1;39m,\n", 234 | " \u001b[0m\u001b[34;1m\"X-Amzn-Trace-Id\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Root=1-66a91ed2-5bfa363b569ce54744523bab\"\u001b[0m\u001b[1;39m\n", 235 | " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", 236 | " \u001b[0m\u001b[34;1m\"json\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;30mnull\u001b[0m\u001b[1;39m,\n", 237 | " \u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"GET\"\u001b[0m\u001b[1;39m,\n", 238 | " \u001b[0m\u001b[34;1m\"origin\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"173.16.32.166\"\u001b[0m\u001b[1;39m,\n", 239 | " \u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"http://httpbin.org/anything/1\"\u001b[0m\u001b[1;39m\n", 240 | " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", 241 | "\u001b[1;39m}\u001b[0m\n" 242 | ] 243 | } 244 | ], 245 | "source": [ 246 | "echo \"http://httpbin.org/anything/1\" |\\\n", 247 | " ganda -s -J |\\\n", 248 | " jq '.'" 249 | ] 250 | }, 251 | { 252 | "cell_type": "markdown", 253 | "metadata": { 254 | "vscode": { 255 | "languageId": "shellscript" 256 | } 257 | }, 258 | "source": [ 259 | "The body of the response is assumed to be JSON as a default. This emits the `raw` response bytes after the `\"body\"` property. If the response isn't JSON, you've got a few options for escaping/encoding the response using the `-B/--response-body ` flag:\n", 260 | "\n", 261 | "1. `raw` - the default, shown above\n", 262 | "2. `base64` - encode the bytes as a `base64` string, useful for binary content.\n", 263 | "3. `discard` - drop the bytes and set the body to `null` \n", 264 | "4. `escaped` - escape the JSON and emit the value as a String\n", 265 | "5. `sha256` - calculate the sha256 value of the body, useful for checking if the response has changed" 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": 6, 271 | "metadata": { 272 | "vscode": { 273 | "languageId": "shellscript" 274 | } 275 | }, 276 | "outputs": [ 277 | { 278 | "name": "stdout", 279 | "output_type": "stream", 280 | "text": [ 281 | "\u001b[1;39m{\n", 282 | " \u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"http://httpbin.org/anything/1\"\u001b[0m\u001b[1;39m,\n", 283 | " \u001b[0m\u001b[34;1m\"code\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m200\u001b[0m\u001b[1;39m,\n", 284 | " \u001b[0m\u001b[34;1m\"body\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ewogICJhcmdzIjoge30sIAogICJkYXRhIjogIiIsIAogICJmaWxlcyI6IHt9LCAKICAiZm9ybSI6IHt9LCAKICAiaGVhZGVycyI6IHsKICAgICJBY2NlcHQtRW5jb2RpbmciOiAiZ3ppcCIsIAogICAgIkhvc3QiOiAiaHR0cGJpbi5vcmciLCAKICAgICJVc2VyLUFnZW50IjogIkdvLWh0dHAtY2xpZW50LzEuMSIsIAogICAgIlgtQW16bi1UcmFjZS1JZCI6ICJSb290PTEtNjZhOTFlZDMtMDYwNTVmOTk0MDViNmYyOTBmOTA5ZTMzIgogIH0sIAogICJqc29uIjogbnVsbCwgCiAgIm1ldGhvZCI6ICJHRVQiLCAKICAib3JpZ2luIjogIjE3My4xNi4zMi4xNjYiLCAKICAidXJsIjogImh0dHA6Ly9odHRwYmluLm9yZy9hbnl0aGluZy8xIgp9Cg==\"\u001b[0m\u001b[1;39m\n", 285 | "\u001b[1;39m}\u001b[0m\n" 286 | ] 287 | } 288 | ], 289 | "source": [ 290 | "# base64 encode the response body\n", 291 | "echo \"http://httpbin.org/anything/1\" |\\\n", 292 | " ganda -s -J -B base64 |\\\n", 293 | " jq '.'" 294 | ] 295 | }, 296 | { 297 | "cell_type": "code", 298 | "execution_count": 7, 299 | "metadata": { 300 | "vscode": { 301 | "languageId": "shellscript" 302 | } 303 | }, 304 | "outputs": [ 305 | { 306 | "name": "stdout", 307 | "output_type": "stream", 308 | "text": [ 309 | "\u001b[1;39m{\n", 310 | " \u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"http://httpbin.org/anything/1\"\u001b[0m\u001b[1;39m,\n", 311 | " \u001b[0m\u001b[34;1m\"code\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m200\u001b[0m\u001b[1;39m,\n", 312 | " \u001b[0m\u001b[34;1m\"body\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;30mnull\u001b[0m\u001b[1;39m\n", 313 | "\u001b[1;39m}\u001b[0m\n" 314 | ] 315 | } 316 | ], 317 | "source": [ 318 | "# discard the response body\n", 319 | "echo \"http://httpbin.org/anything/1\" |\\\n", 320 | " ganda -s -J -B discard |\\\n", 321 | " jq '.'" 322 | ] 323 | }, 324 | { 325 | "cell_type": "code", 326 | "execution_count": 8, 327 | "metadata": { 328 | "vscode": { 329 | "languageId": "shellscript" 330 | } 331 | }, 332 | "outputs": [ 333 | { 334 | "name": "stdout", 335 | "output_type": "stream", 336 | "text": [ 337 | "\u001b[1;39m{\n", 338 | " \u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"http://httpbin.org/anything/1\"\u001b[0m\u001b[1;39m,\n", 339 | " \u001b[0m\u001b[34;1m\"code\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m200\u001b[0m\u001b[1;39m,\n", 340 | " \u001b[0m\u001b[34;1m\"body\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"{\\n \\\"args\\\": {}, \\n \\\"data\\\": \\\"\\\", \\n \\\"files\\\": {}, \\n \\\"form\\\": {}, \\n \\\"headers\\\": {\\n \\\"Accept-Encoding\\\": \\\"gzip\\\", \\n \\\"Host\\\": \\\"httpbin.org\\\", \\n \\\"User-Agent\\\": \\\"Go-http-client/1.1\\\", \\n \\\"X-Amzn-Trace-Id\\\": \\\"Root=1-66a91ed4-5001f6e743bce7dd42a1f5c8\\\"\\n }, \\n \\\"json\\\": null, \\n \\\"method\\\": \\\"GET\\\", \\n \\\"origin\\\": \\\"173.16.32.166\\\", \\n \\\"url\\\": \\\"http://httpbin.org/anything/1\\\"\\n}\\n\"\u001b[0m\u001b[1;39m\n", 341 | "\u001b[1;39m}\u001b[0m\n" 342 | ] 343 | } 344 | ], 345 | "source": [ 346 | "# JSON escape the response body\n", 347 | "echo \"http://httpbin.org/anything/1\" |\\\n", 348 | " ganda -s -J -B escaped |\\\n", 349 | " jq '.'" 350 | ] 351 | }, 352 | { 353 | "cell_type": "code", 354 | "execution_count": 9, 355 | "metadata": { 356 | "vscode": { 357 | "languageId": "shellscript" 358 | } 359 | }, 360 | "outputs": [ 361 | { 362 | "name": "stdout", 363 | "output_type": "stream", 364 | "text": [ 365 | "\u001b[1;39m{\n", 366 | " \u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"http://httpbin.org/anything/1\"\u001b[0m\u001b[1;39m,\n", 367 | " \u001b[0m\u001b[34;1m\"code\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m200\u001b[0m\u001b[1;39m,\n", 368 | " \u001b[0m\u001b[34;1m\"body\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"bdf9764f7059125ec2541df1e001e00a4590810e933e23c5513b0fe55962c36f\"\u001b[0m\u001b[1;39m\n", 369 | "\u001b[1;39m}\u001b[0m\n" 370 | ] 371 | } 372 | ], 373 | "source": [ 374 | "# calculate the sha256 hash of the response body\n", 375 | "echo \"http://httpbin.org/anything/1\" |\\\n", 376 | " ganda -s -J -B sha256 |\\\n", 377 | " jq '.'" 378 | ] 379 | }, 380 | { 381 | "cell_type": "markdown", 382 | "metadata": { 383 | "vscode": { 384 | "languageId": "shellscript" 385 | } 386 | }, 387 | "source": [ 388 | "## Customizing Requests with JSON Request Syntax\n", 389 | "\n", 390 | "`ganda` supports an alternate JSON-lines syntax for requests. The [JSON schema](../request.schema.json) is available, but the summary of fields it allows is:\n", 391 | "- `\"url\"` - required string - is the only required field - the request URL\n", 392 | "- `\"method\"` - optional string - a valid HTTP method (`GET|PUT|POST|DELETE|...`) - defaults to `GET`\n", 393 | "- `\"headers\"` - optional JSON object - string key/value pairs\n", 394 | "- `\"context\"` - optional JSON value - carried forward into the JSON output of the response, used to correlate requests and responses, can be a string, array, or object\n", 395 | "- `\"body\"` - optional JSON value - a string or valid JSON object, the body of the request\n", 396 | "- `\"bodyType\"` - optional enum - one of: `json` (default), `escaped`, or `base64`" 397 | ] 398 | }, 399 | { 400 | "cell_type": "markdown", 401 | "metadata": {}, 402 | "source": [ 403 | "### Adding a Request Body\n", 404 | "\n", 405 | "What if you want to `POST` instead of `GET`? `ganda` supports the same `-X ` syntax that `curl` uses:" 406 | ] 407 | }, 408 | { 409 | "cell_type": "code", 410 | "execution_count": 10, 411 | "metadata": { 412 | "vscode": { 413 | "languageId": "shellscript" 414 | } 415 | }, 416 | "outputs": [ 417 | { 418 | "name": "stdout", 419 | "output_type": "stream", 420 | "text": [ 421 | "\u001b[1;39m{\u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m:\u001b[0m\u001b[0;32m\"POST\"\u001b[0m\u001b[1;39m,\u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m:\u001b[0m\u001b[0;32m\"http://httpbin.org/anything/1\"\u001b[0m\u001b[1;39m\u001b[1;39m}\u001b[0m\n", 422 | "\u001b[1;39m{\u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m:\u001b[0m\u001b[0;32m\"POST\"\u001b[0m\u001b[1;39m,\u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m:\u001b[0m\u001b[0;32m\"http://httpbin.org/anything/2\"\u001b[0m\u001b[1;39m\u001b[1;39m}\u001b[0m\n", 423 | "\u001b[1;39m{\u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m:\u001b[0m\u001b[0;32m\"POST\"\u001b[0m\u001b[1;39m,\u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m:\u001b[0m\u001b[0;32m\"http://httpbin.org/anything/3\"\u001b[0m\u001b[1;39m\u001b[1;39m}\u001b[0m\n" 424 | ] 425 | } 426 | ], 427 | "source": [ 428 | "seq 3 |\\\n", 429 | " awk '{printf \"http://httpbin.org/anything/%s\\n\", $1}' |\\\n", 430 | " ganda -s -X POST |\\\n", 431 | " jq -c '{method, url}'" 432 | ] 433 | }, 434 | { 435 | "cell_type": "markdown", 436 | "metadata": { 437 | "vscode": { 438 | "languageId": "shellscript" 439 | } 440 | }, 441 | "source": [ 442 | "But, along with most `POST` requests, you'll want to include a body. `ganda` has an alternate JSON-lines syntax for requests that allows specifying the method and body:\n" 443 | ] 444 | }, 445 | { 446 | "cell_type": "code", 447 | "execution_count": 11, 448 | "metadata": { 449 | "vscode": { 450 | "languageId": "shellscript" 451 | } 452 | }, 453 | "outputs": [ 454 | { 455 | "name": "stdout", 456 | "output_type": "stream", 457 | "text": [ 458 | "\u001b[1;39m{\n", 459 | " \u001b[0m\u001b[34;1m\"args\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 460 | " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"{\\\"key1\\\":\\\"value1\\\",\\\"key2\\\":\\\"value2\\\"}\"\u001b[0m\u001b[1;39m,\n", 461 | " \u001b[0m\u001b[34;1m\"files\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 462 | " \u001b[0m\u001b[34;1m\"form\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 463 | " \u001b[0m\u001b[34;1m\"headers\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 464 | " \u001b[0m\u001b[34;1m\"Accept-Encoding\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"gzip\"\u001b[0m\u001b[1;39m,\n", 465 | " \u001b[0m\u001b[34;1m\"Content-Length\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"33\"\u001b[0m\u001b[1;39m,\n", 466 | " \u001b[0m\u001b[34;1m\"Host\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"httpbin.org\"\u001b[0m\u001b[1;39m,\n", 467 | " \u001b[0m\u001b[34;1m\"User-Agent\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Go-http-client/1.1\"\u001b[0m\u001b[1;39m,\n", 468 | " \u001b[0m\u001b[34;1m\"X-Amzn-Trace-Id\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Root=1-66a91ed6-4f8f6bd2228d0ee141fdaec3\"\u001b[0m\u001b[1;39m\n", 469 | " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", 470 | " \u001b[0m\u001b[34;1m\"json\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 471 | " \u001b[0m\u001b[34;1m\"key1\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"value1\"\u001b[0m\u001b[1;39m,\n", 472 | " \u001b[0m\u001b[34;1m\"key2\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"value2\"\u001b[0m\u001b[1;39m\n", 473 | " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", 474 | " \u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"POST\"\u001b[0m\u001b[1;39m,\n", 475 | " \u001b[0m\u001b[34;1m\"origin\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"173.16.32.166\"\u001b[0m\u001b[1;39m,\n", 476 | " \u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"http://httpbin.org/anything/1\"\u001b[0m\u001b[1;39m\n", 477 | "\u001b[1;39m}\u001b[0m\n" 478 | ] 479 | } 480 | ], 481 | "source": [ 482 | "echo '\n", 483 | "{ \n", 484 | " \"method\": \"POST\", \n", 485 | " \"url\": \"http://httpbin.org/anything/1\", \n", 486 | " \"body\": { \"key1\": \"value1\", \"key2\": \"value2\" } \n", 487 | "}' |\\\n", 488 | " # ganda wants JSON-lines input, use jq -c to compact the JSON to a single line\n", 489 | " jq -c '.' |\\\n", 490 | " ganda -s |\\\n", 491 | " jq '.'" 492 | ] 493 | }, 494 | { 495 | "cell_type": "markdown", 496 | "metadata": {}, 497 | "source": [ 498 | "By default, it assumes that the body in the request JSON is also a valid JSON object.\n", 499 | "\n", 500 | "If you have a body value that isn't valid JSON, `ganda` provides two different transformers via the `\"bodyType\"` JSON field:\n", 501 | "1. `\"bodyType\": \"escaped\"` - `ganda` will unescape the value before making the request. This can be useful for escaped JSON (maybe that contains newlines), or for a uuencoded/base64 value that you store as a JSON string, but want the string quotes (`\"`) stripped from the ends.\n", 502 | "2. `\"bodyType\": \"base64\"` - `ganda` will `base64` decode the value before sending it as the body. This is most often used with binary data that you want to `POST`." 503 | ] 504 | }, 505 | { 506 | "cell_type": "code", 507 | "execution_count": 12, 508 | "metadata": { 509 | "vscode": { 510 | "languageId": "shellscript" 511 | } 512 | }, 513 | "outputs": [ 514 | { 515 | "name": "stdout", 516 | "output_type": "stream", 517 | "text": [ 518 | "\u001b[1;39m{\n", 519 | " \u001b[0m\u001b[34;1m\"args\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 520 | " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"{ \\\"key1\\\": \\\"value1\\\" }\"\u001b[0m\u001b[1;39m,\n", 521 | " \u001b[0m\u001b[34;1m\"files\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 522 | " \u001b[0m\u001b[34;1m\"form\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 523 | " \u001b[0m\u001b[34;1m\"headers\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 524 | " \u001b[0m\u001b[34;1m\"Accept-Encoding\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"gzip\"\u001b[0m\u001b[1;39m,\n", 525 | " \u001b[0m\u001b[34;1m\"Content-Length\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"20\"\u001b[0m\u001b[1;39m,\n", 526 | " \u001b[0m\u001b[34;1m\"Host\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"httpbin.org\"\u001b[0m\u001b[1;39m,\n", 527 | " \u001b[0m\u001b[34;1m\"User-Agent\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Go-http-client/1.1\"\u001b[0m\u001b[1;39m,\n", 528 | " \u001b[0m\u001b[34;1m\"X-Amzn-Trace-Id\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Root=1-66a91ed7-19039110465d24d47e32e9ff\"\u001b[0m\u001b[1;39m\n", 529 | " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", 530 | " \u001b[0m\u001b[34;1m\"json\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 531 | " \u001b[0m\u001b[34;1m\"key1\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"value1\"\u001b[0m\u001b[1;39m\n", 532 | " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", 533 | " \u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"POST\"\u001b[0m\u001b[1;39m,\n", 534 | " \u001b[0m\u001b[34;1m\"origin\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"173.16.32.166\"\u001b[0m\u001b[1;39m,\n", 535 | " \u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"http://httpbin.org/anything/1\"\u001b[0m\u001b[1;39m\n", 536 | "\u001b[1;39m}\u001b[0m\n" 537 | ] 538 | } 539 | ], 540 | "source": [ 541 | "# \"bodyType\": \"escaped\" - ganda will unescape before making the request\n", 542 | "echo '\n", 543 | "{ \n", 544 | " \"url\": \"http://httpbin.org/anything/1\", \n", 545 | " \"bodyType\": \"escaped\", \n", 546 | " \"body\": \"{ \\\"key1\\\": \\\"value1\\\" }\" \n", 547 | "}' |\\\n", 548 | " jq -c '.' |\\\n", 549 | " ganda -s -X POST |\\\n", 550 | " jq '.'" 551 | ] 552 | }, 553 | { 554 | "cell_type": "markdown", 555 | "metadata": {}, 556 | "source": [ 557 | "So above, `escaped` told `ganda` to unescape the value before sending it, and this is the JSON object that was sent:\n", 558 | "```json\n", 559 | "{ \"key1\": \"value1\" }\n", 560 | "```\n", 561 | "\n", 562 | "next, we'll show how a `base64` encoded value, sent in the JSON as a string (with double quotes around it), will `base64` decode the string and send the literal JSON value of: \n", 563 | "\n", 564 | "```\n", 565 | "{ \"value\": \"was base64 escaped\" }\n", 566 | "```\n", 567 | "\n", 568 | "Normally, this is used for encoding binary files, but that doesn't render well in the response, so we're `base64` encoding JSON as an example." 569 | ] 570 | }, 571 | { 572 | "cell_type": "code", 573 | "execution_count": 13, 574 | "metadata": { 575 | "vscode": { 576 | "languageId": "shellscript" 577 | } 578 | }, 579 | "outputs": [ 580 | { 581 | "name": "stdout", 582 | "output_type": "stream", 583 | "text": [ 584 | "\u001b[1;39m{\n", 585 | " \u001b[0m\u001b[34;1m\"args\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 586 | " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"{ \\\"value\\\": \\\"was base64 escaped\\\" }\"\u001b[0m\u001b[1;39m,\n", 587 | " \u001b[0m\u001b[34;1m\"files\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 588 | " \u001b[0m\u001b[34;1m\"form\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 589 | " \u001b[0m\u001b[34;1m\"headers\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 590 | " \u001b[0m\u001b[34;1m\"Accept-Encoding\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"gzip\"\u001b[0m\u001b[1;39m,\n", 591 | " \u001b[0m\u001b[34;1m\"Content-Length\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"33\"\u001b[0m\u001b[1;39m,\n", 592 | " \u001b[0m\u001b[34;1m\"Host\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"httpbin.org\"\u001b[0m\u001b[1;39m,\n", 593 | " \u001b[0m\u001b[34;1m\"User-Agent\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Go-http-client/1.1\"\u001b[0m\u001b[1;39m,\n", 594 | " \u001b[0m\u001b[34;1m\"X-Amzn-Trace-Id\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Root=1-66a91ed8-78025df11baf73b06bfeed38\"\u001b[0m\u001b[1;39m\n", 595 | " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", 596 | " \u001b[0m\u001b[34;1m\"json\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 597 | " \u001b[0m\u001b[34;1m\"value\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"was base64 escaped\"\u001b[0m\u001b[1;39m\n", 598 | " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", 599 | " \u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"POST\"\u001b[0m\u001b[1;39m,\n", 600 | " \u001b[0m\u001b[34;1m\"origin\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"173.16.32.166\"\u001b[0m\u001b[1;39m,\n", 601 | " \u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"http://httpbin.org/anything/1\"\u001b[0m\u001b[1;39m\n", 602 | "\u001b[1;39m}\u001b[0m\n" 603 | ] 604 | } 605 | ], 606 | "source": [ 607 | "# \"bodyType\": \"base64\" - ganda will decode before making the request\n", 608 | "# generated with: echo -n '{ \"value\": \"was base64 escaped\" }' | base64\n", 609 | "echo '\n", 610 | "{ \n", 611 | " \"url\": \"http://httpbin.org/anything/1\", \n", 612 | " \"bodyType\": \"base64\", \n", 613 | " \"body\": \"eyAidmFsdWUiOiAid2FzIGJhc2U2NCBlc2NhcGVkIiB9\" \n", 614 | "}' |\\\n", 615 | " jq -c '.' |\\\n", 616 | " ganda -s -X POST |\\\n", 617 | " jq '.'" 618 | ] 619 | }, 620 | { 621 | "cell_type": "markdown", 622 | "metadata": { 623 | "vscode": { 624 | "languageId": "shellscript" 625 | } 626 | }, 627 | "source": [ 628 | "### Request Headers\n", 629 | "\n", 630 | "`ganda` allows adding static headers to every request with the `-H key:value` syntax. Multiple headers can be specified, and they'll override defaulted values:" 631 | ] 632 | }, 633 | { 634 | "cell_type": "code", 635 | "execution_count": 14, 636 | "metadata": { 637 | "vscode": { 638 | "languageId": "shellscript" 639 | } 640 | }, 641 | "outputs": [ 642 | { 643 | "name": "stdout", 644 | "output_type": "stream", 645 | "text": [ 646 | "\u001b[1;39m{\n", 647 | " \u001b[0m\u001b[34;1m\"args\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 648 | " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"\"\u001b[0m\u001b[1;39m,\n", 649 | " \u001b[0m\u001b[34;1m\"files\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 650 | " \u001b[0m\u001b[34;1m\"form\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 651 | " \u001b[0m\u001b[34;1m\"headers\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 652 | " \u001b[0m\u001b[34;1m\"Accept-Encoding\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"gzip\"\u001b[0m\u001b[1;39m,\n", 653 | " \u001b[0m\u001b[34;1m\"Host\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"httpbin.org\"\u001b[0m\u001b[1;39m,\n", 654 | " \u001b[0m\u001b[34;1m\"User-Agent\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"static-ganda\"\u001b[0m\u001b[1;39m,\n", 655 | " \u001b[0m\u001b[34;1m\"X-Amzn-Trace-Id\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Root=1-66a91ed9-7c95dd6518c096e9249ae3ef\"\u001b[0m\u001b[1;39m,\n", 656 | " \u001b[0m\u001b[34;1m\"X-My-Header\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"1234\"\u001b[0m\u001b[1;39m\n", 657 | " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", 658 | " \u001b[0m\u001b[34;1m\"json\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;30mnull\u001b[0m\u001b[1;39m,\n", 659 | " \u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"GET\"\u001b[0m\u001b[1;39m,\n", 660 | " \u001b[0m\u001b[34;1m\"origin\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"173.16.32.166\"\u001b[0m\u001b[1;39m,\n", 661 | " \u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"http://httpbin.org/anything/1\"\u001b[0m\u001b[1;39m\n", 662 | "\u001b[1;39m}\u001b[0m\n" 663 | ] 664 | } 665 | ], 666 | "source": [ 667 | "echo '{ \"url\": \"http://httpbin.org/anything/1\" }' |\\\n", 668 | " ganda -s -H \"X-My-Header: 1234\" -H \"User-Agent: static-ganda\" |\\\n", 669 | " jq '.'" 670 | ] 671 | }, 672 | { 673 | "cell_type": "markdown", 674 | "metadata": { 675 | "vscode": { 676 | "languageId": "shellscript" 677 | } 678 | }, 679 | "source": [ 680 | "The JSON-lines syntax can also specify per-request headers that will override static headers.\n", 681 | "\n", 682 | "Here, the `User-Agent` in the JSON overrides the static header `User-Agent` from the `-H` flag:" 683 | ] 684 | }, 685 | { 686 | "cell_type": "code", 687 | "execution_count": 15, 688 | "metadata": { 689 | "vscode": { 690 | "languageId": "shellscript" 691 | } 692 | }, 693 | "outputs": [ 694 | { 695 | "name": "stdout", 696 | "output_type": "stream", 697 | "text": [ 698 | "\u001b[1;39m{\n", 699 | " \u001b[0m\u001b[34;1m\"args\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 700 | " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"\"\u001b[0m\u001b[1;39m,\n", 701 | " \u001b[0m\u001b[34;1m\"files\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 702 | " \u001b[0m\u001b[34;1m\"form\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", 703 | " \u001b[0m\u001b[34;1m\"headers\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 704 | " \u001b[0m\u001b[34;1m\"Accept-Encoding\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"gzip\"\u001b[0m\u001b[1;39m,\n", 705 | " \u001b[0m\u001b[34;1m\"Host\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"httpbin.org\"\u001b[0m\u001b[1;39m,\n", 706 | " \u001b[0m\u001b[34;1m\"User-Agent\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"per-request-ganda\"\u001b[0m\u001b[1;39m,\n", 707 | " \u001b[0m\u001b[34;1m\"X-Amzn-Trace-Id\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Root=1-66a91eda-3f42e5e7611abb683e39c0d4\"\u001b[0m\u001b[1;39m,\n", 708 | " \u001b[0m\u001b[34;1m\"X-My-Header\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"1234\"\u001b[0m\u001b[1;39m,\n", 709 | " \u001b[0m\u001b[34;1m\"X-Second-Header\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"5678\"\u001b[0m\u001b[1;39m\n", 710 | " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", 711 | " \u001b[0m\u001b[34;1m\"json\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;30mnull\u001b[0m\u001b[1;39m,\n", 712 | " \u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"GET\"\u001b[0m\u001b[1;39m,\n", 713 | " \u001b[0m\u001b[34;1m\"origin\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"173.16.32.166\"\u001b[0m\u001b[1;39m,\n", 714 | " \u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"http://httpbin.org/anything/1\"\u001b[0m\u001b[1;39m\n", 715 | "\u001b[1;39m}\u001b[0m\n" 716 | ] 717 | } 718 | ], 719 | "source": [ 720 | "echo '\n", 721 | "{ \n", 722 | " \"url\": \"http://httpbin.org/anything/1\", \n", 723 | " \"headers\": {\"X-Second-Header\": \"5678\", \"User-Agent\": \"per-request-ganda\" } \n", 724 | "}' |\\\n", 725 | " jq -c '.' |\\\n", 726 | " ganda -s -H \"X-My-Header: 1234\" -H \"User-Agent: static-ganda\" |\\\n", 727 | " jq '.'" 728 | ] 729 | }, 730 | { 731 | "cell_type": "markdown", 732 | "metadata": { 733 | "vscode": { 734 | "languageId": "shellscript" 735 | } 736 | }, 737 | "source": [ 738 | "## Request Context\n", 739 | "\n", 740 | "Your requests are part of a pipeline, what if you want to carry context through your pipeline that isn't part of the HTTP request/response?\n", 741 | "\n", 742 | "An example would be calling an HTTP endpoint to generate a new UUID, but the response does not include the ID that we want to associate with the UUID.\n", 743 | "\n", 744 | "`ganda` allows you to specify values along with the URL that will still be present in the JSON envelope output.\n", 745 | "\n", 746 | "This can be done with the simple request syntax by specifying tab-separated values after the URL:\n" 747 | ] 748 | }, 749 | { 750 | "cell_type": "code", 751 | "execution_count": 16, 752 | "metadata": { 753 | "vscode": { 754 | "languageId": "shellscript" 755 | } 756 | }, 757 | "outputs": [ 758 | { 759 | "name": "stdout", 760 | "output_type": "stream", 761 | "text": [ 762 | "http://httpbin.org/uuid\t1\t\"single\tvalue\twith\ttabs\"\n" 763 | ] 764 | } 765 | ], 766 | "source": [ 767 | "# echo can emit tab separated values for correlating requests and responses\n", 768 | "# here is what is being passed to ganda:\n", 769 | "echo -e 'http://httpbin.org/uuid\\t1\\t\"single\\tvalue\\twith\\ttabs\"'" 770 | ] 771 | }, 772 | { 773 | "cell_type": "code", 774 | "execution_count": 17, 775 | "metadata": { 776 | "vscode": { 777 | "languageId": "shellscript" 778 | } 779 | }, 780 | "outputs": [ 781 | { 782 | "name": "stdout", 783 | "output_type": "stream", 784 | "text": [ 785 | "\u001b[1;39m{\n", 786 | " \u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"http://httpbin.org/uuid\"\u001b[0m\u001b[1;39m,\n", 787 | " \u001b[0m\u001b[34;1m\"code\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m200\u001b[0m\u001b[1;39m,\n", 788 | " \u001b[0m\u001b[34;1m\"body\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 789 | " \u001b[0m\u001b[34;1m\"uuid\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"3046801f-8a41-435f-ac47-12ebf593d0c5\"\u001b[0m\u001b[1;39m\n", 790 | " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", 791 | " \u001b[0m\u001b[34;1m\"context\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", 792 | " \u001b[0;32m\"1\"\u001b[0m\u001b[1;39m,\n", 793 | " \u001b[0;32m\"single\\tvalue\\twith\\ttabs\"\u001b[0m\u001b[1;39m\n", 794 | " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", 795 | "\u001b[1;39m}\u001b[0m\n" 796 | ] 797 | } 798 | ], 799 | "source": [ 800 | "echo -e 'http://httpbin.org/uuid\\t1\\t\"single\\tvalue\\twith\\ttabs\"' |\\\n", 801 | " ganda -s -J |\\\n", 802 | " jq '.'" 803 | ] 804 | }, 805 | { 806 | "cell_type": "markdown", 807 | "metadata": {}, 808 | "source": [ 809 | "Notice the `\"context\"` emitted at the bottom of the JSON.\n", 810 | "\n", 811 | "The JSON-lines request format also allows context to be specified, and it can be any valid JSON object (string, array, or object):" 812 | ] 813 | }, 814 | { 815 | "cell_type": "code", 816 | "execution_count": 18, 817 | "metadata": { 818 | "vscode": { 819 | "languageId": "shellscript" 820 | } 821 | }, 822 | "outputs": [ 823 | { 824 | "name": "stdout", 825 | "output_type": "stream", 826 | "text": [ 827 | "\u001b[1;39m{\n", 828 | " \u001b[0m\u001b[34;1m\"url\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"http://httpbin.org/uuid\"\u001b[0m\u001b[1;39m,\n", 829 | " \u001b[0m\u001b[34;1m\"code\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m200\u001b[0m\u001b[1;39m,\n", 830 | " \u001b[0m\u001b[34;1m\"body\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 831 | " \u001b[0m\u001b[34;1m\"uuid\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"108e190d-4d8b-4873-8bf5-4d608c5117a6\"\u001b[0m\u001b[1;39m\n", 832 | " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", 833 | " \u001b[0m\u001b[34;1m\"context\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 834 | " \u001b[0m\u001b[34;1m\"id\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1\u001b[0m\u001b[1;39m,\n", 835 | " \u001b[0m\u001b[34;1m\"value\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"correlation value\"\u001b[0m\u001b[1;39m\n", 836 | " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", 837 | "\u001b[1;39m}\u001b[0m\n" 838 | ] 839 | } 840 | ], 841 | "source": [ 842 | "echo '\n", 843 | "{ \n", 844 | " \"url\": \"http://httpbin.org/uuid\", \n", 845 | " \"context\": { \"id\": 1, \"value\": \"correlation value\"} \n", 846 | "}' |\\\n", 847 | " jq -c '.' |\\\n", 848 | " ganda -s -J |\\\n", 849 | " jq '.'" 850 | ] 851 | }, 852 | { 853 | "cell_type": "markdown", 854 | "metadata": { 855 | "vscode": { 856 | "languageId": "shellscript" 857 | } 858 | }, 859 | "source": [ 860 | "## `ganda echoserver` - a simple server that echoes requests \n", 861 | "\n", 862 | "`ganda` comes with a built-in echo server to make verifying requests easier. \n", 863 | "\n", 864 | "We don't want to hammer the public `httpbin.org` server, so let's fire up `ganda echoserver` as a background process and use that instead. " 865 | ] 866 | }, 867 | { 868 | "cell_type": "code", 869 | "execution_count": 19, 870 | "metadata": { 871 | "vscode": { 872 | "languageId": "shellscript" 873 | } 874 | }, 875 | "outputs": [ 876 | { 877 | "name": "stdout", 878 | "output_type": "stream", 879 | "text": [ 880 | "NAME:\n", 881 | " ganda echoserver - Starts an echo server, --port to override the default port of 8080\n", 882 | "\n", 883 | "USAGE:\n", 884 | " ganda echoserver [command [command options]] \n", 885 | "\n", 886 | "OPTIONS:\n", 887 | " --port value Port number to start the echo server on (default: 8080)\n", 888 | " --delay-millis value Number of milliseconds to delay responding (default: 0)\n", 889 | " --help, -h show help (default: false)\n" 890 | ] 891 | } 892 | ], 893 | "source": [ 894 | "ganda echoserver --help" 895 | ] 896 | }, 897 | { 898 | "cell_type": "markdown", 899 | "metadata": { 900 | "vscode": { 901 | "languageId": "shellscript" 902 | } 903 | }, 904 | "source": [ 905 | "Normally, we'd run `ganda echoserver` in another terminal window with a command like:\n", 906 | "\n", 907 | "```\n", 908 | "ganda echoserver --port 9090\n", 909 | "``` \n", 910 | "For this notebook, we'll run the echoserver in the background with a 1 second delay on every response. We'll also suppress its logging output to stdout:" 911 | ] 912 | }, 913 | { 914 | "cell_type": "code", 915 | "execution_count": 20, 916 | "metadata": { 917 | "vscode": { 918 | "languageId": "shellscript" 919 | } 920 | }, 921 | "outputs": [ 922 | { 923 | "name": "stdout", 924 | "output_type": "stream", 925 | "text": [ 926 | "[1] 39607\n" 927 | ] 928 | } 929 | ], 930 | "source": [ 931 | "# run the echoserver in the background. give it a 1000ms/1s delay for responding to each request. \n", 932 | "# If you're running it in a separate terminal, you can omit the `>/dev/null &` part\n", 933 | "ganda echoserver --port 9090 --delay-millis 1000 >/dev/null &" 934 | ] 935 | }, 936 | { 937 | "cell_type": "markdown", 938 | "metadata": { 939 | "vscode": { 940 | "languageId": "shellscript" 941 | } 942 | }, 943 | "source": [ 944 | "Let's use `ganda` to make a single request to our echoserver so we can see its output. It takes about a second because of the echoserver delay." 945 | ] 946 | }, 947 | { 948 | "cell_type": "code", 949 | "execution_count": 21, 950 | "metadata": { 951 | "vscode": { 952 | "languageId": "shellscript" 953 | } 954 | }, 955 | "outputs": [ 956 | { 957 | "name": "stdout", 958 | "output_type": "stream", 959 | "text": [ 960 | "\u001b[1;39m{\n", 961 | " \u001b[0m\u001b[34;1m\"time\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"2024-07-30T12:11:59-05:00\"\u001b[0m\u001b[1;39m,\n", 962 | " \u001b[0m\u001b[34;1m\"id\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"\"\u001b[0m\u001b[1;39m,\n", 963 | " \u001b[0m\u001b[34;1m\"remote_ip\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"::1\"\u001b[0m\u001b[1;39m,\n", 964 | " \u001b[0m\u001b[34;1m\"host\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"localhost:9090\"\u001b[0m\u001b[1;39m,\n", 965 | " \u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"GET\"\u001b[0m\u001b[1;39m,\n", 966 | " \u001b[0m\u001b[34;1m\"uri\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"/anything/1\"\u001b[0m\u001b[1;39m,\n", 967 | " \u001b[0m\u001b[34;1m\"user_agent\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Go-http-client/1.1\"\u001b[0m\u001b[1;39m,\n", 968 | " \u001b[0m\u001b[34;1m\"status\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m200\u001b[0m\u001b[1;39m,\n", 969 | " \u001b[0m\u001b[34;1m\"headers\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 970 | " \u001b[0m\u001b[34;1m\"Accept-Encoding\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"gzip\"\u001b[0m\u001b[1;39m,\n", 971 | " \u001b[0m\u001b[34;1m\"Connection\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"keep-alive\"\u001b[0m\u001b[1;39m,\n", 972 | " \u001b[0m\u001b[34;1m\"User-Agent\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Go-http-client/1.1\"\u001b[0m\u001b[1;39m\n", 973 | " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", 974 | " \u001b[0m\u001b[34;1m\"request_body\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"\"\u001b[0m\u001b[1;39m\n", 975 | "\u001b[1;39m}\u001b[0m\n", 976 | "\n", 977 | "real\t0m1.019s\n", 978 | "user\t0m0.027s\n", 979 | "sys\t0m0.008s\n" 980 | ] 981 | } 982 | ], 983 | "source": [ 984 | "time echo \"http://localhost:9090/anything/1\" | ganda -s | jq '.'" 985 | ] 986 | }, 987 | { 988 | "cell_type": "markdown", 989 | "metadata": { 990 | "vscode": { 991 | "languageId": "shellscript" 992 | } 993 | }, 994 | "source": [ 995 | "## Parallelizing Requests\n", 996 | "\n", 997 | "By default, `ganda` using a single worker thread and a single connection to make requests. This will guarantee that requests are made in the order they are received. \n", 998 | "\n", 999 | "If order doesn't matter, and you'd like to increase throughput, we can use the `-W ` command." 1000 | ] 1001 | }, 1002 | { 1003 | "cell_type": "code", 1004 | "execution_count": 22, 1005 | "metadata": { 1006 | "vscode": { 1007 | "languageId": "shellscript" 1008 | } 1009 | }, 1010 | "outputs": [ 1011 | { 1012 | "name": "stdout", 1013 | "output_type": "stream", 1014 | "text": [ 1015 | "Response: 200 http://localhost:9090/slow-api/1\n", 1016 | "Response: 200 http://localhost:9090/slow-api/2\n", 1017 | "Response: 200 http://localhost:9090/slow-api/3\n", 1018 | "Response: 200 http://localhost:9090/slow-api/4\n", 1019 | "Response: 200 http://localhost:9090/slow-api/5\n", 1020 | "Response: 200 http://localhost:9090/slow-api/6\n", 1021 | "Response: 200 http://localhost:9090/slow-api/7\n", 1022 | "Response: 200 http://localhost:9090/slow-api/8\n", 1023 | "Response: 200 http://localhost:9090/slow-api/9\n", 1024 | "Response: 200 http://localhost:9090/slow-api/10\n", 1025 | "10.0 0:00:10 [ 995m/s] [ 995m/s]\n" 1026 | ] 1027 | } 1028 | ], 1029 | "source": [ 1030 | "# our echoserver is running with a 1000ms delay. about 10 seconds to complete with the single default worker\n", 1031 | "seq 10 |\\\n", 1032 | " awk '{printf \"http://localhost:9090/slow-api/%s\\n\", $1}' |\\\n", 1033 | " ganda |\\\n", 1034 | " # use pv - pipeviewer - to show the total number of requests, the total time taken, and the rate of requests per second\n", 1035 | " pv -albert > /dev/null" 1036 | ] 1037 | }, 1038 | { 1039 | "cell_type": "markdown", 1040 | "metadata": { 1041 | "vscode": { 1042 | "languageId": "shellscript" 1043 | } 1044 | }, 1045 | "source": [ 1046 | "If we increase the number of workers to 10, we should finish in about a second" 1047 | ] 1048 | }, 1049 | { 1050 | "cell_type": "code", 1051 | "execution_count": 23, 1052 | "metadata": { 1053 | "vscode": { 1054 | "languageId": "shellscript" 1055 | } 1056 | }, 1057 | "outputs": [ 1058 | { 1059 | "name": "stdout", 1060 | "output_type": "stream", 1061 | "text": [ 1062 | "Response: 200 http://localhost:9090/slow-api/8\n", 1063 | "Response: 200 http://localhost:9090/slow-api/2\n", 1064 | "Response: 200 http://localhost:9090/slow-api/10\n", 1065 | "Response: 200 http://localhost:9090/slow-api/5\n", 1066 | "Response: 200 http://localhost:9090/slow-api/4\n", 1067 | "Response: 200 http://localhost:9090/slow-api/9\n", 1068 | "Response: 200 http://localhost:9090/slow-api/6\n", 1069 | "Response: 200 http://localhost:9090/slow-api/1\n", 1070 | "Response: 200 http://localhost:9090/slow-api/7\n", 1071 | "Response: 200 http://localhost:9090/slow-api/3\n", 1072 | "10.0 0:00:01 [9.78 /s] [9.78 /s]\n" 1073 | ] 1074 | } 1075 | ], 1076 | "source": [ 1077 | "# our echoserver is running with a 1 second delay so the 10 requests \n", 1078 | "# should be handled by 10 workers in about 1 second\n", 1079 | "seq 10 |\\\n", 1080 | " awk '{printf \"http://localhost:9090/slow-api/%s\\n\", $1}' |\\\n", 1081 | " ganda -W 10 |\\\n", 1082 | " pv -albert > /dev/null" 1083 | ] 1084 | }, 1085 | { 1086 | "cell_type": "code", 1087 | "execution_count": 24, 1088 | "metadata": { 1089 | "vscode": { 1090 | "languageId": "shellscript" 1091 | } 1092 | }, 1093 | "outputs": [ 1094 | { 1095 | "name": "stdout", 1096 | "output_type": "stream", 1097 | "text": [ 1098 | "1.00k 0:00:10 [99.0 /s] [99.0 /s]\n" 1099 | ] 1100 | } 1101 | ], 1102 | "source": [ 1103 | "# if we use 100 parallel workers and make 1k requests that \n", 1104 | "# each take 1 second, it should take about 10 seconds\n", 1105 | "seq 1000 |\\\n", 1106 | " awk '{printf \"http://localhost:9090/slow-api/%s\\n\", $1}' |\\\n", 1107 | " ganda -s -W 100 | \n", 1108 | " pv -albert > /dev/null" 1109 | ] 1110 | }, 1111 | { 1112 | "cell_type": "markdown", 1113 | "metadata": { 1114 | "vscode": { 1115 | "languageId": "shellscript" 1116 | } 1117 | }, 1118 | "source": [ 1119 | "`ganda` also supports throttling the number of requests its workers will make using the `--throttle-per-second ` flag.\n", 1120 | "\n", 1121 | "If we use 100 parallel workers, but throttle them so that they can only make 5 requests per second, it should take about 20 seconds to complete 100 requests." 1122 | ] 1123 | }, 1124 | { 1125 | "cell_type": "code", 1126 | "execution_count": 25, 1127 | "metadata": { 1128 | "vscode": { 1129 | "languageId": "shellscript" 1130 | } 1131 | }, 1132 | "outputs": [ 1133 | { 1134 | "name": "stdout", 1135 | "output_type": "stream", 1136 | "text": [ 1137 | " 100 0:00:21 [4.76 /s] [4.76 /s]\n" 1138 | ] 1139 | } 1140 | ], 1141 | "source": [ 1142 | "seq 100 |\\\n", 1143 | " awk '{printf \"http://localhost:9090/slow-api/%s\\n\", $1}' |\\\n", 1144 | " ganda -s -W 100 --throttle-per-second 5 |\\\n", 1145 | " pv -albert > /dev/null" 1146 | ] 1147 | }, 1148 | { 1149 | "cell_type": "code", 1150 | "execution_count": 26, 1151 | "metadata": { 1152 | "vscode": { 1153 | "languageId": "shellscript" 1154 | } 1155 | }, 1156 | "outputs": [ 1157 | { 1158 | "name": "stdout", 1159 | "output_type": "stream", 1160 | "text": [ 1161 | "echoserver stopped\n" 1162 | ] 1163 | } 1164 | ], 1165 | "source": [ 1166 | "# clean up the delay echoserver that we'd previously run in the background\n", 1167 | "pkill ganda && echo \"echoserver stopped\" || echo \"echoserver not stopped\"" 1168 | ] 1169 | }, 1170 | { 1171 | "cell_type": "markdown", 1172 | "metadata": { 1173 | "vscode": { 1174 | "languageId": "shellscript" 1175 | } 1176 | }, 1177 | "source": [ 1178 | "## Saving Individual Responses to Files\n", 1179 | "\n", 1180 | "`ganda` also supports saving individual responses to files. This can be useful for debugging or for saving responses for later analysis. \n", 1181 | "\n", 1182 | "The `--output-directory ` flag is used to specify the directory where the responses should be saved." 1183 | ] 1184 | }, 1185 | { 1186 | "cell_type": "code", 1187 | "execution_count": 27, 1188 | "metadata": { 1189 | "vscode": { 1190 | "languageId": "shellscript" 1191 | } 1192 | }, 1193 | "outputs": [ 1194 | { 1195 | "name": "stdout", 1196 | "output_type": "stream", 1197 | "text": [ 1198 | "[1] 39629\n" 1199 | ] 1200 | } 1201 | ], 1202 | "source": [ 1203 | "# start up a echoserver that has no delay on port 9090 and put it in the background\n", 1204 | "ganda echoserver --port 9090 >/dev/null &" 1205 | ] 1206 | }, 1207 | { 1208 | "cell_type": "markdown", 1209 | "metadata": { 1210 | "vscode": { 1211 | "languageId": "shellscript" 1212 | } 1213 | }, 1214 | "source": [ 1215 | "Let's make 10 requests and save them as individual files in the `scratch/ten` directory" 1216 | ] 1217 | }, 1218 | { 1219 | "cell_type": "code", 1220 | "execution_count": 28, 1221 | "metadata": { 1222 | "vscode": { 1223 | "languageId": "shellscript" 1224 | } 1225 | }, 1226 | "outputs": [ 1227 | { 1228 | "name": "stdout", 1229 | "output_type": "stream", 1230 | "text": [ 1231 | "Response: 200 http://localhost:9090/fast-api/1 -> scratch/ten/http-localhost-9090-fast-api-1\n", 1232 | "Response: 200 http://localhost:9090/fast-api/2 -> scratch/ten/http-localhost-9090-fast-api-2\n", 1233 | "Response: 200 http://localhost:9090/fast-api/3 -> scratch/ten/http-localhost-9090-fast-api-3\n", 1234 | "Response: 200 http://localhost:9090/fast-api/4 -> scratch/ten/http-localhost-9090-fast-api-4\n", 1235 | "Response: 200 http://localhost:9090/fast-api/5 -> scratch/ten/http-localhost-9090-fast-api-5\n", 1236 | "Response: 200 http://localhost:9090/fast-api/6 -> scratch/ten/http-localhost-9090-fast-api-6\n", 1237 | "Response: 200 http://localhost:9090/fast-api/7 -> scratch/ten/http-localhost-9090-fast-api-7\n", 1238 | "Response: 200 http://localhost:9090/fast-api/8 -> scratch/ten/http-localhost-9090-fast-api-8\n", 1239 | "Response: 200 http://localhost:9090/fast-api/9 -> scratch/ten/http-localhost-9090-fast-api-9\n", 1240 | "Response: 200 http://localhost:9090/fast-api/10 -> scratch/ten/http-localhost-9090-fast-api-10\n", 1241 | "0.00 0:00:00 [0.00 /s] [0.00 /s]\n" 1242 | ] 1243 | } 1244 | ], 1245 | "source": [ 1246 | "seq 10 |\\\n", 1247 | " awk '{printf \"http://localhost:9090/fast-api/%s\\n\", $1}' |\\\n", 1248 | " ganda -W 1 --output-directory scratch/ten |\\\n", 1249 | " pv -albert > /dev/null" 1250 | ] 1251 | }, 1252 | { 1253 | "cell_type": "code", 1254 | "execution_count": 29, 1255 | "metadata": { 1256 | "vscode": { 1257 | "languageId": "shellscript" 1258 | } 1259 | }, 1260 | "outputs": [ 1261 | { 1262 | "name": "stdout", 1263 | "output_type": "stream", 1264 | "text": [ 1265 | "http-localhost-9090-fast-api-1\thttp-localhost-9090-fast-api-5\n", 1266 | "http-localhost-9090-fast-api-10\thttp-localhost-9090-fast-api-6\n", 1267 | "http-localhost-9090-fast-api-2\thttp-localhost-9090-fast-api-7\n", 1268 | "http-localhost-9090-fast-api-3\thttp-localhost-9090-fast-api-8\n", 1269 | "http-localhost-9090-fast-api-4\thttp-localhost-9090-fast-api-9\n" 1270 | ] 1271 | } 1272 | ], 1273 | "source": [ 1274 | "ls scratch/ten" 1275 | ] 1276 | }, 1277 | { 1278 | "cell_type": "code", 1279 | "execution_count": 30, 1280 | "metadata": { 1281 | "vscode": { 1282 | "languageId": "shellscript" 1283 | } 1284 | }, 1285 | "outputs": [ 1286 | { 1287 | "name": "stdout", 1288 | "output_type": "stream", 1289 | "text": [ 1290 | "\u001b[1;39m{\n", 1291 | " \u001b[0m\u001b[34;1m\"time\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"2024-07-30T12:12:43-05:00\"\u001b[0m\u001b[1;39m,\n", 1292 | " \u001b[0m\u001b[34;1m\"id\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"\"\u001b[0m\u001b[1;39m,\n", 1293 | " \u001b[0m\u001b[34;1m\"remote_ip\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"::1\"\u001b[0m\u001b[1;39m,\n", 1294 | " \u001b[0m\u001b[34;1m\"host\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"localhost:9090\"\u001b[0m\u001b[1;39m,\n", 1295 | " \u001b[0m\u001b[34;1m\"method\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"GET\"\u001b[0m\u001b[1;39m,\n", 1296 | " \u001b[0m\u001b[34;1m\"uri\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"/fast-api/1\"\u001b[0m\u001b[1;39m,\n", 1297 | " \u001b[0m\u001b[34;1m\"user_agent\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Go-http-client/1.1\"\u001b[0m\u001b[1;39m,\n", 1298 | " \u001b[0m\u001b[34;1m\"status\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m200\u001b[0m\u001b[1;39m,\n", 1299 | " \u001b[0m\u001b[34;1m\"headers\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", 1300 | " \u001b[0m\u001b[34;1m\"Accept-Encoding\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"gzip\"\u001b[0m\u001b[1;39m,\n", 1301 | " \u001b[0m\u001b[34;1m\"Connection\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"keep-alive\"\u001b[0m\u001b[1;39m,\n", 1302 | " \u001b[0m\u001b[34;1m\"User-Agent\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Go-http-client/1.1\"\u001b[0m\u001b[1;39m\n", 1303 | " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", 1304 | " \u001b[0m\u001b[34;1m\"request_body\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"\"\u001b[0m\u001b[1;39m\n", 1305 | "\u001b[1;39m}\u001b[0m\n" 1306 | ] 1307 | } 1308 | ], 1309 | "source": [ 1310 | "# show the JSON envelope response for one of the requests\n", 1311 | "cat scratch/ten/http-localhost-9090-fast-api-1 | jq '.'" 1312 | ] 1313 | }, 1314 | { 1315 | "cell_type": "markdown", 1316 | "metadata": {}, 1317 | "source": [ 1318 | "That's great for a low number of files, but filesystems get cranky when you get more than low thousands of files in a single directory.\n", 1319 | "\n", 1320 | "`ganda` supports a `--subdir-length /-S ` flag that will hash the url and put the response in a subdirectory for that hash.\n", 1321 | "\n", 1322 | "So with a `--subdir-length 2` the `http://localhost:9090/fast-api/1` response gets hashed to the `d8` subdirectory under our `scratch/ten-subdir-length-two` output directory." 1323 | ] 1324 | }, 1325 | { 1326 | "cell_type": "code", 1327 | "execution_count": 31, 1328 | "metadata": { 1329 | "vscode": { 1330 | "languageId": "shellscript" 1331 | } 1332 | }, 1333 | "outputs": [ 1334 | { 1335 | "name": "stdout", 1336 | "output_type": "stream", 1337 | "text": [ 1338 | "Response: 200 http://localhost:9090/fast-api/3 -> scratch/ten-subdir-length-two/55/http-localhost-9090-fast-api-3\n", 1339 | "Response: 200 http://localhost:9090/fast-api/9 -> scratch/ten-subdir-length-two/91/http-localhost-9090-fast-api-9\n", 1340 | "Response: 200 http://localhost:9090/fast-api/2 -> scratch/ten-subdir-length-two/1d/http-localhost-9090-fast-api-2\n", 1341 | "Response: 200 http://localhost:9090/fast-api/4 -> scratch/ten-subdir-length-two/47/http-localhost-9090-fast-api-4\n", 1342 | "Response: 200 http://localhost:9090/fast-api/10 -> scratch/ten-subdir-length-two/eb/http-localhost-9090-fast-api-10\n", 1343 | "Response: 200 http://localhost:9090/fast-api/1 -> scratch/ten-subdir-length-two/d8/http-localhost-9090-fast-api-1\n", 1344 | "Response: 200 http://localhost:9090/fast-api/6 -> scratch/ten-subdir-length-two/3f/http-localhost-9090-fast-api-6\n", 1345 | "Response: 200 http://localhost:9090/fast-api/8 -> scratch/ten-subdir-length-two/d1/http-localhost-9090-fast-api-8\n", 1346 | "Response: 200 http://localhost:9090/fast-api/7 -> scratch/ten-subdir-length-two/80/http-localhost-9090-fast-api-7\n", 1347 | "Response: 200 http://localhost:9090/fast-api/5 -> scratch/ten-subdir-length-two/44/http-localhost-9090-fast-api-5\n", 1348 | "0.00 0:00:00 [0.00 /s] [0.00 /s]\n" 1349 | ] 1350 | } 1351 | ], 1352 | "source": [ 1353 | "seq 10 |\\\n", 1354 | " awk '{printf \"http://localhost:9090/fast-api/%s\\n\", $1}' |\\\n", 1355 | " ganda -W 10 --output-directory scratch/ten-subdir-length-two --subdir-length 2 |\\\n", 1356 | " pv -albert > /dev/null" 1357 | ] 1358 | }, 1359 | { 1360 | "cell_type": "markdown", 1361 | "metadata": {}, 1362 | "source": [ 1363 | "Let's make 100k requests and see how many of them get hashed to the `d8` subdirectory that the `/fast-api/1` request hashed to above." 1364 | ] 1365 | }, 1366 | { 1367 | "cell_type": "code", 1368 | "execution_count": 32, 1369 | "metadata": { 1370 | "vscode": { 1371 | "languageId": "shellscript" 1372 | } 1373 | }, 1374 | "outputs": [ 1375 | { 1376 | "name": "stdout", 1377 | "output_type": "stream", 1378 | "text": [ 1379 | " 100k 0:00:13 [7.57k/s] [7.57k/s]\n" 1380 | ] 1381 | } 1382 | ], 1383 | "source": [ 1384 | "seq 100000 |\\\n", 1385 | " awk '{printf \"http://localhost:9090/fast-api/%s\\n\", $1}' |\\\n", 1386 | " ganda -W 3 --output-directory scratch/10k-subdir-length-two --subdir-length 2 2>&1 |\\\n", 1387 | " pv -albert > /dev/null" 1388 | ] 1389 | }, 1390 | { 1391 | "cell_type": "markdown", 1392 | "metadata": { 1393 | "vscode": { 1394 | "languageId": "shellscript" 1395 | } 1396 | }, 1397 | "source": [ 1398 | "So it took less than 15 seconds on my machine to make 100k requests to the echoserver and save each response to its own file." 1399 | ] 1400 | }, 1401 | { 1402 | "cell_type": "code", 1403 | "execution_count": 33, 1404 | "metadata": { 1405 | "vscode": { 1406 | "languageId": "shellscript" 1407 | } 1408 | }, 1409 | "outputs": [ 1410 | { 1411 | "name": "stdout", 1412 | "output_type": "stream", 1413 | "text": [ 1414 | " 256\n" 1415 | ] 1416 | } 1417 | ], 1418 | "source": [ 1419 | "# how many subdirectories were created\n", 1420 | "ls scratch/10k-subdir-length-two | wc -l " 1421 | ] 1422 | }, 1423 | { 1424 | "cell_type": "code", 1425 | "execution_count": 34, 1426 | "metadata": { 1427 | "vscode": { 1428 | "languageId": "shellscript" 1429 | } 1430 | }, 1431 | "outputs": [ 1432 | { 1433 | "name": "stdout", 1434 | "output_type": "stream", 1435 | "text": [ 1436 | "00\n", 1437 | "01\n", 1438 | "02\n", 1439 | "03\n", 1440 | "04\n" 1441 | ] 1442 | } 1443 | ], 1444 | "source": [ 1445 | "# show the first 5 subdirectories - expect 00, 01, 02, 03, 04\n", 1446 | "ls scratch/10k-subdir-length-two | head -n 5 " 1447 | ] 1448 | }, 1449 | { 1450 | "cell_type": "code", 1451 | "execution_count": 35, 1452 | "metadata": { 1453 | "vscode": { 1454 | "languageId": "shellscript" 1455 | } 1456 | }, 1457 | "outputs": [ 1458 | { 1459 | "name": "stdout", 1460 | "output_type": "stream", 1461 | "text": [ 1462 | " 406\n" 1463 | ] 1464 | } 1465 | ], 1466 | "source": [ 1467 | "# how many files are in the d8 subdirectory\n", 1468 | "ls scratch/10k-subdir-length-two/d8 | wc -l" 1469 | ] 1470 | }, 1471 | { 1472 | "cell_type": "code", 1473 | "execution_count": 36, 1474 | "metadata": { 1475 | "vscode": { 1476 | "languageId": "shellscript" 1477 | } 1478 | }, 1479 | "outputs": [ 1480 | { 1481 | "name": "stdout", 1482 | "output_type": "stream", 1483 | "text": [ 1484 | "http-localhost-9090-fast-api-1\n", 1485 | "http-localhost-9090-fast-api-1000\n", 1486 | "http-localhost-9090-fast-api-10057\n", 1487 | "http-localhost-9090-fast-api-10125\n", 1488 | "http-localhost-9090-fast-api-10135\n" 1489 | ] 1490 | } 1491 | ], 1492 | "source": [ 1493 | "# show the first 5 files in the d8 subdirectory\n", 1494 | "ls scratch/10k-subdir-length-two/d8 | head -n 5" 1495 | ] 1496 | }, 1497 | { 1498 | "cell_type": "markdown", 1499 | "metadata": { 1500 | "vscode": { 1501 | "languageId": "shellscript" 1502 | } 1503 | }, 1504 | "source": [ 1505 | "There are 256 subdirectories, so every hash value was hit, and our sample subdirectory had 406 files in it. A nice distribution." 1506 | ] 1507 | }, 1508 | { 1509 | "cell_type": "code", 1510 | "execution_count": 37, 1511 | "metadata": { 1512 | "vscode": { 1513 | "languageId": "shellscript" 1514 | } 1515 | }, 1516 | "outputs": [ 1517 | { 1518 | "name": "stdout", 1519 | "output_type": "stream", 1520 | "text": [ 1521 | "echoserver stopped\n" 1522 | ] 1523 | } 1524 | ], 1525 | "source": [ 1526 | "# clean up the scratch directory\n", 1527 | "rm -rf scratch\n", 1528 | "\n", 1529 | "# clean up the background echoserver\n", 1530 | "pkill ganda && echo \"echoserver stopped\" || echo \"echoserver not stopped\"" 1531 | ] 1532 | }, 1533 | { 1534 | "cell_type": "markdown", 1535 | "metadata": { 1536 | "vscode": { 1537 | "languageId": "shellscript" 1538 | } 1539 | }, 1540 | "source": [ 1541 | "That's it! A whirlwind tour of what `ganda` can do." 1542 | ] 1543 | } 1544 | ], 1545 | "metadata": { 1546 | "kernelspec": { 1547 | "display_name": "Bash", 1548 | "language": "bash", 1549 | "name": "bash" 1550 | }, 1551 | "language_info": { 1552 | "codemirror_mode": "shell", 1553 | "file_extension": ".sh", 1554 | "mimetype": "text/x-sh", 1555 | "name": "bash" 1556 | } 1557 | }, 1558 | "nbformat": 4, 1559 | "nbformat_minor": 2 1560 | } 1561 | --------------------------------------------------------------------------------