├── .go-version ├── contrib └── handlers │ ├── .gitignore │ ├── testdata │ ├── .gitignore │ ├── amex_statement.csv │ ├── amex_statement.xls │ ├── smbc_statement.csv │ ├── smbc_statement2.csv │ ├── smbc_card_statement.csv │ ├── sony_bank_statement.csv │ ├── smbc_card_statement2.csv │ ├── rakuten_bank_statement.csv │ ├── sbi_sumishin_net_bank_statement.csv │ ├── sbi_securities_global_banking_statement.csv │ ├── sbi_securities_global_execution_history.csv │ └── rakuten_card_statement.csv │ ├── exporting_test.go │ ├── doc.go │ ├── sony_bank_test.go │ ├── sbi_sumishin_net_bank_test.go │ ├── rakuten_bank_test.go │ ├── rakuten_card_test.go │ ├── amex_test.go │ ├── amex_csv_test.go │ ├── rakuten_bank.go │ ├── README.md │ ├── sony_bank.go │ ├── sbi_sumishin_net_bank.go │ ├── smbc.go │ ├── sbi_securities_test.go │ ├── handlers.go │ ├── smbc_card_test.go │ ├── amex_csv.go │ ├── rakuten_card.go │ ├── smbc_card.go │ ├── sbi_securities.go │ ├── smbc_test.go │ ├── amex.go │ └── handlers_test.go ├── examples ├── quickstart │ ├── .gitignore │ ├── source.csv │ ├── policy.json.tpl │ ├── go.mod │ ├── README.md │ ├── source.utf8.csv │ ├── .envrc.example │ ├── .gcloudignore │ ├── schema.json │ ├── bqload.go │ ├── deploy.sh │ └── go.sum ├── preprocessor │ ├── .gitignore │ ├── policy.json.tpl │ ├── source_202009.csv │ ├── go.mod │ ├── README.md │ ├── .envrc.example │ ├── schema.json │ ├── .gcloudignore │ ├── bqload.go │ ├── deploy.sh │ └── go.sum └── pre_configured_handlers │ ├── .gitignore │ ├── 202012.csv │ ├── sbi_bank_statement.csv │ ├── go.mod │ ├── sbi_securities_global_execution_history.csv │ ├── policy.json.tpl │ ├── run.sh │ ├── .envrc.example │ ├── README.md │ ├── .gcloudignore │ ├── sbi_bank.json │ ├── smbc_card.json │ ├── bqload.go │ ├── sbi_sec.json │ └── deploy.sh ├── diagram.png ├── .gitignore ├── parser.go ├── event.go ├── context.go ├── .github ├── workflows │ ├── main.yaml │ └── pull_request.yaml └── renovate.json5 ├── extractor.go ├── LICENSE ├── option.go ├── loader.go ├── go.mod ├── notifier_test.go ├── bqloader_test.go ├── notifier.go ├── doc.go ├── handler_test.go ├── README.md ├── bqloader.go └── handler.go /.go-version: -------------------------------------------------------------------------------- 1 | 1.13.5 2 | -------------------------------------------------------------------------------- /contrib/handlers/.gitignore: -------------------------------------------------------------------------------- 1 | tmp/ 2 | -------------------------------------------------------------------------------- /contrib/handlers/testdata/.gitignore: -------------------------------------------------------------------------------- 1 | !*.csv 2 | -------------------------------------------------------------------------------- /examples/quickstart/.gitignore: -------------------------------------------------------------------------------- 1 | .envrc 2 | policy.json 3 | -------------------------------------------------------------------------------- /examples/preprocessor/.gitignore: -------------------------------------------------------------------------------- 1 | .envrc 2 | policy.json 3 | -------------------------------------------------------------------------------- /examples/pre_configured_handlers/.gitignore: -------------------------------------------------------------------------------- 1 | .envrc 2 | policy.json 3 | -------------------------------------------------------------------------------- /diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/diagram.png -------------------------------------------------------------------------------- /contrib/handlers/exporting_test.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | var ParseSMBCDate = parseSMBCDate 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .envrc 2 | 3 | /.gcloudignore 4 | deploy.envrc 5 | /deploy.sh 6 | func.go 7 | /*.csv 8 | 9 | -------------------------------------------------------------------------------- /examples/quickstart/source.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/examples/quickstart/source.csv -------------------------------------------------------------------------------- /contrib/handlers/testdata/amex_statement.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/contrib/handlers/testdata/amex_statement.csv -------------------------------------------------------------------------------- /contrib/handlers/testdata/amex_statement.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/contrib/handlers/testdata/amex_statement.xls -------------------------------------------------------------------------------- /contrib/handlers/testdata/smbc_statement.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/contrib/handlers/testdata/smbc_statement.csv -------------------------------------------------------------------------------- /examples/pre_configured_handlers/202012.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/examples/pre_configured_handlers/202012.csv -------------------------------------------------------------------------------- /contrib/handlers/testdata/smbc_statement2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/contrib/handlers/testdata/smbc_statement2.csv -------------------------------------------------------------------------------- /contrib/handlers/testdata/smbc_card_statement.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/contrib/handlers/testdata/smbc_card_statement.csv -------------------------------------------------------------------------------- /contrib/handlers/testdata/sony_bank_statement.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/contrib/handlers/testdata/sony_bank_statement.csv -------------------------------------------------------------------------------- /contrib/handlers/testdata/smbc_card_statement2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/contrib/handlers/testdata/smbc_card_statement2.csv -------------------------------------------------------------------------------- /contrib/handlers/testdata/rakuten_bank_statement.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/contrib/handlers/testdata/rakuten_bank_statement.csv -------------------------------------------------------------------------------- /examples/pre_configured_handlers/sbi_bank_statement.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/examples/pre_configured_handlers/sbi_bank_statement.csv -------------------------------------------------------------------------------- /contrib/handlers/testdata/sbi_sumishin_net_bank_statement.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/contrib/handlers/testdata/sbi_sumishin_net_bank_statement.csv -------------------------------------------------------------------------------- /examples/pre_configured_handlers/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/nownabe/go-bqloader/examples/pre_configured_handlers 2 | 3 | go 1.13 4 | 5 | require go.nownabe.dev/bqloader v1.3.0 6 | -------------------------------------------------------------------------------- /contrib/handlers/testdata/sbi_securities_global_banking_statement.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/contrib/handlers/testdata/sbi_securities_global_banking_statement.csv -------------------------------------------------------------------------------- /contrib/handlers/testdata/sbi_securities_global_execution_history.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/contrib/handlers/testdata/sbi_securities_global_execution_history.csv -------------------------------------------------------------------------------- /examples/preprocessor/policy.json.tpl: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "members": ["serviceAccount:$SERVICE_ACCOUNT"], 5 | "role": "roles/bigquery.dataEditor" 6 | } 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /examples/quickstart/policy.json.tpl: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "members": ["serviceAccount:$SERVICE_ACCOUNT"], 5 | "role": "roles/bigquery.dataEditor" 6 | } 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /examples/pre_configured_handlers/sbi_securities_global_execution_history.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nownabe/go-bqloader/HEAD/examples/pre_configured_handlers/sbi_securities_global_execution_history.csv -------------------------------------------------------------------------------- /examples/pre_configured_handlers/policy.json.tpl: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "members": ["serviceAccount:$SERVICE_ACCOUNT"], 5 | "role": "roles/bigquery.dataEditor" 6 | } 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /contrib/handlers/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Package handlers includes pre-configured handlers for bqloader. 4 | 5 | See an example. https://github.com/nownabe/go-bqloader/blob/main/examples/pre_configured_handlers/bqload.go 6 | 7 | */ 8 | package handlers 9 | -------------------------------------------------------------------------------- /examples/preprocessor/source_202009.csv: -------------------------------------------------------------------------------- 1 | Date,Description,Credits 2 | 2020/08/31,Don Shop,6061 3 | 2020/08/30,Taxi,5022 4 | ,Rate　　　1.000Yen, 5 | 2020/08/22,Supermarket,3164 6 | 2020/08/19,Amazon,1020 7 | 2020/08/12,Supermarket,780 8 | 2020/08/10,Amazon,500 9 | -------------------------------------------------------------------------------- /examples/quickstart/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/nownabe/go-bqloader/examples/quickstart 2 | 3 | go 1.13 4 | 5 | require ( 6 | go.nownabe.dev/bqloader v1.2.0 7 | golang.org/x/text v0.3.4 8 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 9 | ) 10 | -------------------------------------------------------------------------------- /examples/preprocessor/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/nownabe/go-bqloader/examples/quickstart 2 | 3 | go 1.13 4 | 5 | require ( 6 | go.nownabe.dev/bqloader v1.2.0 7 | golang.org/x/text v0.3.4 8 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 9 | ) 10 | -------------------------------------------------------------------------------- /examples/pre_configured_handlers/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | if [[ ! -e .envrc ]]; then 8 | echo ".envrc doesn't exist" >&2 9 | exit 1 10 | fi 11 | 12 | # shellcheck disable=SC1091 13 | source .envrc 14 | 15 | gsutil cp ./*.csv "gs://$SOURCE_BUCKET/csv/" 16 | -------------------------------------------------------------------------------- /examples/quickstart/README.md: -------------------------------------------------------------------------------- 1 | Quickstart 2 | =========== 3 | 4 | Create GCP project for this quickstart tutorial. 5 | 6 | Create your .envrc file: 7 | 8 | ```bash 9 | cp .envrc.example .envrc 10 | vi .envrc 11 | ``` 12 | 13 | (direnv is not necessary.) 14 | 15 | Deploy and run Cloud Function: 16 | 17 | ```bash 18 | ./deploy.sh 19 | ``` 20 | 21 | Check BigQuery. 22 | -------------------------------------------------------------------------------- /examples/quickstart/source.utf8.csv: -------------------------------------------------------------------------------- 1 | "Date","Description","Withdrawls","Deposits","Balance","Memo" 2 | "2020/09/27","クレジットカード","1,091",,"73,744","-" 3 | "2020/09/26","定期預金","10,000",,"74,835","-" 4 | "2020/09/25","給与 BAR",,"4,021","84,835","-" 5 | "2020/09/09","口座振替 bar","2,594",,"80,814","-" 6 | "2020/09/07","振込 foo","3,266",,"83,408","-" 7 | "2020/09/04","振込 FOO",,"600","86,674","-" 8 | -------------------------------------------------------------------------------- /contrib/handlers/testdata/rakuten_card_statement.csv: -------------------------------------------------------------------------------- 1 | "利用日","利用店名・商品名","利用者","支払方法","利用金額","支払手数料","支払総額","12月支払金額","1月繰越残高","新規サイン" 2 | "2020/12/03","foo","本人","1回払い","2750","0","2750","2750","0","*" 3 | "2020/11/20","bar","家族","1回払い","9968","0","9968","9968","0","*" 4 | "","現地利用額　　　　　　９９６８．０００変換レート　　　１．０００円","","","","","","","","" 5 | "2020/11/20","baz","本人","1回払い","1570","0","1570","1570","0","*" 6 | -------------------------------------------------------------------------------- /examples/preprocessor/README.md: -------------------------------------------------------------------------------- 1 | Preprocessor Example 2 | ==================== 3 | 4 | Create GCP project for this example. 5 | You can use the project for quickstart. 6 | 7 | Create your .envrc file: 8 | 9 | ```bash 10 | cp .envrc.example .envrc 11 | vi .envrc 12 | ``` 13 | 14 | (direnv is not necessary.) 15 | 16 | Deploy and run Cloud Function: 17 | 18 | ```bash 19 | ./deploy.sh 20 | ``` 21 | 22 | Check BigQuery. 23 | -------------------------------------------------------------------------------- /examples/preprocessor/.envrc.example: -------------------------------------------------------------------------------- 1 | export GCP_PROJECT="your-project" 2 | 3 | export SERVICE_ACCOUNT_NAME="bqloader" 4 | export SOURCE_BUCKET="$GCP_PROJECT-bqloader" 5 | export FUNCTION_NAME="bqloader" 6 | 7 | export BIGQUERY_PROJECT_ID="$GCP_PROJECT" 8 | export BIGQUERY_DATASET_ID="bqloader" 9 | export BIGQUERY_TABLE_ID="quickstart" 10 | 11 | export SLACK_TOKEN="your slack bot token" 12 | export SLACK_CHANNEL="#your-channel" 13 | -------------------------------------------------------------------------------- /examples/quickstart/.envrc.example: -------------------------------------------------------------------------------- 1 | export GCP_PROJECT="your-project" 2 | 3 | export SERVICE_ACCOUNT_NAME="bqloader" 4 | export SOURCE_BUCKET="$GCP_PROJECT-bqloader" 5 | export FUNCTION_NAME="bqloader" 6 | 7 | export BIGQUERY_PROJECT_ID="$GCP_PROJECT" 8 | export BIGQUERY_DATASET_ID="bqloader" 9 | export BIGQUERY_TABLE_ID="quickstart" 10 | 11 | export SLACK_TOKEN="your slack bot token" 12 | export SLACK_CHANNEL="#your-channel" 13 | -------------------------------------------------------------------------------- /examples/pre_configured_handlers/.envrc.example: -------------------------------------------------------------------------------- 1 | export GCP_PROJECT="your-project" 2 | export SLACK_TOKEN="your slack bot token" 3 | export SLACK_CHANNEL="#your-channel" 4 | 5 | 6 | # You don't have to change following variables. 7 | 8 | export SERVICE_ACCOUNT_NAME="bqloader" 9 | export SOURCE_BUCKET="$GCP_PROJECT-bqloader" 10 | export FUNCTION_NAME="bqloader" 11 | 12 | export BIGQUERY_PROJECT_ID="$GCP_PROJECT" 13 | export BIGQUERY_DATASET_ID="bqloader" 14 | -------------------------------------------------------------------------------- /examples/pre_configured_handlers/README.md: -------------------------------------------------------------------------------- 1 | Pre-Configured Handlers 2 | ======================= 3 | 4 | This is an example to use [pre-configured handlers](https://github.com/nownabe/go-bqloader/tree/main/contrib/handlers). 5 | 6 | Create your .envrc file: 7 | 8 | ```bash 9 | cp .envrc.example .envrc 10 | vi .envrc 11 | ``` 12 | 13 | (direnv is not necessary.) 14 | 15 | Deploy to Cloud Function: 16 | 17 | ```bash 18 | ./deploy.sh 19 | ``` 20 | 21 | Upload CSVs and run the function. 22 | 23 | ```bash 24 | ./run.sh 25 | ``` 26 | 27 | Check BigQuery. 28 | -------------------------------------------------------------------------------- /parser.go: -------------------------------------------------------------------------------- 1 | package bqloader 2 | 3 | import ( 4 | "context" 5 | "encoding/csv" 6 | "io" 7 | 8 | "golang.org/x/xerrors" 9 | ) 10 | 11 | // Parser parses files from storage. 12 | type Parser func(context.Context, io.Reader) ([][]string, error) 13 | 14 | // CSVParser provides a parser to parse CSV files. 15 | func CSVParser() Parser { 16 | return func(_ context.Context, r io.Reader) ([][]string, error) { 17 | records, err := csv.NewReader(r).ReadAll() 18 | if err != nil { 19 | return nil, xerrors.Errorf("failed to parse as CSV: %w", err) 20 | } 21 | return records, nil 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /examples/preprocessor/schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "date", 4 | "type": "DATE", 5 | "mode": "REQUIRED", 6 | "description": "transaction date" 7 | }, 8 | { 9 | "name": "description", 10 | "type": "STRING", 11 | "mode": "NULLABLE", 12 | "description": "what this transaction is" 13 | }, 14 | { 15 | "name": "credits", 16 | "type": "NUMERIC", 17 | "mode": "NULLABLE", 18 | "description": "credit amount (yen)" 19 | }, 20 | { 21 | "name": "payment_month", 22 | "type": "DATE", 23 | "mode": "REQUIRED", 24 | "description": "payment month" 25 | } 26 | ] 27 | -------------------------------------------------------------------------------- /examples/quickstart/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | #!include:.gitignore 18 | -------------------------------------------------------------------------------- /examples/preprocessor/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | #!include:.gitignore 18 | 19 | *.sh 20 | *.json 21 | policy.json.tpl 22 | *.md 23 | *.csv 24 | -------------------------------------------------------------------------------- /examples/pre_configured_handlers/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | node_modules 17 | #!include:.gitignore 18 | 19 | *.json 20 | policy.json.tpl 21 | README.md 22 | *.sh 23 | .envrc 24 | .envrc.example 25 | -------------------------------------------------------------------------------- /event.go: -------------------------------------------------------------------------------- 1 | package bqloader 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "time" 7 | 8 | "github.com/rs/zerolog" 9 | ) 10 | 11 | // Event is an event from Cloud Storage. 12 | type Event struct { 13 | Name string `json:"name"` 14 | Bucket string `json:"bucket"` 15 | TimeCreated time.Time `json:"timeCreated"` 16 | 17 | // for test 18 | source io.Reader 19 | } 20 | 21 | // FullPath returns full path of storage object beginning with gs://. 22 | func (e *Event) FullPath() string { 23 | return fmt.Sprintf("gs://%s/%s", e.Bucket, e.Name) 24 | } 25 | 26 | func (e *Event) logger(l *zerolog.Logger) *zerolog.Logger { 27 | d := zerolog.Dict(). 28 | Str("name", e.Name). 29 | Str("bucket", e.Bucket). 30 | Time("timeCreated", e.TimeCreated) 31 | 32 | logger := l.With().Dict("event", d).Logger() 33 | return &logger 34 | } 35 | -------------------------------------------------------------------------------- /contrib/handlers/sony_bank_test.go: -------------------------------------------------------------------------------- 1 | package handlers_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "go.nownabe.dev/bqloader" 8 | "go.nownabe.dev/bqloader/contrib/handlers" 9 | ) 10 | 11 | func Test_SonyBankStatement(t *testing.T) { 12 | t.Parallel() 13 | 14 | const csv = "testdata/sony_bank_statement.csv" 15 | 16 | expected := [][]string{ 17 | {"2020-12-12", "積み立て定期預金へ振替", "", "", "10000", "661450"}, 18 | {"2020-12-15", "振込 ソニー　タロウ", "", "220000", "", "881450"}, 19 | } 20 | 21 | h, tl := buildTestHandler(t, csv, handlers.SonyBankStatement) 22 | 23 | name := "path_to/sony_bank_statement.csv" 24 | e := bqloader.Event{Name: name, Bucket: "bucket"} 25 | 26 | if err := h.Handle(context.Background(), e); err != nil { 27 | t.Errorf("Unexpected error: %v", err) 28 | } 29 | 30 | assertEqual(t, expected, tl.result) 31 | } 32 | -------------------------------------------------------------------------------- /examples/pre_configured_handlers/sbi_bank.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "date", 4 | "type": "DATE", 5 | "mode": "REQUIRED", 6 | "description": "日付" 7 | }, 8 | { 9 | "name": "description", 10 | "type": "STRING", 11 | "mode": "REQUIRED", 12 | "description": "内容" 13 | }, 14 | { 15 | "name": "withdrawal_amount", 16 | "type": "NUMERIC", 17 | "mode": "NULLABLE", 18 | "description": "出金金額 (円)" 19 | }, 20 | { 21 | "name": "deposit_amount", 22 | "type": "NUMERIC", 23 | "mode": "NULLABLE", 24 | "description": "入金金額 (円)" 25 | }, 26 | { 27 | "name": "balance", 28 | "type": "NUMERIC", 29 | "mode": "REQUIRED", 30 | "description": "残高 (円)" 31 | }, 32 | { 33 | "name": "memo", 34 | "type": "STRING", 35 | "mode": "NULLABLE", 36 | "description": "メモ" 37 | } 38 | ] 39 | -------------------------------------------------------------------------------- /context.go: -------------------------------------------------------------------------------- 1 | package bqloader 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | type contextKey string 9 | 10 | const ( 11 | startedTimeKey contextKey = "startedTime" 12 | handlerStartedTimeKey contextKey = "handlerStartedTime" 13 | ) 14 | 15 | func withStartedTime(ctx context.Context) context.Context { 16 | return context.WithValue(ctx, startedTimeKey, time.Now()) 17 | } 18 | 19 | func startedTimeFrom(ctx context.Context) (time.Time, bool) { 20 | t, ok := ctx.Value(startedTimeKey).(time.Time) 21 | 22 | return t, ok 23 | } 24 | 25 | func withHandlerStartedTime(ctx context.Context) context.Context { 26 | return context.WithValue(ctx, handlerStartedTimeKey, time.Now()) 27 | } 28 | 29 | func handlerStartedTimeFrom(ctx context.Context) (time.Time, bool) { 30 | t, ok := ctx.Value(handlerStartedTimeKey).(time.Time) 31 | 32 | return t, ok 33 | } 34 | -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | name: Main Branch Workflow 2 | on: 3 | push: 4 | branches: 5 | - main 6 | 7 | jobs: 8 | test: 9 | name: test job 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | - uses: actions/setup-go@v3 14 | with: 15 | go-version: 1.19 16 | - uses: actions/cache@v3 17 | name: Cache Go Modules 18 | id: cache 19 | with: 20 | path: ~/go/pkg/mod 21 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 22 | restore-keys: | 23 | ${{ runner.os }}-go- 24 | - name: Download Go Modules 25 | if: steps.cache.outputs.cache-hit != 'true' 26 | run: go mod download 27 | - name: Run tests 28 | run: go test -v -race -coverpkg=./... -coverprofile=coverage.txt ./... 29 | - uses: codecov/codecov-action@v3 30 | -------------------------------------------------------------------------------- /contrib/handlers/sbi_sumishin_net_bank_test.go: -------------------------------------------------------------------------------- 1 | package handlers_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "go.nownabe.dev/bqloader" 8 | "go.nownabe.dev/bqloader/contrib/handlers" 9 | ) 10 | 11 | func Test_SBISumishinNetBankStatement(t *testing.T) { 12 | t.Parallel() 13 | 14 | const csv = "testdata/sbi_sumishin_net_bank_statement.csv" 15 | 16 | expected := [][]string{ 17 | {"2020-12-25", "普通　円　フィンビーエゴマ", "220", "", "29447", "-"}, 18 | {"2020-12-25", "振込＊キュウヨ．フリコム．ジヤパン（ド", "", "29667", "29667", "-"}, 19 | } 20 | 21 | h, tl := buildTestHandler(t, csv, handlers.SBISumishinNetBankStatement) 22 | 23 | name := "path_to/sbi_sumishin_net_bank_statement.csv" 24 | e := bqloader.Event{Name: name, Bucket: "bucket"} 25 | 26 | if err := h.Handle(context.Background(), e); err != nil { 27 | t.Errorf("Unexpected error: %v", err) 28 | } 29 | 30 | assertEqual(t, expected, tl.result) 31 | } 32 | -------------------------------------------------------------------------------- /examples/quickstart/schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "date", 4 | "type": "DATE", 5 | "mode": "REQUIRED", 6 | "description": "transaction date" 7 | }, 8 | { 9 | "name": "description", 10 | "type": "STRING", 11 | "mode": "NULLABLE", 12 | "description": "what this transaction is" 13 | }, 14 | { 15 | "name": "withdrawal_amount", 16 | "type": "NUMERIC", 17 | "mode": "NULLABLE", 18 | "description": "withdrawal amount (yen)" 19 | }, 20 | { 21 | "name": "deposit_amount", 22 | "type": "NUMERIC", 23 | "mode": "NULLABLE", 24 | "description": "deposit amount (yen)" 25 | }, 26 | { 27 | "name": "balance", 28 | "type": "NUMERIC", 29 | "mode": "REQUIRED", 30 | "description": "balance (yen)" 31 | }, 32 | { 33 | "name": "memo", 34 | "type": "STRING", 35 | "mode": "NULLABLE", 36 | "description": "memo" 37 | } 38 | ] 39 | -------------------------------------------------------------------------------- /contrib/handlers/rakuten_bank_test.go: -------------------------------------------------------------------------------- 1 | package handlers_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "go.nownabe.dev/bqloader" 8 | "go.nownabe.dev/bqloader/contrib/handlers" 9 | ) 10 | 11 | func Test_RakutenBankStatement(t *testing.T) { 12 | t.Parallel() 13 | 14 | const csv = "testdata/rakuten_bank_statement.csv" 15 | 16 | expected := [][]string{ 17 | {"2020-02-25", "-754", "184655", "ラクテンショウケンカブシキガイシャ （投資信託買付代金）"}, 18 | {"2020-02-28", "8363", "193018", "ラクテンショウケンカブシキガイシャ （自動スイ－プ）"}, 19 | {"2020-03-03", "12033", "205051", "ラクテンショウケンカブシキガイシャ （自動スイ－プ）"}, 20 | } 21 | 22 | h, tl := buildTestHandler(t, csv, handlers.RakutenBankStatement) 23 | 24 | name := "path_to/rakuten_bank_statement.csv" 25 | e := bqloader.Event{Name: name, Bucket: "bucket"} 26 | 27 | if err := h.Handle(context.Background(), e); err != nil { 28 | t.Errorf("Unexpected error: %v", err) 29 | } 30 | 31 | assertEqual(t, expected, tl.result) 32 | } 33 | -------------------------------------------------------------------------------- /contrib/handlers/rakuten_card_test.go: -------------------------------------------------------------------------------- 1 | package handlers_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "go.nownabe.dev/bqloader" 8 | "go.nownabe.dev/bqloader/contrib/handlers" 9 | ) 10 | 11 | func Test_RakutenCardStatement(t *testing.T) { 12 | t.Parallel() 13 | 14 | const csv = "testdata/rakuten_card_statement.csv" 15 | 16 | expected := [][]string{ 17 | {"2020-12-03", "foo", "本人", "1回払い", "2750", "0", "2750", "2750", "0", "*", "2020-12-01"}, 18 | {"2020-11-20", "bar", "家族", "1回払い", "9968", "0", "9968", "9968", "0", "*", "2020-12-01"}, 19 | {"2020-11-20", "baz", "本人", "1回払い", "1570", "0", "1570", "1570", "0", "*", "2020-12-01"}, 20 | } 21 | 22 | h, tl := buildTestHandler(t, csv, handlers.RakutenCardStatement) 23 | 24 | name := "path_to/enavi202012(1234).csv" 25 | e := bqloader.Event{Name: name, Bucket: "bucket"} 26 | 27 | if err := h.Handle(context.Background(), e); err != nil { 28 | t.Errorf("Unexpected error: %v", err) 29 | } 30 | 31 | assertEqual(t, expected, tl.result) 32 | } 33 | -------------------------------------------------------------------------------- /contrib/handlers/amex_test.go: -------------------------------------------------------------------------------- 1 | package handlers_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "go.nownabe.dev/bqloader" 8 | "go.nownabe.dev/bqloader/contrib/handlers" 9 | ) 10 | 11 | func Test_AMEXStatement(t *testing.T) { 12 | t.Parallel() 13 | 14 | const xls = "testdata/amex_statement.xls" 15 | 16 | expected := [][]string{ 17 | {"2022-06-19", "2022-06-20", "GOOGLE *DOMAINS", "TARO AMEX", "1760", "", "", "", "2022-07-01"}, 18 | {"2022-05-29", "2022-05-29", "IWANTMYNAME IWANTMYNAME", "TARO AMEX", "129", "1.00 USD", "129", "", "2022-07-01"}, 19 | {"2022-05-29", "2022-05-29", "IWANTMYNAME IWANTMYNAME", "", "-129", "1.00 USD", "129", "", "2022-07-01"}, 20 | } 21 | 22 | h, tl := buildTestHandler(t, xls, handlers.AMEXStatement) 23 | 24 | name := "path_to/2022-07.xls" 25 | e := bqloader.Event{Name: name, Bucket: "bucket"} 26 | 27 | if err := h.Handle(context.Background(), e); err != nil { 28 | t.Fatalf("Unexpected error: %v", err) 29 | } 30 | 31 | assertEqual(t, expected, tl.result) 32 | } 33 | -------------------------------------------------------------------------------- /extractor.go: -------------------------------------------------------------------------------- 1 | package bqloader 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "cloud.google.com/go/storage" 8 | "golang.org/x/xerrors" 9 | ) 10 | 11 | // Extractor extracts data from source such as Cloud Storage. 12 | type Extractor interface { 13 | Extract(context.Context, Event) (io.Reader, func(), error) 14 | } 15 | 16 | type defaultExtractor struct { 17 | storage *storage.Client 18 | } 19 | 20 | func newDefaultExtractor(ctx context.Context, project string) (Extractor, error) { 21 | s, err := storage.NewClient(ctx) 22 | if err != nil { 23 | return nil, xerrors.Errorf("failed to build storage client for %s: %w", project, err) 24 | } 25 | 26 | return &defaultExtractor{storage: s}, nil 27 | } 28 | 29 | func (e *defaultExtractor) Extract(ctx context.Context, ev Event) (io.Reader, func(), error) { 30 | obj := e.storage.Bucket(ev.Bucket).Object(ev.Name) 31 | r, err := obj.NewReader(ctx) 32 | if err != nil { 33 | return nil, nil, xerrors.Errorf("failed to get reader of %s: %w", ev.FullPath(), err) 34 | } 35 | 36 | return r, func() { r.Close() }, nil 37 | } 38 | -------------------------------------------------------------------------------- /contrib/handlers/amex_csv_test.go: -------------------------------------------------------------------------------- 1 | package handlers_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "go.nownabe.dev/bqloader" 8 | "go.nownabe.dev/bqloader/contrib/handlers" 9 | ) 10 | 11 | func Test_AMEXStatementCSV(t *testing.T) { 12 | t.Parallel() 13 | 14 | const csv = "testdata/amex_statement.csv" 15 | 16 | expected := [][]string{ 17 | {"2023-07-10", "2023-07-10", "前回分口座振替金額", "TARO AMEX", "-4048", "", "", "", "2023-08-01"}, 18 | {"2023-07-08", "2023-07-09", "UBER EATS", "TARO AMEX", "1408", "", "", "", "2023-08-01"}, 19 | {"2023-07-03", "2023-07-04", "GITHUB, INC.", "TARO AMEX", "-1431", "9.68 USD", "147.831", "", "2023-08-01"}, 20 | {"2023-07-02", "2023-07-03", "GITHUB, INC.", "TARO AMEX", "1479", "10.00 USD", "147.9", "", "2023-08-01"}, 21 | } 22 | 23 | h, tl := buildTestHandler(t, csv, handlers.AMEXStatementCSV) 24 | 25 | name := "path_to/2023-08.csv" 26 | e := bqloader.Event{Name: name, Bucket: "bucket"} 27 | 28 | if err := h.Handle(context.Background(), e); err != nil { 29 | t.Fatalf("Unexpected error: %v", err) 30 | } 31 | 32 | assertEqual(t, expected, tl.result) 33 | } 34 | -------------------------------------------------------------------------------- /.github/renovate.json5: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | extends: [ 4 | "config:base", 5 | ":prHourlyLimitNone", 6 | ], 7 | dependencyDashboard: true, 8 | labels: ["renovate"], 9 | packageRules: [ 10 | { 11 | matchUpdateTypes: ["pin"], 12 | automerge: true, 13 | }, 14 | { 15 | matchFiles: [ 16 | "go.mod", 17 | "examples/pre_configured_handlers/go.mod", 18 | "examples/preprocessor/go.mod", 19 | "examples/quickstart/go.mod", 20 | ], 21 | matchUpdateTypes: ["minor"], 22 | automerge: true, 23 | stabilityDays: 7, 24 | }, 25 | { 26 | matchFiles: [ 27 | "go.mod", 28 | "examples/pre_configured_handlers/go.mod", 29 | "examples/preprocessor/go.mod", 30 | "examples/quickstart/go.mod", 31 | ], 32 | matchUpdateTypes: ["patch"], 33 | automerge: true, 34 | stabilityDays: 3, 35 | }, 36 | { 37 | matchFiles: ["go.mod"], 38 | matchUpdateTypes: ["major"], 39 | assignees: ["nownabe"], 40 | } 41 | ], 42 | } 43 | -------------------------------------------------------------------------------- /examples/pre_configured_handlers/smbc_card.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "date", 4 | "type": "DATE", 5 | "mode": "REQUIRED", 6 | "description": "ご利用日" 7 | }, 8 | { 9 | "name": "description", 10 | "type": "STRING", 11 | "mode": "REQUIRED", 12 | "description": "ご利用店名" 13 | }, 14 | { 15 | "name": "usage_amount", 16 | "type": "NUMERIC", 17 | "mode": "NULLABLE", 18 | "description": "ご利用金額" 19 | }, 20 | { 21 | "name": "payment_section", 22 | "type": "STRING", 23 | "mode": "NULLABLE", 24 | "description": "支払区分" 25 | }, 26 | { 27 | "name": "this_time", 28 | "type": "STRING", 29 | "mode": "NULLABLE", 30 | "description": "今回回数" 31 | }, 32 | { 33 | "name": "payment_amount", 34 | "type": "NUMERIC", 35 | "mode": "NULLABLE", 36 | "description": "お支払金額" 37 | }, 38 | { 39 | "name": "other", 40 | "type": "STRING", 41 | "mode": "NULLABLE", 42 | "description": "その他" 43 | }, 44 | { 45 | "name": "payment_month", 46 | "type": "DATE", 47 | "mode": "REQUIRED", 48 | "description": "支払い月" 49 | } 50 | ] 51 | -------------------------------------------------------------------------------- /contrib/handlers/rakuten_bank.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | "time" 7 | 8 | "go.nownabe.dev/bqloader" 9 | "golang.org/x/text/encoding/japanese" 10 | "golang.org/x/xerrors" 11 | ) 12 | 13 | // RakutenBankStatement build a handler for statements for Rakuten Bank (楽天銀行 入出金明細). 14 | func RakutenBankStatement(name, pattern string, table Table, notifier bqloader.Notifier) *bqloader.Handler { 15 | projector := func(ctx context.Context, r []string) ([]string, error) { 16 | // 0: date (取引日) 17 | t, err := time.Parse("20060102", r[0]) 18 | if err != nil { 19 | return nil, xerrors.Errorf("failed to parse date: %w", err) 20 | } 21 | r[0] = t.Format("2006-01-02") 22 | 23 | return r, nil 24 | } 25 | 26 | return &bqloader.Handler{ 27 | Name: name, 28 | Pattern: regexp.MustCompile(pattern), 29 | SkipLeadingRows: 1, 30 | 31 | Encoding: japanese.ShiftJIS, 32 | Parser: bqloader.CSVParser(), 33 | Projector: projector, 34 | Notifier: notifier, 35 | 36 | Project: table.Project, 37 | Dataset: table.Dataset, 38 | Table: table.Table, 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 nownabe 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /contrib/handlers/README.md: -------------------------------------------------------------------------------- 1 | Pre-Configured Handlers 2 | ======================= 3 | 4 | [![PkgGoDev](https://pkg.go.dev/badge/go.nownabe.dev/bqloader/contrib/handlers)](https://pkg.go.dev/go.nownabe.dev/bqloader/contrib/handlers) 5 | 6 | ## Usage 7 | 8 | See an [example](https://github.com/nownabe/go-bqloader/blob/main/examples/pre_configured_handlers/bqload.go). 9 | 10 | ## List of Handlers 11 | 12 | ### Bank 13 | 14 | * `handlers.RakutenBankStatement`: Statements of Rakuten Bank (楽天銀行 入出金明細) 15 | * `handlers.SBISumishinNetBankStatement`: Statements of Sumishin SBI Net Bank (住信SBIネット銀行 入出金明細) 16 | * `handlers.SMBCStatement`: Statements of SMBC (三井住友銀行 入出金明細) 17 | * `handlers.SonyBankStatement`: Statements of Sony Bank (ソニー銀行 入出金明細) 18 | 19 | ### Credit Card 20 | 21 | * `handlers.RakutenCardStatement`: Statements of Rakuten Card (楽天カード) 22 | * `handlers.SMBCCardStatement`: Statements of SMBC Card (三井住友カード) 23 | 24 | ### Securities 25 | 26 | * `handlers.SBISecuritiesGlobalBankingStatement`: Receive and pay of SBI Securities Global (SBI証券 外国株 入出金明細) 27 | * `handlers.SBISecuritiesGlobalExecutionHistory`: Execution hisotry of SBI Securities Global (SBI証券 外国株 約定履歴) 28 | -------------------------------------------------------------------------------- /option.go: -------------------------------------------------------------------------------- 1 | package bqloader 2 | 3 | import "github.com/rs/zerolog" 4 | 5 | // Option configures BQLoader. 6 | type Option interface { 7 | apply(*bqloader) error 8 | } 9 | 10 | type optionFunc func(*bqloader) error 11 | 12 | func (f optionFunc) apply(l *bqloader) error { 13 | return f(l) 14 | } 15 | 16 | // WithPrettyLogging configures BQLoader to print human friendly logs. 17 | func WithPrettyLogging() Option { 18 | return optionFunc(func(l *bqloader) error { 19 | l.prettyLogging = true 20 | 21 | return nil 22 | }) 23 | } 24 | 25 | // WithLogLevel configures log level to print logs. 26 | // Allowed values are trace, debug, info, warn, error, fatal or panic. 27 | func WithLogLevel(l string) Option { 28 | return optionFunc(func(bq *bqloader) error { 29 | l, err := zerolog.ParseLevel(l) 30 | if err != nil { 31 | return err 32 | } 33 | bq.logLevel = l 34 | 35 | return nil 36 | }) 37 | } 38 | 39 | // WithConcurrency configures the concurrency of projectors. 40 | // Before setting this, confirm GOMAXPROCS. 41 | func WithConcurrency(n int) Option { 42 | return optionFunc(func(bq *bqloader) error { 43 | bq.concurrency = n 44 | 45 | return nil 46 | }) 47 | } 48 | -------------------------------------------------------------------------------- /contrib/handlers/sony_bank.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | "time" 7 | 8 | "go.nownabe.dev/bqloader" 9 | "golang.org/x/text/encoding/japanese" 10 | "golang.org/x/xerrors" 11 | ) 12 | 13 | // SonyBankStatement build a handler for statements of Sony Bank (ソニー銀行). 14 | func SonyBankStatement(name, pattern string, table Table, notifier bqloader.Notifier) *bqloader.Handler { 15 | projector := func(_ context.Context, r []string) ([]string, error) { 16 | t, err := time.Parse("2006年01月02日", r[0]) 17 | if err != nil { 18 | return nil, xerrors.Errorf("failed to parse date: %w", err) 19 | } 20 | 21 | r[0] = t.Format("2006-01-02") 22 | 23 | // Remove commas 24 | r[3] = CleanNumber(r[3]) 25 | r[4] = CleanNumber(r[4]) 26 | r[5] = CleanNumber(r[5]) 27 | 28 | return r, nil 29 | } 30 | 31 | return &bqloader.Handler{ 32 | Name: name, 33 | Pattern: regexp.MustCompile(pattern), 34 | SkipLeadingRows: 1, 35 | 36 | Encoding: japanese.ShiftJIS, 37 | Parser: bqloader.CSVParser(), 38 | Projector: projector, 39 | Notifier: notifier, 40 | 41 | Project: table.Project, 42 | Dataset: table.Dataset, 43 | Table: table.Table, 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /examples/pre_configured_handlers/bqload.go: -------------------------------------------------------------------------------- 1 | package bqload 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "runtime" 7 | 8 | "go.nownabe.dev/bqloader" 9 | "go.nownabe.dev/bqloader/contrib/handlers" 10 | ) 11 | 12 | var loader bqloader.BQLoader 13 | 14 | func init() { 15 | c := runtime.NumCPU() 16 | runtime.GOMAXPROCS(c) 17 | 18 | var err error 19 | loader, err = bqloader.New(bqloader.WithLogLevel("debug"), bqloader.WithConcurrency(c)) 20 | if err != nil { 21 | panic(err) 22 | } 23 | 24 | t := handlers.TableGenerator(os.Getenv("BIGQUERY_PROJECT_ID"), os.Getenv("BIGQUERY_DATASET_ID")) 25 | n := &bqloader.SlackNotifier{ 26 | Token: os.Getenv("SLACK_TOKEN"), 27 | Channel: os.Getenv("SLACK_CHANNEL"), 28 | } 29 | 30 | handlers.MustAddHandlers(context.Background(), loader, 31 | handlers.SBISumishinNetBankStatement("SBI Bank", `^csv/sbi_bank`, t("sbi_bank"), n), 32 | handlers.SBISecuritiesGlobalExecutionHistory("SBI Sec", `^csv/sbi_securities`, t("sbi_sec"), n), 33 | handlers.SMBCCardStatement("SMBC Card", `^csv/\d+\.csv$`, t("smbc_card"), n), 34 | ) 35 | } 36 | 37 | // BQLoad is the entrypoint for Cloud Functions. 38 | func BQLoad(ctx context.Context, e bqloader.Event) error { 39 | return loader.Handle(ctx, e) 40 | } 41 | -------------------------------------------------------------------------------- /contrib/handlers/sbi_sumishin_net_bank.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | "time" 7 | 8 | "go.nownabe.dev/bqloader" 9 | "golang.org/x/text/encoding/japanese" 10 | "golang.org/x/xerrors" 11 | ) 12 | 13 | // SBISumishinNetBankStatement build a handler for statements of SBI bank (住信SBIネット銀行). 14 | func SBISumishinNetBankStatement(name, pattern string, table Table, notifier bqloader.Notifier) *bqloader.Handler { 15 | projector := func(_ context.Context, r []string) ([]string, error) { 16 | t, err := time.Parse("2006/01/02", r[0]) 17 | if err != nil { 18 | return nil, xerrors.Errorf("failed to parse date: %w", err) 19 | } 20 | 21 | r[0] = t.Format("2006-01-02") 22 | 23 | // Remove commas 24 | r[2] = CleanNumber(r[2]) 25 | r[3] = CleanNumber(r[3]) 26 | r[4] = CleanNumber(r[4]) 27 | 28 | return r, nil 29 | } 30 | 31 | return &bqloader.Handler{ 32 | Name: name, 33 | Pattern: regexp.MustCompile(pattern), 34 | SkipLeadingRows: 1, 35 | 36 | Encoding: japanese.ShiftJIS, 37 | Parser: bqloader.CSVParser(), 38 | Projector: projector, 39 | Notifier: notifier, 40 | 41 | Project: table.Project, 42 | Dataset: table.Dataset, 43 | Table: table.Table, 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /examples/pre_configured_handlers/sbi_sec.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "contract_date", 4 | "type": "DATE", 5 | "mode": "REQUIRED", 6 | "description": "国内約定日" 7 | }, 8 | { 9 | "name": "name", 10 | "type": "STRING", 11 | "mode": "REQUIRED", 12 | "description": "銘柄名" 13 | }, 14 | { 15 | "name": "code", 16 | "type": "STRING", 17 | "mode": "REQUIRED", 18 | "description": "銘柄コード" 19 | }, 20 | { 21 | "name": "market", 22 | "type": "STRING", 23 | "mode": "REQUIRED", 24 | "description": "市場" 25 | }, 26 | { 27 | "name": "stock_type", 28 | "type": "STRING", 29 | "mode": "REQUIRED", 30 | "description": "商品区分" 31 | }, 32 | { 33 | "name": "order_type", 34 | "type": "STRING", 35 | "mode": "REQUIRED", 36 | "description": "注文種別" 37 | }, 38 | { 39 | "name": "deal_type", 40 | "type": "STRING", 41 | "mode": "REQUIRED", 42 | "description": "取引" 43 | }, 44 | { 45 | "name": "account_type", 46 | "type": "STRING", 47 | "mode": "REQUIRED", 48 | "description": "預り区分" 49 | }, 50 | { 51 | "name": "quantity", 52 | "type": "NUMERIC", 53 | "mode": "REQUIRED", 54 | "description": "約定数量" 55 | }, 56 | { 57 | "name": "unitPrice", 58 | "type": "NUMERIC", 59 | "mode": "REQUIRED", 60 | "description": "約定単価 (USD)" 61 | }, 62 | { 63 | "name": "delivery_date", 64 | "type": "DATE", 65 | "mode": "REQUIRED", 66 | "description": "国内受渡日" 67 | }, 68 | { 69 | "name": "deliveryAmount", 70 | "type": "NUMERIC", 71 | "mode": "REQUIRED", 72 | "description": "受渡金額 (JPY)" 73 | } 74 | ] 75 | -------------------------------------------------------------------------------- /loader.go: -------------------------------------------------------------------------------- 1 | package bqloader 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/csv" 7 | 8 | "cloud.google.com/go/bigquery" 9 | "golang.org/x/xerrors" 10 | ) 11 | 12 | // Loader loads projected data into a destination such as BigQuery. 13 | type Loader interface { 14 | Load(context.Context, [][]string) error 15 | } 16 | 17 | type defaultLoader struct { 18 | table *bigquery.Table 19 | } 20 | 21 | func newDefaultLoader(ctx context.Context, project, dataset, table string) (Loader, error) { 22 | bq, err := bigquery.NewClient(ctx, project) 23 | if err != nil { 24 | return nil, xerrors.Errorf("failed to build bigquery client for %s.%s.%s: %w", 25 | project, dataset, table, err) 26 | } 27 | 28 | t := bq.Dataset(dataset).Table(table) 29 | 30 | return &defaultLoader{table: t}, nil 31 | } 32 | 33 | func (l *defaultLoader) Load(ctx context.Context, records [][]string) error { 34 | buf := &bytes.Buffer{} 35 | if err := csv.NewWriter(buf).WriteAll(records); err != nil { 36 | return xerrors.Errorf("failed to write csv into buffer: %w", err) 37 | } 38 | 39 | rs := bigquery.NewReaderSource(buf) 40 | rs.AllowQuotedNewlines = true 41 | 42 | loader := l.table.LoaderFrom(rs) 43 | loader.LoadConfig.CreateDisposition = bigquery.CreateNever 44 | 45 | job, err := loader.Run(ctx) 46 | if err != nil { 47 | return xerrors.Errorf("failed to run bigquery load job: %w", err) 48 | } 49 | 50 | status, err := job.Wait(ctx) 51 | if err != nil { 52 | return xerrors.Errorf("failed to wait bigquery job: %w", err) 53 | } 54 | 55 | if status.Err() != nil { 56 | return xerrors.Errorf("bigquery load job failed: %w", status.Err()) 57 | } 58 | 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module go.nownabe.dev/bqloader 2 | 3 | go 1.19 4 | 5 | require ( 6 | cloud.google.com/go/bigquery v1.32.0 7 | cloud.google.com/go/functions v0.2.0 8 | cloud.google.com/go/storage v1.23.0 9 | github.com/extrame/xls v0.0.1 10 | github.com/rs/zerolog v1.27.0 11 | gitlab.com/osaki-lab/iowrapper v0.0.0-20201210013351-bab12bc19f54 12 | golang.org/x/sync v0.11.0 13 | golang.org/x/text v0.22.0 14 | golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 15 | ) 16 | 17 | require ( 18 | cloud.google.com/go v0.102.1 // indirect 19 | cloud.google.com/go/compute v1.7.0 // indirect 20 | cloud.google.com/go/iam v0.3.0 // indirect 21 | github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 // indirect 22 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect 23 | github.com/golang/protobuf v1.5.2 // indirect 24 | github.com/google/go-cmp v0.5.8 // indirect 25 | github.com/google/uuid v1.3.0 // indirect 26 | github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect 27 | github.com/googleapis/gax-go/v2 v2.4.0 // indirect 28 | github.com/googleapis/go-type-adapters v1.0.0 // indirect 29 | github.com/mattn/go-colorable v0.1.12 // indirect 30 | github.com/mattn/go-isatty v0.0.14 // indirect 31 | go.opencensus.io v0.23.0 // indirect 32 | golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 // indirect 33 | golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb // indirect 34 | golang.org/x/sys v0.5.0 // indirect 35 | google.golang.org/api v0.85.0 // indirect 36 | google.golang.org/appengine v1.6.7 // indirect 37 | google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad // indirect 38 | google.golang.org/grpc v1.47.0 // indirect 39 | google.golang.org/protobuf v1.28.0 // indirect 40 | ) 41 | -------------------------------------------------------------------------------- /contrib/handlers/smbc.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "regexp" 7 | "strconv" 8 | "time" 9 | 10 | "go.nownabe.dev/bqloader" 11 | "golang.org/x/text/encoding/japanese" 12 | "golang.org/x/xerrors" 13 | ) 14 | 15 | func parseSMBCDate(date string) (time.Time, error) { 16 | t, err := time.Parse("2006/1/2", date) 17 | if err == nil { 18 | return t, nil 19 | } 20 | 21 | if len(date) != 9 { 22 | return time.Time{}, xerrors.Errorf("invalid date format: %s", date) 23 | } 24 | 25 | var rekiBase int 26 | switch date[0] { 27 | case 'H': 28 | rekiBase = 1988 29 | case 'R': 30 | rekiBase = 2018 31 | default: 32 | return time.Time{}, xerrors.Errorf("%s is not supported", date) 33 | } 34 | 35 | wareki, err := strconv.Atoi(date[1:3]) 36 | if err != nil { 37 | return time.Time{}, xerrors.Errorf("failed to parse wareki as int: %w", err) 38 | } 39 | 40 | return time.Parse("2006.01.02", fmt.Sprintf("%d%s", wareki+rekiBase, date[3:9])) 41 | } 42 | 43 | // SMBCStatement builds a handler for statements for SMBC (三井住友銀行 入出金明細). 44 | func SMBCStatement(name, pattern string, t Table, n bqloader.Notifier) *bqloader.Handler { 45 | projector := func(ctx context.Context, r []string) ([]string, error) { 46 | // 0: date (年月日) 47 | t, err := parseSMBCDate(r[0]) 48 | if err != nil { 49 | return nil, xerrors.Errorf("failed to parse date: %w", err) 50 | } 51 | r[0] = t.Format("2006-01-02") 52 | 53 | return r, nil 54 | } 55 | 56 | return &bqloader.Handler{ 57 | Name: name, 58 | Pattern: regexp.MustCompile(pattern), 59 | SkipLeadingRows: 1, 60 | 61 | Encoding: japanese.ShiftJIS, 62 | Parser: bqloader.CSVParser(), 63 | Projector: projector, 64 | Notifier: n, 65 | 66 | Project: t.Project, 67 | Dataset: t.Dataset, 68 | Table: t.Table, 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /contrib/handlers/sbi_securities_test.go: -------------------------------------------------------------------------------- 1 | package handlers_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "go.nownabe.dev/bqloader" 8 | "go.nownabe.dev/bqloader/contrib/handlers" 9 | ) 10 | 11 | func Test_SBISecuritiesGlobalBankingStatement(t *testing.T) { 12 | t.Parallel() 13 | 14 | const csv = "testdata/sbi_securities_global_banking_statement.csv" 15 | 16 | expected := [][]string{ 17 | {"2022-12-30", "分配金", "米ドル", "BND 銘柄名:VG TBM", "", "0.64"}, 18 | {"2022-12-28", "出金", "米ドル", "米国株式TECL外国源泉税の過日徴収", "3.58", ""}, 19 | {"2022-11-18", "入金", "米ドル", "米国BND211229配当税還付 税減額日:220311", "", "0.07"}, 20 | {"2022-10-03", "入金", "米ドル", "住信SBIネット銀行から外貨入金", "", "100.00"}, 21 | } 22 | 23 | h, tl := buildTestHandler(t, csv, handlers.SBISecuritiesGlobalBankingStatement) 24 | 25 | name := "path_to/sbi_securities_global_banking_statement.csv" 26 | e := bqloader.Event{Name: name, Bucket: "bucket"} 27 | 28 | if err := h.Handle(context.Background(), e); err != nil { 29 | t.Errorf("Unexpected error: %v", err) 30 | } 31 | 32 | assertEqual(t, expected, tl.result) 33 | } 34 | 35 | func Test_SBISecuritiesGlobalExecutionHistory(t *testing.T) { 36 | t.Parallel() 37 | 38 | const csv = "testdata/sbi_securities_global_execution_history.csv" 39 | 40 | expected := [][]string{ 41 | {"2022-12-02", "グローバルＸ SP 500 カバードコール50 ETF", "XYLG", "NYSE Arca", "米国株式", "成行", "現買", "特定", "30", "26.4100", "2022-12-06", "796.22"}, 42 | {"2022-01-11", "iシェアーズ コア　米国高配当株 ETF", "HDV", "NYSE Arca", "米国株式", "成行", "現売", "特定", "29", "102.4450", "2022-01-13", "2956.21"}, 43 | } 44 | 45 | h, tl := buildTestHandler(t, csv, handlers.SBISecuritiesGlobalExecutionHistory) 46 | 47 | name := "path_to/sbi_securities_global_execution_history.csv" 48 | e := bqloader.Event{Name: name, Bucket: "bucket"} 49 | 50 | if err := h.Handle(context.Background(), e); err != nil { 51 | t.Errorf("Unexpected error: %v", err) 52 | } 53 | 54 | assertEqual(t, expected, tl.result) 55 | } 56 | -------------------------------------------------------------------------------- /contrib/handlers/handlers.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/csv" 7 | "io" 8 | "io/ioutil" 9 | "strings" 10 | 11 | "go.nownabe.dev/bqloader" 12 | "golang.org/x/xerrors" 13 | ) 14 | 15 | type contextKey string 16 | 17 | // Table identifies BigQuery table. 18 | type Table struct { 19 | Project string 20 | Dataset string 21 | Table string 22 | } 23 | 24 | // TableGenerator returns Table generator func. 25 | func TableGenerator(project, dataset string) func(string) Table { 26 | return func(table string) Table { 27 | return Table{ 28 | Project: project, 29 | Dataset: dataset, 30 | Table: table, 31 | } 32 | } 33 | } 34 | 35 | // CleanNumber cleans numbers includes commas and currency marks. 36 | func CleanNumber(n string) string { 37 | cn := "" 38 | for i, c := range n { 39 | if '0' <= c && c <= '9' || c == '.' { 40 | cn += string(c) 41 | } else if i == 0 && c == '-' { 42 | cn += string(c) 43 | } 44 | } 45 | 46 | if cn == "-" { 47 | return "" 48 | } 49 | 50 | return cn 51 | } 52 | 53 | // MustAddHandlers adds handlers into BQLoader. 54 | func MustAddHandlers(ctx context.Context, loader bqloader.BQLoader, handlers ...*bqloader.Handler) { 55 | for _, handler := range handlers { 56 | loader.MustAddHandler(ctx, handler) 57 | } 58 | } 59 | 60 | // PartialCSVParser builds a parser for CSV with invalid head and tail lines. 61 | func PartialCSVParser(skipHeadRows uint, skipTailRows uint, sep string) bqloader.Parser { 62 | return func(_ context.Context, r io.Reader) ([][]string, error) { 63 | body, err := ioutil.ReadAll(r) 64 | if err != nil { 65 | return nil, xerrors.Errorf("failed to read: %w", err) 66 | } 67 | 68 | lines := strings.Split(string(body), sep) 69 | csvBody := strings.Join(lines[skipHeadRows:uint(len(lines))-skipTailRows], sep) 70 | records, err := csv.NewReader(bytes.NewReader([]byte(csvBody))).ReadAll() 71 | if err != nil { 72 | return nil, xerrors.Errorf("failed to read as CSV: %w", err) 73 | } 74 | 75 | return records, nil 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /.github/workflows/pull_request.yaml: -------------------------------------------------------------------------------- 1 | name: Pull Request Workflow 2 | on: 3 | pull_request: 4 | 5 | jobs: 6 | test: 7 | name: test job 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v3 11 | - uses: actions/setup-go@v3 12 | with: 13 | go-version: 1.19 14 | - uses: actions/cache@v3 15 | name: Cache Go Modules 16 | id: cache 17 | with: 18 | path: ~/go/pkg/mod 19 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 20 | restore-keys: | 21 | ${{ runner.os }}-go- 22 | - name: Download Go Modules 23 | if: steps.cache.outputs.cache-hit != 'true' 24 | run: go mod download 25 | - name: Run tests 26 | run: go test -v -race -coverpkg=./... -coverprofile=coverage.txt ./... 27 | - uses: codecov/codecov-action@v3 28 | 29 | golangci-lint: 30 | name: golangci-lint job 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: actions/checkout@v3 34 | - uses: reviewdog/action-golangci-lint@v2 35 | with: 36 | golangci_lint_flags: "--enable-all --exclude-use-default=false" 37 | level: warning 38 | reporter: github-pr-review 39 | fail_on_error: true 40 | 41 | languagetool: 42 | name: languagetool job 43 | runs-on: ubuntu-latest 44 | steps: 45 | - uses: actions/checkout@v3 46 | - uses: reviewdog/action-languagetool@v1 47 | with: 48 | reporter: github-pr-review 49 | level: info 50 | 51 | shellcheck: 52 | name: shellcheck job 53 | runs-on: ubuntu-latest 54 | steps: 55 | - uses: actions/checkout@v3 56 | - uses: reviewdog/action-shellcheck@v1 57 | with: 58 | reporter: github-pr-review 59 | level: warning 60 | fail_on_error: true 61 | 62 | alex: 63 | name: alex job 64 | runs-on: ubuntu-latest 65 | steps: 66 | - uses: actions/checkout@v3 67 | - uses: reviewdog/action-alex@v1 68 | with: 69 | reporter: github-pr-review 70 | level: info 71 | -------------------------------------------------------------------------------- /contrib/handlers/smbc_card_test.go: -------------------------------------------------------------------------------- 1 | package handlers_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "go.nownabe.dev/bqloader" 8 | "go.nownabe.dev/bqloader/contrib/handlers" 9 | ) 10 | 11 | func Test_SMBCCardStatement(t *testing.T) { 12 | t.Parallel() 13 | 14 | const csv = "testdata/smbc_card_statement.csv" 15 | 16 | expected := [][]string{ 17 | {"2020-11-29", "Ａｍａｚｏｎ　Ｄｏｗｎｌｏａｄｓ", "288", "", "", "", "", "2020-12-01"}, 18 | {"2020-11-14", "UBER *EATS (HELP.UBER.COM)", "1650", "", "", "", "1650.00　JPY　1.0000　11 16", "2020-12-01"}, 19 | {"2020-11-30", "ゴールドカード年会費", "", "", "", "5500", "（うち消費税等５００円）", "2020-12-01"}, 20 | } 21 | 22 | h, tl := buildTestHandler(t, csv, handlers.SMBCCardStatement) 23 | 24 | name := "path_to/202012.csv" 25 | e := bqloader.Event{Name: name, Bucket: "bucket"} 26 | 27 | if err := h.Handle(context.Background(), e); err != nil { 28 | t.Errorf("Unexpected error: %v", err) 29 | } 30 | 31 | assertEqual(t, expected, tl.result) 32 | } 33 | 34 | func Test_SMBCCardStatement2(t *testing.T) { 35 | t.Parallel() 36 | 37 | const csv = "testdata/smbc_card_statement2.csv" 38 | 39 | expected := [][]string{ 40 | {"2022-10-31", "インターネットイニシアティブ", "5989", "", "", "", "", "2022-12-01"}, 41 | {"2022-11-01", "ＡＭＡＺＯＮ．ＣＯ．ＪＰ", "17673", "", "", "", "", "2022-12-01"}, 42 | {"2022-11-05", "ＡＭＡＺＯＮ．ＣＯ．ＪＰ", "2490", "", "", "", "", "2022-12-01"}, 43 | {"2022-11-05", "Ａｍａｚｏｎ　Ｄｏｗｎｌｏａｄｓ", "594", "", "", "", "", "2022-12-01"}, 44 | {"2022-11-13", "ＡＭＡＺＯＮ．ＣＯ．ＪＰ", "-500", "", "", "", "返品", "2022-12-01"}, 45 | {"2022-11-13", "ＡＭＡＺＯＮ．ＣＯ．ＪＰ", "-17673", "", "", "", "返品", "2022-12-01"}, 46 | {"2022-11-30", "Ａｍａｚｏｎ　Ｄｏｗｎｌｏａｄｓ", "673", "", "", "", "", "2022-12-01"}, 47 | {"2022-11-29", "ABC-COMPANY (SERVICE )", "3507", "", "", "", "24.90　USD　140.873　11 30", "2022-12-01"}, 48 | } 49 | 50 | h, tl := buildTestHandler(t, csv, handlers.SMBCCardStatement) 51 | 52 | name := "path_to/202212.csv" 53 | e := bqloader.Event{Name: name, Bucket: "bucket"} 54 | 55 | if err := h.Handle(context.Background(), e); err != nil { 56 | t.Fatalf("Unexpected error: %v", err) 57 | } 58 | 59 | assertEqual(t, expected, tl.result) 60 | } 61 | -------------------------------------------------------------------------------- /contrib/handlers/amex_csv.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | "time" 7 | 8 | "go.nownabe.dev/bqloader" 9 | "golang.org/x/text/encoding/japanese" 10 | "golang.org/x/xerrors" 11 | ) 12 | 13 | func AMEXStatementCSV(name, pattern string, table Table, notifier bqloader.Notifier) *bqloader.Handler { 14 | var monthKey contextKey = "month" 15 | 16 | filenameRE := regexp.MustCompile(`/(\d\d\d\d-\d\d)\.csv$`) 17 | 18 | preprocessor := func(ctx context.Context, e bqloader.Event) (context.Context, error) { 19 | match := filenameRE.FindStringSubmatch(e.Name) 20 | if len(match) < 2 { 21 | return ctx, xerrors.Errorf("wrong object path: %s", e.Name) 22 | } 23 | 24 | month, err := time.Parse("2006-01", match[1]) 25 | if err != nil { 26 | return ctx, xerrors.Errorf("failed to parse payment month from object path: %s: %w", match[1], err) 27 | } 28 | 29 | return context.WithValue(ctx, monthKey, month.Format("2006-01-02")), nil 30 | } 31 | 32 | projector := func(ctx context.Context, r []string) ([]string, error) { 33 | paymentMonth, ok := ctx.Value(monthKey).(string) 34 | if !ok { 35 | return nil, xerrors.Errorf("failed to get payment month from context") 36 | } 37 | 38 | // 0: date (ご利用日) 39 | t, err := time.Parse("2006/01/02", r[0]) 40 | if err != nil { 41 | return nil, xerrors.Errorf("failed to parse date: %v", err) 42 | } 43 | r[0] = t.Format("2006-01-02") 44 | 45 | // 1: データ処理日 46 | t, err = time.Parse("2006/01/02", r[1]) 47 | if err != nil { 48 | return nil, xerrors.Errorf("failed to parse date: %v", err) 49 | } 50 | r[1] = t.Format("2006-01-02") 51 | 52 | // 4: 金額 53 | r[4] = CleanNumber(r[5]) 54 | 55 | // 5: 海外通貨利用金額 56 | r[5] = r[6] 57 | 58 | // 6: 換算レート 59 | r[6] = r[7] 60 | 61 | // 7: 追加情報 62 | r[7] = "" 63 | 64 | // 8: payment_month (支払い月) 65 | r = append(r, paymentMonth) 66 | 67 | return r, nil 68 | } 69 | 70 | return &bqloader.Handler{ 71 | Name: name, 72 | Pattern: regexp.MustCompile(pattern), 73 | SkipLeadingRows: 1, 74 | 75 | Encoding: japanese.ShiftJIS, 76 | Parser: bqloader.CSVParser(), 77 | Projector: projector, 78 | Preprocessor: preprocessor, 79 | Notifier: notifier, 80 | 81 | Project: table.Project, 82 | Dataset: table.Dataset, 83 | Table: table.Table, 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /examples/quickstart/bqload.go: -------------------------------------------------------------------------------- 1 | package bqload 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "regexp" 7 | "runtime" 8 | "strings" 9 | "time" 10 | 11 | "golang.org/x/text/encoding/japanese" 12 | "golang.org/x/xerrors" 13 | 14 | "go.nownabe.dev/bqloader" 15 | ) 16 | 17 | var loader bqloader.BQLoader 18 | 19 | func init() { 20 | c := runtime.NumCPU() 21 | runtime.GOMAXPROCS(c) 22 | 23 | var err error 24 | loader, err = bqloader.New(bqloader.WithLogLevel("debug"), bqloader.WithConcurrency(c)) 25 | if err != nil { 26 | panic(err) 27 | } 28 | loader.MustAddHandler(context.Background(), newHandler()) 29 | } 30 | 31 | func newHandler() *bqloader.Handler { 32 | /* 33 | This projector converts date fields formatted as "2006/01/02" 34 | at the first column into strings like "2006-01-02" that satisfies 35 | BigQuery date type, and removes commas in numeric fields. 36 | */ 37 | projector := func(_ context.Context, r []string) ([]string, error) { 38 | t, err := time.Parse("2006/01/02", r[0]) 39 | if err != nil { 40 | return nil, xerrors.Errorf("Column 0 cannot parse as a date: %w", err) 41 | } 42 | 43 | r[0] = t.Format("2006-01-02") 44 | r[2] = strings.ReplaceAll(r[2], ",", "") 45 | r[3] = strings.ReplaceAll(r[3], ",", "") 46 | r[4] = strings.ReplaceAll(r[4], ",", "") 47 | 48 | return r, nil 49 | } 50 | 51 | return &bqloader.Handler{ 52 | Name: "quickstart", // Handler name used in logs and notifications. 53 | Pattern: regexp.MustCompile("^example_bank/"), // This handler processes files matched to this pattern. 54 | Encoding: japanese.ShiftJIS, // Source file encoding. 55 | Parser: bqloader.CSVParser(), // Parser parses source file into records. 56 | Notifier: &bqloader.SlackNotifier{ 57 | Token: os.Getenv("SLACK_TOKEN"), 58 | Channel: os.Getenv("SLACK_CHANNEL"), 59 | }, 60 | Projector: projector, // Projector transforms each row. 61 | SkipLeadingRows: 1, // Skip header row. 62 | 63 | // Destination. 64 | Project: os.Getenv("BIGQUERY_PROJECT_ID"), 65 | Dataset: os.Getenv("BIGQUERY_DATASET_ID"), 66 | Table: os.Getenv("BIGQUERY_TABLE_ID"), 67 | } 68 | } 69 | 70 | // BQLoad is the entrypoint for Cloud Functions. 71 | func BQLoad(ctx context.Context, e bqloader.Event) error { 72 | return loader.Handle(ctx, e) 73 | } 74 | -------------------------------------------------------------------------------- /contrib/handlers/rakuten_card.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "context" 5 | "encoding/csv" 6 | "io" 7 | "regexp" 8 | "time" 9 | 10 | "go.nownabe.dev/bqloader" 11 | "golang.org/x/xerrors" 12 | ) 13 | 14 | // RakutenCardStatement build a handler for statements of Rakuten Card (楽天カード 明細). 15 | // To add column of payment month, keep the file name when you downloaded it. 16 | func RakutenCardStatement(name, pattern string, table Table, notifier bqloader.Notifier) *bqloader.Handler { 17 | var monthKey contextKey = "month" 18 | 19 | re := regexp.MustCompile(`enavi(\d+)\(\d+\)`) 20 | preprocessor := func(ctx context.Context, e bqloader.Event) (context.Context, error) { 21 | match := re.FindStringSubmatch(e.Name) 22 | if len(match) < 2 { 23 | return ctx, xerrors.Errorf("wrong object path: %s", e.Name) 24 | } 25 | 26 | month, err := time.Parse("200601", match[1]) 27 | if err != nil { 28 | return ctx, xerrors.Errorf("failed to parse payment month from object path: %s: %w", match[1], err) 29 | } 30 | 31 | return context.WithValue(ctx, monthKey, month.Format("2006-01-02")), nil 32 | } 33 | 34 | projector := func(ctx context.Context, r []string) ([]string, error) { 35 | if r[0] == "" { 36 | return nil, nil 37 | } 38 | 39 | paymentMonth, ok := ctx.Value(monthKey).(string) 40 | if !ok { 41 | return nil, xerrors.Errorf("failed to get payment month from context: %v", paymentMonth) 42 | } 43 | 44 | t, err := time.Parse("2006/01/02", r[0]) 45 | if err != nil { 46 | return nil, xerrors.Errorf("failed to parse date: %w", err) 47 | } 48 | 49 | r[0] = t.Format("2006-01-02") 50 | r = append(r, paymentMonth) 51 | 52 | return r, nil 53 | } 54 | 55 | parser := bqloader.Parser(func(_ context.Context, r io.Reader) ([][]string, error) { 56 | reader := csv.NewReader(r) 57 | reader.LazyQuotes = true 58 | 59 | records, err := reader.ReadAll() 60 | if err != nil { 61 | return nil, xerrors.Errorf("failed to read content as a CSV: %w", err) 62 | } 63 | 64 | return records, nil 65 | }) 66 | 67 | return &bqloader.Handler{ 68 | Name: name, 69 | Pattern: regexp.MustCompile(pattern), 70 | SkipLeadingRows: 1, 71 | 72 | Parser: parser, 73 | Projector: projector, 74 | Preprocessor: preprocessor, 75 | Notifier: notifier, 76 | 77 | Project: table.Project, 78 | Dataset: table.Dataset, 79 | Table: table.Table, 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /examples/preprocessor/bqload.go: -------------------------------------------------------------------------------- 1 | package bqload 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "regexp" 7 | "runtime" 8 | "time" 9 | 10 | "golang.org/x/xerrors" 11 | 12 | "go.nownabe.dev/bqloader" 13 | ) 14 | 15 | var loader bqloader.BQLoader 16 | 17 | func init() { 18 | c := runtime.NumCPU() 19 | runtime.GOMAXPROCS(c) 20 | 21 | var err error 22 | loader, err = bqloader.New(bqloader.WithLogLevel("debug"), bqloader.WithConcurrency(c)) 23 | if err != nil { 24 | panic(err) 25 | } 26 | loader.MustAddHandler(context.Background(), newHandler()) 27 | } 28 | 29 | func newHandler() *bqloader.Handler { 30 | // Extract payment month from object path in this preprocessor. 31 | type contextKey string 32 | const monthKey contextKey = "month" 33 | 34 | re := regexp.MustCompile(`source_(\d+)\.csv`) 35 | preprocessor := func(ctx context.Context, e bqloader.Event) (context.Context, error) { 36 | match := re.FindStringSubmatch(e.Name) 37 | if len(match) < 2 { 38 | return ctx, xerrors.Errorf("wrong object path: %s", e.Name) 39 | } 40 | 41 | paymentMonth, err := time.Parse("200601", match[1]) 42 | if err != nil { 43 | return ctx, xerrors.Errorf("failed to parse payment month from object path: %s: %w", match[1], err) 44 | } 45 | 46 | return context.WithValue(ctx, monthKey, paymentMonth.Format("2006-01-02")), nil 47 | } 48 | 49 | projector := func(ctx context.Context, r []string) ([]string, error) { 50 | // Skip noisy row. 51 | if r[0] == "" { 52 | return nil, nil 53 | } 54 | 55 | // Get payment month from context. 56 | paymentMonth, ok := ctx.Value(monthKey).(string) 57 | if !ok { 58 | return nil, xerrors.Errorf("failed to get payment month from context: %v", paymentMonth) 59 | } 60 | 61 | t, err := time.Parse("2006/01/02", r[0]) 62 | if err != nil { 63 | return nil, xerrors.Errorf("Column 0 cannot parse as a date: %w", err) 64 | } 65 | 66 | r[0] = t.Format("2006-01-02") 67 | r = append(r, paymentMonth) 68 | 69 | return r, nil 70 | } 71 | 72 | return &bqloader.Handler{ 73 | Name: "preprocessor", 74 | Pattern: regexp.MustCompile("^preprocessor/"), 75 | Parser: bqloader.CSVParser(), 76 | Projector: projector, 77 | Preprocessor: preprocessor, 78 | SkipLeadingRows: 1, 79 | 80 | Project: os.Getenv("BIGQUERY_PROJECT_ID"), 81 | Dataset: os.Getenv("BIGQUERY_DATASET_ID"), 82 | Table: os.Getenv("BIGQUERY_TABLE_ID"), 83 | } 84 | } 85 | 86 | // BQLoad is the entrypoint for Cloud Functions. 87 | func BQLoad(ctx context.Context, e bqloader.Event) error { 88 | return loader.Handle(ctx, e) 89 | } 90 | -------------------------------------------------------------------------------- /examples/quickstart/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | if [[ ! -e .envrc ]]; then 8 | echo ".envrc doesn't exist" >&2 9 | exit 1 10 | fi 11 | 12 | source .envrc 13 | 14 | export SERVICE_ACCOUNT="$SERVICE_ACCOUNT_NAME@$GCP_PROJECT.iam.gserviceaccount.com" 15 | 16 | read -p "Create new service account ($SERVICE_ACCOUNT_NAME)? [y/N]: " yn 17 | if [[ $yn = [yY] ]]; then 18 | gcloud iam service-accounts create \ 19 | $SERVICE_ACCOUNT_NAME \ 20 | --project $GCP_PROJECT \ 21 | --description "Used by Cloud Functions which executes a quickstart example of go.nownabe.dev/bqloader" 22 | 23 | gcloud projects add-iam-policy-binding \ 24 | $GCP_PROJECT \ 25 | --member serviceAccount:$SERVICE_ACCOUNT \ 26 | --role roles/bigquery.jobUser 27 | fi 28 | 29 | read -p "Create new storage bucket ($SOURCE_BUCKET)? [y/N]: " yn 30 | if [[ $yn = [yY] ]]; then 31 | gsutil mb -p $GCP_PROJECT -l US-EAST1 gs://$SOURCE_BUCKET 32 | gsutil iam ch serviceAccount:${SERVICE_ACCOUNT}:roles/storage.objectViewer gs://$SOURCE_BUCKET 33 | fi 34 | 35 | read -p "Create new BigQuery dataset ($BIGQUERY_DATASET_ID) and table ($BIGQUERY_TABLE_ID)? [y/N]: " yn 36 | if [[ $yn = [yY] ]]; then 37 | bq --location us-east1 mk --dataset \ 38 | --description "Used for a quickstart example of go.nownabe.dev/bqloader" \ 39 | $BIGQUERY_PROJECT_ID:$BIGQUERY_DATASET_ID 40 | bq mk --table \ 41 | --description "Used for a quickstart example of go.nownabe.dev/bqloader" \ 42 | --require_partition_filter \ 43 | --time_partitioning_field date \ 44 | --time_partitioning_type MONTH \ 45 | $BIGQUERY_PROJECT_ID:$BIGQUERY_DATASET_ID.$BIGQUERY_TABLE_ID schema.json 46 | 47 | envsubst < policy.json.tpl > policy.json 48 | bq set-iam-policy \ 49 | $BIGQUERY_PROJECT_ID:$BIGQUERY_DATASET_ID.$BIGQUERY_TABLE_ID policy.json 50 | fi 51 | 52 | read -p "Deploy Function ($FUNCTION_NAME)? [y/N]: " yn 53 | if [[ $yn = [yY] ]]; then 54 | gcloud functions deploy $FUNCTION_NAME \ 55 | --project $GCP_PROJECT \ 56 | --region us-east1 \ 57 | --runtime go113 \ 58 | --trigger-resource $SOURCE_BUCKET \ 59 | --trigger-event google.storage.object.finalize \ 60 | --entry-point BQLoad \ 61 | --service-account $SERVICE_ACCOUNT \ 62 | --set-env-vars "BIGQUERY_PROJECT_ID=$BIGQUERY_PROJECT_ID,BIGQUERY_DATASET_ID=$BIGQUERY_DATASET_ID,BIGQUERY_TABLE_ID=$BIGQUERY_TABLE_ID" \ 63 | --set-env-vars "SLACK_TOKEN=$SLACK_TOKEN,SLACK_CHANNEL=$SLACK_CHANNEL" 64 | fi 65 | 66 | gsutil cp source.csv gs://$SOURCE_BUCKET/example_bank/source.csv 67 | -------------------------------------------------------------------------------- /examples/pre_configured_handlers/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | if [[ ! -e .envrc ]]; then 8 | echo ".envrc doesn't exist" >&2 9 | exit 1 10 | fi 11 | 12 | # shellcheck disable=SC1091 13 | source .envrc 14 | 15 | export SERVICE_ACCOUNT="$SERVICE_ACCOUNT_NAME@$GCP_PROJECT.iam.gserviceaccount.com" 16 | 17 | read -rp "Create new service account ($SERVICE_ACCOUNT_NAME)? [y/N]: " yn 18 | if [[ $yn = [yY] ]]; then 19 | gcloud iam service-accounts create \ 20 | "$SERVICE_ACCOUNT_NAME" \ 21 | --project "$GCP_PROJECT" \ 22 | --description "An example of go.nownabe.dev/bqloader" 23 | 24 | gcloud projects add-iam-policy-binding \ 25 | "$GCP_PROJECT" \ 26 | --member "serviceAccount:$SERVICE_ACCOUNT" \ 27 | --role roles/bigquery.jobUser 28 | fi 29 | 30 | read -rp "Create new storage bucket ($SOURCE_BUCKET)? [y/N]: " yn 31 | if [[ $yn = [yY] ]]; then 32 | gsutil mb -p "$GCP_PROJECT" -l US-EAST1 "gs://$SOURCE_BUCKET" 33 | gsutil iam ch "serviceAccount:${SERVICE_ACCOUNT}:roles/storage.objectViewer" "gs://$SOURCE_BUCKET" 34 | fi 35 | 36 | read -rp "Create new BigQuery dataset ($BIGQUERY_DATASET_ID)? [y/N]: " yn 37 | if [[ $yn = [yY] ]]; then 38 | bq --location us-east1 mk --dataset \ 39 | --description "An example of go.nownabe.dev/bqloader" \ 40 | "$BIGQUERY_PROJECT_ID:$BIGQUERY_DATASET_ID" 41 | fi 42 | 43 | tables=("sbi_sec" "sbi_bank" "smbc_card") 44 | for table in "${tables[@]}"; do 45 | read -rp "Create new BigQuery table ($table)? [y/N]: " yn 46 | if [[ $yn = [yY] ]]; then 47 | bq mk --table \ 48 | --description "An example of go.nownabe.dev/bqloader" \ 49 | "$BIGQUERY_PROJECT_ID:$BIGQUERY_DATASET_ID.$table" \ 50 | "$table.json" 51 | 52 | envsubst < policy.json.tpl > policy.json 53 | bq set-iam-policy \ 54 | "$BIGQUERY_PROJECT_ID:$BIGQUERY_DATASET_ID.$table" policy.json 55 | fi 56 | done 57 | 58 | read -rp "Deploy Function ($FUNCTION_NAME)? [y/N]: " yn 59 | if [[ $yn = [yY] ]]; then 60 | gcloud services enable \ 61 | cloudfunctions.googleapis.com \ 62 | cloudbuild.googleapis.com \ 63 | --project "$GCP_PROJECT" 64 | 65 | gcloud functions deploy "$FUNCTION_NAME" \ 66 | --project "$GCP_PROJECT" \ 67 | --region us-east1 \ 68 | --runtime go113 \ 69 | --trigger-resource "$SOURCE_BUCKET" \ 70 | --trigger-event google.storage.object.finalize \ 71 | --entry-point BQLoad \ 72 | --service-account "$SERVICE_ACCOUNT" \ 73 | --set-env-vars "BIGQUERY_PROJECT_ID=$BIGQUERY_PROJECT_ID,BIGQUERY_DATASET_ID=$BIGQUERY_DATASET_ID" \ 74 | --set-env-vars "SLACK_TOKEN=$SLACK_TOKEN,SLACK_CHANNEL=$SLACK_CHANNEL" 75 | fi 76 | -------------------------------------------------------------------------------- /examples/preprocessor/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | if [[ ! -e .envrc ]]; then 8 | echo ".envrc doesn't exist" >&2 9 | exit 1 10 | fi 11 | 12 | # shellcheck disable=SC1091 13 | source .envrc 14 | 15 | export SERVICE_ACCOUNT="$SERVICE_ACCOUNT_NAME@$GCP_PROJECT.iam.gserviceaccount.com" 16 | 17 | read -rp "Create new service account ($SERVICE_ACCOUNT_NAME)? [y/N]: " yn 18 | if [[ $yn = [yY] ]]; then 19 | gcloud iam service-accounts create \ 20 | "$SERVICE_ACCOUNT_NAME" \ 21 | --project "$GCP_PROJECT" \ 22 | --description "Used by Cloud Functions which executes a quickstart example of go.nownabe.dev/bqloader" 23 | 24 | gcloud projects add-iam-policy-binding \ 25 | "$GCP_PROJECT" \ 26 | --member "serviceAccount:$SERVICE_ACCOUNT" \ 27 | --role roles/bigquery.jobUser 28 | fi 29 | 30 | read -rp "Create new storage bucket ($SOURCE_BUCKET)? [y/N]: " yn 31 | if [[ $yn = [yY] ]]; then 32 | gsutil mb -p "$GCP_PROJECT" -l US-EAST1 "gs://$SOURCE_BUCKET" 33 | gsutil iam ch "serviceAccount:${SERVICE_ACCOUNT}:roles/storage.objectViewer" "gs://$SOURCE_BUCKET" 34 | fi 35 | 36 | read -rp "Create new BigQuery dataset ($BIGQUERY_DATASET_ID)? [y/N]: " yn 37 | if [[ $yn = [yY] ]]; then 38 | bq --location us-east1 mk --dataset \ 39 | --description "Used for a quickstart example of go.nownabe.dev/bqloader" \ 40 | "$BIGQUERY_PROJECT_ID:$BIGQUERY_DATASET_ID" 41 | fi 42 | 43 | read -rp "Create new BigQuery table ($BIGQUERY_TABLE_ID)? [y/N]: " yn 44 | if [[ $yn = [yY] ]]; then 45 | bq mk --table \ 46 | --description "Used for a quickstart example of go.nownabe.dev/bqloader" \ 47 | --require_partition_filter \ 48 | --time_partitioning_field date \ 49 | --time_partitioning_type MONTH \ 50 | "$BIGQUERY_PROJECT_ID:$BIGQUERY_DATASET_ID.$BIGQUERY_TABLE_ID" schema.json 51 | 52 | envsubst < policy.json.tpl > policy.json 53 | bq set-iam-policy \ 54 | "$BIGQUERY_PROJECT_ID:$BIGQUERY_DATASET_ID.$BIGQUERY_TABLE_ID" policy.json 55 | fi 56 | 57 | read -rp "Deploy Function ($FUNCTION_NAME)? [y/N]: " yn 58 | if [[ $yn = [yY] ]]; then 59 | gcloud functions deploy "$FUNCTION_NAME" \ 60 | --project "$GCP_PROJECT" \ 61 | --region us-east1 \ 62 | --runtime go113 \ 63 | --trigger-resource "$SOURCE_BUCKET" \ 64 | --trigger-event google.storage.object.finalize \ 65 | --entry-point BQLoad \ 66 | --service-account "$SERVICE_ACCOUNT" \ 67 | --set-env-vars "BIGQUERY_PROJECT_ID=$BIGQUERY_PROJECT_ID,BIGQUERY_DATASET_ID=$BIGQUERY_DATASET_ID,BIGQUERY_TABLE_ID=$BIGQUERY_TABLE_ID" \ 68 | --set-env-vars "SLACK_TOKEN=$SLACK_TOKEN,SLACK_CHANNEL=$SLACK_CHANNEL" 69 | fi 70 | 71 | gsutil cp source_202009.csv "gs://$SOURCE_BUCKET/preprocessor/source_202009.csv" 72 | -------------------------------------------------------------------------------- /contrib/handlers/smbc_card.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/csv" 7 | "io" 8 | "io/ioutil" 9 | "regexp" 10 | "strings" 11 | "time" 12 | 13 | "go.nownabe.dev/bqloader" 14 | "golang.org/x/text/encoding/japanese" 15 | "golang.org/x/xerrors" 16 | ) 17 | 18 | // SMBCCardStatement build a *bqloader.Handler for statements of SMBC card (三井住友VISAカード). 19 | // To add column of payment month, keep the file name when you downloaded it. 20 | func SMBCCardStatement(name, pattern string, table Table, notifier bqloader.Notifier) *bqloader.Handler { 21 | var monthKey contextKey = "month" 22 | 23 | parser := func(_ context.Context, r io.Reader) ([][]string, error) { 24 | body, err := ioutil.ReadAll(r) 25 | if err != nil { 26 | return nil, xerrors.Errorf("failed to read: %w", err) 27 | } 28 | 29 | csvBody := "" 30 | 31 | lines := strings.Split(string(body), "\r\n") 32 | for _, line := range lines { 33 | if len(line) > 4 && line[4] == '/' { 34 | csvBody += line + "\r\n" 35 | } 36 | } 37 | 38 | records, err := csv.NewReader(bytes.NewReader([]byte(csvBody))).ReadAll() 39 | if err != nil { 40 | return nil, xerrors.Errorf("failed to read as CSV: %w", err) 41 | } 42 | 43 | return records, nil 44 | } 45 | 46 | re := regexp.MustCompile(`/(\d+)\.csv`) 47 | preprocessor := func(ctx context.Context, e bqloader.Event) (context.Context, error) { 48 | match := re.FindStringSubmatch(e.Name) 49 | if len(match) < 2 { 50 | return ctx, xerrors.Errorf("wrong object path: %s", e.Name) 51 | } 52 | 53 | month, err := time.Parse("200601", match[1]) 54 | if err != nil { 55 | return ctx, xerrors.Errorf("failed to parse payment month from object path: %s: %w", match[1], err) 56 | } 57 | 58 | return context.WithValue(ctx, monthKey, month.Format("2006-01-02")), nil 59 | } 60 | 61 | projector := func(ctx context.Context, r []string) ([]string, error) { 62 | if r[0] == "" { 63 | return nil, nil 64 | } 65 | 66 | paymentMonth, ok := ctx.Value(monthKey).(string) 67 | if !ok { 68 | return nil, xerrors.Errorf("failed to get payment month from context: %v", paymentMonth) 69 | } 70 | 71 | // 0: date (ご利用日) 72 | t, err := time.Parse("2006/01/02", r[0]) 73 | if err != nil { 74 | return nil, xerrors.Errorf("failed to parse date: %v", err) 75 | } 76 | r[0] = t.Format("2006-01-02") 77 | 78 | // 7: payment_month (支払い月) 79 | r = append(r, paymentMonth) 80 | 81 | return r, nil 82 | } 83 | 84 | return &bqloader.Handler{ 85 | Name: name, 86 | Pattern: regexp.MustCompile(pattern), 87 | SkipLeadingRows: 0, 88 | 89 | Encoding: japanese.ShiftJIS, 90 | // Parser: PartialCSVParser(1, 0, "\r\n"), 91 | Parser: parser, 92 | Projector: projector, 93 | Preprocessor: preprocessor, 94 | Notifier: notifier, 95 | 96 | Project: table.Project, 97 | Dataset: table.Dataset, 98 | Table: table.Table, 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /contrib/handlers/sbi_securities.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | "time" 7 | 8 | "go.nownabe.dev/bqloader" 9 | "golang.org/x/text/encoding/japanese" 10 | "golang.org/x/xerrors" 11 | ) 12 | 13 | // SBISecuritiesGlobalBankingStatement build a handler for banking statement of SBI Securities Global (SBI証券 外国株式 入出金明細). 14 | func SBISecuritiesGlobalBankingStatement(name, pattern string, table Table, notifier bqloader.Notifier) *bqloader.Handler { 15 | projector := func(_ context.Context, r []string) ([]string, error) { 16 | projected := make([]string, 6) 17 | 18 | // 0 -> 0: date (入出金日) 19 | t, err := time.Parse("2006/01/02", r[0]) 20 | if err != nil { 21 | return nil, xerrors.Errorf("failed to parse data: %w", err) 22 | } 23 | projected[0] = t.Format("2006-01-02") 24 | 25 | // 1 -> 1: 取引 26 | projected[1] = r[1] 27 | 28 | // 5 -> 2: 通貨 29 | projected[2] = r[5] 30 | 31 | // 2 -> 3: 摘要 32 | projected[3] = r[2] 33 | 34 | // 3 -> 4: 出金額 35 | projected[4] = CleanNumber(r[3]) 36 | 37 | // 4 -> 5: 入金額 38 | projected[5] = CleanNumber(r[4]) 39 | 40 | return projected, nil 41 | } 42 | 43 | return &bqloader.Handler{ 44 | Name: name, 45 | Pattern: regexp.MustCompile(pattern), 46 | SkipLeadingRows: 1, 47 | 48 | Encoding: japanese.ShiftJIS, 49 | Parser: PartialCSVParser(6, 0, "\n"), 50 | Projector: projector, 51 | Notifier: notifier, 52 | 53 | Project: table.Project, 54 | Dataset: table.Dataset, 55 | Table: table.Table, 56 | } 57 | } 58 | 59 | // SBISecuritiesGlobalExecutionHistory build a handler for execution history of SBI Securities Global (SBI証券 外国株式 約定履歴). 60 | func SBISecuritiesGlobalExecutionHistory(name, pattern string, table Table, notifier bqloader.Notifier) *bqloader.Handler { 61 | projector := func(_ context.Context, r []string) ([]string, error) { 62 | // 0: contract_date (国内約定日) 63 | contractDate, err := time.Parse("2006/01/02", r[0]) 64 | if err != nil { 65 | return nil, xerrors.Errorf("failed to parse data at column 0: %w", err) 66 | } 67 | r[0] = contractDate.Format("2006-01-02") 68 | 69 | // 8: quantity (約定数量) 70 | r[8] = CleanNumber(r[8]) 71 | 72 | // 9: unit_price (約定単価) 73 | r[9] = CleanNumber(r[9]) 74 | 75 | // 10: delivery_date (国内受渡日) 76 | deliveryDate, err := time.Parse("2006/01/02", r[10]) 77 | if err != nil { 78 | return nil, xerrors.Errorf("failed to parse data at column 10: %w", err) 79 | } 80 | r[10] = deliveryDate.Format("2006-01-02") 81 | 82 | // 11: delivery_amount (受渡金額) 83 | r[11] = CleanNumber(r[11]) 84 | 85 | return r, nil 86 | } 87 | 88 | return &bqloader.Handler{ 89 | Name: name, 90 | Pattern: regexp.MustCompile(pattern), 91 | SkipLeadingRows: 1, 92 | 93 | Encoding: japanese.ShiftJIS, 94 | Parser: PartialCSVParser(6, 0, "\n"), 95 | Projector: projector, 96 | Notifier: notifier, 97 | 98 | Project: table.Project, 99 | Dataset: table.Dataset, 100 | Table: table.Table, 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /contrib/handlers/smbc_test.go: -------------------------------------------------------------------------------- 1 | package handlers_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "go.nownabe.dev/bqloader" 9 | "go.nownabe.dev/bqloader/contrib/handlers" 10 | ) 11 | 12 | func Test_parseSMBCDate(t *testing.T) { 13 | t.Parallel() 14 | 15 | cases := []struct { 16 | input string 17 | expect time.Time 18 | e bool 19 | }{ 20 | {input: "S64.01.07", expect: time.Time{}, e: true}, 21 | {input: "H01.01.08", expect: time.Date(1989, 1, 8, 0, 0, 0, 0, time.UTC), e: false}, 22 | {input: "H31.04.30", expect: time.Date(2019, 4, 30, 0, 0, 0, 0, time.UTC), e: false}, 23 | {input: "R01.05.01", expect: time.Date(2019, 5, 1, 0, 0, 0, 0, time.UTC), e: false}, 24 | {input: "R1.5.2", expect: time.Time{}, e: true}, 25 | {input: "2021/12/17", expect: time.Date(2021, 12, 17, 0, 0, 0, 0, time.UTC), e: false}, 26 | {input: "2021/1/1", expect: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC), e: false}, 27 | } 28 | 29 | for _, c := range cases { 30 | c := c 31 | t.Run(c.input, func(t *testing.T) { 32 | t.Parallel() 33 | 34 | actual, err := handlers.ParseSMBCDate(c.input) 35 | 36 | if c.e { 37 | if err == nil { 38 | t.Errorf("Expected error didn't occur") 39 | } 40 | } else { 41 | if err != nil { 42 | t.Errorf("Unexpected error: %v", err) 43 | } 44 | 45 | if !actual.Equal(c.expect) { 46 | t.Errorf("Expected %s but %s", c.expect, actual) 47 | } 48 | } 49 | }) 50 | } 51 | } 52 | 53 | func Test_SMBCStatement(t *testing.T) { 54 | t.Parallel() 55 | 56 | const csv = "testdata/smbc_statement.csv" 57 | 58 | expected := [][]string{ 59 | {"2019-12-04", "10389", "", "カ)ビユ-カ-ド", "124001"}, 60 | {"2019-12-21", "", "160000", "振込　スミトモ タロウ", "284001"}, 61 | {"2019-12-26", "80980", "", "ミツイスミトモカ-ド (カ", "203021"}, 62 | } 63 | 64 | h, tl := buildTestHandler(t, csv, handlers.SMBCStatement) 65 | 66 | name := "path_to/smbc_statement.csv" 67 | e := bqloader.Event{Name: name, Bucket: "bucket"} 68 | 69 | if err := h.Handle(context.Background(), e); err != nil { 70 | t.Errorf("Unexpected error: %v", err) 71 | } 72 | 73 | assertEqual(t, expected, tl.result) 74 | } 75 | 76 | func Test_SMBCStatement2(t *testing.T) { 77 | t.Parallel() 78 | 79 | const csv = "testdata/smbc_statement2.csv" 80 | 81 | expected := [][]string{ 82 | {"2020-10-12", "-12200", "", "ミツイスミトモカ-ド (カ", "172991", "", ""}, 83 | {"2020-10-05", "-1900", "", "ミツイスミトモカ-ド (カ", "185191", "", ""}, 84 | {"2020-09-28", "-12571", "", "ミツイスミトモカ-ド (カ", "187091", "", ""}, 85 | {"2020-09-28", "-15000", "", "ミツイスミトモカ-ド (カ", "199662", "", ""}, 86 | {"2020-09-28", "-194873", "", "ミツイスミトモカ-ド (カ", "214662", "", ""}, 87 | {"2020-09-17", "", "240000", "振込　スミトモタロウ", "409535", "", ""}, 88 | } 89 | 90 | h, tl := buildTestHandler(t, csv, handlers.SMBCStatement) 91 | 92 | name := "path_to/smbc_statement2.csv" 93 | e := bqloader.Event{Name: name, Bucket: "bucket"} 94 | 95 | if err := h.Handle(context.Background(), e); err != nil { 96 | t.Errorf("Unexpected error: %v", err) 97 | } 98 | 99 | assertEqual(t, expected, tl.result) 100 | } 101 | -------------------------------------------------------------------------------- /notifier_test.go: -------------------------------------------------------------------------------- 1 | package bqloader_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io/ioutil" 9 | "net/http" 10 | "testing" 11 | 12 | "go.nownabe.dev/bqloader" 13 | ) 14 | 15 | type roundTripperFunc func(req *http.Request) *http.Response 16 | 17 | func (f roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { 18 | return f(req), nil 19 | } 20 | 21 | func newTestClient(f roundTripperFunc) *http.Client { 22 | return &http.Client{Transport: f} 23 | } 24 | 25 | type slackMessage struct { 26 | Channel string `json:"channel"` 27 | IconEmoji string `json:"icon_emoji,omitempty"` 28 | Text string `json:"text"` 29 | Username string `json:"username,omitempty"` 30 | } 31 | 32 | const validSlackToken = "validToken" 33 | 34 | func newSlackClient() *http.Client { 35 | return newTestClient(func(req *http.Request) *http.Response { 36 | resBody := func() string { 37 | if req.Header.Get("Authorization") != "Bearer "+validSlackToken { 38 | return `{"ok":false,"error":"not_authed"}` 39 | } 40 | 41 | reqBody, err := ioutil.ReadAll(req.Body) 42 | if err != nil { 43 | return "" 44 | } 45 | 46 | var msg slackMessage 47 | if err := json.Unmarshal(reqBody, &msg); err != nil { 48 | return `{"ok":false,"error":"invalid_form_data"}` 49 | } 50 | 51 | if msg.Channel == "" { 52 | return `{"ok":false,"error":"channel_not_found"}` 53 | } 54 | 55 | return `{"ok":true}` 56 | }() 57 | 58 | if resBody == "" { 59 | return &http.Response{StatusCode: http.StatusInternalServerError} 60 | } 61 | 62 | return &http.Response{ 63 | StatusCode: http.StatusOK, 64 | Body: ioutil.NopCloser(bytes.NewBufferString(resBody)), 65 | Header: http.Header{}, 66 | } 67 | }) 68 | } 69 | 70 | func TestSlackNotifier(t *testing.T) { 71 | t.Parallel() 72 | 73 | result := &bqloader.Result{ 74 | Event: bqloader.Event{Name: "testfile"}, 75 | Handler: &bqloader.Handler{Name: "myhandler"}, 76 | } 77 | 78 | cases := map[string]struct { 79 | notifier *bqloader.SlackNotifier 80 | result *bqloader.Result 81 | expectedHasError bool 82 | }{ 83 | "succeeded": { 84 | notifier: &bqloader.SlackNotifier{Channel: "#channel", Token: validSlackToken}, 85 | result: result, 86 | expectedHasError: false, 87 | }, 88 | "failed": { 89 | notifier: &bqloader.SlackNotifier{Channel: "#channel", Token: validSlackToken}, 90 | result: &bqloader.Result{ 91 | Event: bqloader.Event{Name: "testfile"}, 92 | Handler: &bqloader.Handler{Name: "myhandler"}, 93 | Error: fmt.Errorf("some error"), 94 | }, 95 | expectedHasError: false, 96 | }, 97 | "no token": { 98 | notifier: &bqloader.SlackNotifier{Channel: "#channel"}, 99 | result: result, 100 | expectedHasError: true, 101 | }, 102 | "with options": { 103 | notifier: &bqloader.SlackNotifier{ 104 | Channel: "#channel", 105 | Token: validSlackToken, 106 | IconEmoji: ":poop:", 107 | Username: "username", 108 | }, 109 | result: result, 110 | expectedHasError: false, 111 | }, 112 | } 113 | 114 | slackClient := newSlackClient() 115 | 116 | for name, c := range cases { 117 | c.notifier.HTTPClient = slackClient 118 | 119 | t.Run(name, func(t *testing.T) { 120 | t.Parallel() 121 | 122 | err := c.notifier.Notify(context.Background(), c.result) 123 | if c.expectedHasError && err == nil { 124 | t.Errorf("expected error but no error occurred") 125 | } 126 | if !c.expectedHasError && err != nil { 127 | t.Errorf("unexpected error: %s", err) 128 | } 129 | }) 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /bqloader_test.go: -------------------------------------------------------------------------------- 1 | package bqloader 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "regexp" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func TestLoader(t *testing.T) { 14 | t.Parallel() 15 | 16 | projector := func(_ context.Context, r []string) ([]string, error) { 17 | t, err := time.Parse("2006/01/02", r[0]) 18 | if err != nil { 19 | return nil, fmt.Errorf("Failed to parse date: %v", err) 20 | } 21 | 22 | r[0] = t.Format("2006-01-02") 23 | 24 | return r, nil 25 | } 26 | 27 | te := newTestExtractor() 28 | tl := newTestLoader() 29 | tn := newTestNotifier() 30 | 31 | handler := &Handler{ 32 | Name: "test-handler", 33 | Pattern: regexp.MustCompile("^test/"), 34 | Parser: CSVParser(), 35 | Notifier: tn, 36 | Projector: projector, 37 | Extractor: te, 38 | Loader: tl, 39 | } 40 | 41 | ctx := context.Background() 42 | 43 | loader, err := New(WithPrettyLogging(), WithLogLevel("debug"), WithConcurrency(4)) 44 | if err != nil { 45 | t.Fatal(err) 46 | } 47 | loader.MustAddHandler(ctx, handler) 48 | 49 | src := bytes.NewBufferString("2020/11/21,foo,123") 50 | e := Event{Name: "test/name", Bucket: "bucket", source: src} 51 | 52 | if err := loader.Handle(ctx, e); err != nil { 53 | t.Fatal(err) 54 | } 55 | 56 | res := tl.(*testLoader) 57 | 58 | if len(res.result) != 1 { 59 | t.Fatalf("Size of result records should be 1, but %d.", len(res.result)) 60 | } 61 | 62 | if len(res.result[0]) != 3 { 63 | t.Fatalf("Size of each record be 3, but %d", len(res.result[0])) 64 | } 65 | 66 | if res.result[0][0] != "2020-11-21" { 67 | t.Errorf(`results[0][0] should be "2020-11-21", but "%s"`, res.result[0][0]) 68 | } 69 | 70 | if res.result[0][1] != "foo" { 71 | t.Errorf(`results[0][1] should be "foo", but "%s"`, res.result[0][1]) 72 | } 73 | 74 | if res.result[0][2] != "123" { 75 | t.Errorf(`results[0][2] should be "123", but "%s"`, res.result[0][2]) 76 | } 77 | } 78 | 79 | func TestBQLoader_error(t *testing.T) { 80 | t.Parallel() 81 | 82 | projector := func(_ context.Context, r []string) ([]string, error) { 83 | return nil, fmt.Errorf("projector error") 84 | } 85 | 86 | te := newTestExtractor() 87 | tl := newTestLoader() 88 | tn := newTestNotifier() 89 | 90 | handler := &Handler{ 91 | Name: "test-handler", 92 | Pattern: regexp.MustCompile("^test/"), 93 | Parser: CSVParser(), 94 | Notifier: tn, 95 | Projector: projector, 96 | Extractor: te, 97 | Loader: tl, 98 | } 99 | 100 | ctx := context.Background() 101 | 102 | loader, err := New(WithPrettyLogging(), WithLogLevel("debug")) 103 | if err != nil { 104 | t.Fatal(err) 105 | } 106 | loader.MustAddHandler(ctx, handler) 107 | 108 | src := bytes.NewBufferString("2020/11/21,foo,123") 109 | e := Event{Name: "test/name", Bucket: "bucket", source: src} 110 | 111 | if err := loader.Handle(ctx, e); err == nil { 112 | t.Error("expected error but no error occurred") 113 | } 114 | } 115 | 116 | type testExtractor struct{} 117 | 118 | func newTestExtractor() Extractor { 119 | return &testExtractor{} 120 | } 121 | 122 | func (e *testExtractor) Extract(_ context.Context, ev Event) (io.Reader, func(), error) { 123 | return ev.source, func() {}, nil 124 | } 125 | 126 | type testLoader struct { 127 | result [][]string 128 | } 129 | 130 | func newTestLoader() Loader { 131 | return &testLoader{} 132 | } 133 | 134 | func (l *testLoader) Load(ctx context.Context, rs [][]string) error { 135 | l.result = rs 136 | return nil 137 | } 138 | 139 | type testNotifier struct{} 140 | 141 | func newTestNotifier() Notifier { 142 | return &testNotifier{} 143 | } 144 | 145 | func (n *testNotifier) Notify(ctx context.Context, r *Result) error { 146 | return nil 147 | } 148 | -------------------------------------------------------------------------------- /notifier.go: -------------------------------------------------------------------------------- 1 | package bqloader 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io/ioutil" 9 | "net/http" 10 | "sync" 11 | 12 | "github.com/rs/zerolog/log" 13 | "golang.org/x/xerrors" 14 | ) 15 | 16 | // Notifier notifies results for each event. 17 | type Notifier interface { 18 | Notify(context.Context, *Result) error 19 | } 20 | 21 | // Result is a result for each event. 22 | type Result struct { 23 | Event Event 24 | Handler *Handler 25 | Error error 26 | } 27 | 28 | // SlackNotifier is a notifier for Slack. 29 | // SlackNotifier requires bot token and permissions. 30 | // Recommended permissions are chat:write, chat:write.customize and chat:write.public. 31 | type SlackNotifier struct { 32 | Channel string 33 | Token string 34 | 35 | // Optional. 36 | IconEmoji string 37 | 38 | // Optional. 39 | Username string 40 | 41 | // Optional. 42 | HTTPClient *http.Client 43 | 44 | once sync.Once 45 | } 46 | 47 | type slackMessage struct { 48 | Channel string `json:"channel"` 49 | IconEmoji string `json:"icon_emoji,omitempty"` 50 | Text string `json:"text"` 51 | Username string `json:"username,omitempty"` 52 | } 53 | 54 | type slackResponse struct { 55 | OK bool `json:"ok"` 56 | Error string `json:"error"` 57 | } 58 | 59 | // Notify notifies results to Slack channel. 60 | func (n *SlackNotifier) Notify(ctx context.Context, r *Result) error { 61 | l := log.Ctx(ctx) 62 | 63 | n.once.Do(func() { 64 | if n.HTTPClient == nil { 65 | n.HTTPClient = &http.Client{} 66 | } 67 | }) 68 | 69 | var text string 70 | if r.Error == nil { 71 | text = fmt.Sprintf(`:white_check_mark: %s handler successfully loaded %s`, r.Handler.Name, r.Event.Name) 72 | } else { 73 | text = fmt.Sprintf(`:x: %s handler failed to load %s: %s`, r.Handler.Name, r.Event.Name, r.Error) 74 | } 75 | m := &slackMessage{ 76 | Channel: n.Channel, 77 | IconEmoji: n.IconEmoji, 78 | Text: text, 79 | Username: n.Username, 80 | } 81 | l.Debug().Msgf("m = %+v", m) 82 | 83 | if err := n.postMessage(ctx, m); err != nil { 84 | return xerrors.Errorf("slack postMessage failed: %w", err) 85 | } 86 | 87 | return nil 88 | } 89 | 90 | func (n *SlackNotifier) postMessage(ctx context.Context, m *slackMessage) error { 91 | l := log.Ctx(ctx) 92 | 93 | reqJSON, err := json.Marshal(m) 94 | if err != nil { 95 | return xerrors.Errorf("failed to marshal json: %w", err) 96 | } 97 | 98 | req, err := http.NewRequest("POST", "https://slack.com/api/chat.postMessage", bytes.NewReader(reqJSON)) 99 | if err != nil { 100 | return xerrors.Errorf("failed to build http request: %v", err) 101 | } 102 | 103 | req.Header.Set("Content-Type", "application/json") 104 | l.Debug().Msgf("req = %+v", req) 105 | req.Header.Set("Authorization", "Bearer "+n.Token) 106 | 107 | resp, err := n.HTTPClient.Do(req.WithContext(ctx)) 108 | if err != nil { 109 | return xerrors.Errorf("failed to send request: %w", err) 110 | } 111 | defer resp.Body.Close() 112 | 113 | l.Debug().Msgf("resp = %+v", resp) 114 | 115 | body, err := ioutil.ReadAll(resp.Body) 116 | if err != nil { 117 | return xerrors.Errorf("failed to read response body: %w", err) 118 | } 119 | 120 | l.Debug().Msgf("body = %s", body) 121 | 122 | if resp.StatusCode >= http.StatusBadRequest { 123 | return xerrors.Errorf( 124 | "slack webhook request failed with status code %d (%s)", resp.StatusCode, body) 125 | } 126 | 127 | var sres slackResponse 128 | if err := json.Unmarshal(body, &sres); err != nil { 129 | return xerrors.Errorf("failed to unmarshal response body: %w", err) 130 | } 131 | 132 | if !sres.OK { 133 | return xerrors.Errorf("failed to send message: %s", sres.Error) 134 | } 135 | 136 | return nil 137 | } 138 | -------------------------------------------------------------------------------- /contrib/handlers/amex.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | "regexp" 8 | "time" 9 | 10 | "github.com/extrame/xls" 11 | "gitlab.com/osaki-lab/iowrapper" 12 | "go.nownabe.dev/bqloader" 13 | "golang.org/x/xerrors" 14 | ) 15 | 16 | var ( 17 | errAMEXStatementNoSheet = errors.New("no sheet found") 18 | ) 19 | 20 | // AMEXStatement build a *bqloader.Handler for statements of AMEX (American Express). 21 | // To add column of payment month, keep the file name as the payment month like '2022-07.xls'. 22 | func AMEXStatement(name, pattern string, table Table, notifier bqloader.Notifier) *bqloader.Handler { 23 | var monthKey contextKey = "month" 24 | 25 | getRow := func(sheet *xls.WorkSheet, row int) (r *xls.Row, ok bool) { 26 | defer func() { recover() }() 27 | 28 | r = nil 29 | ok = false 30 | 31 | return sheet.Row(row), true 32 | } 33 | 34 | dateRE := regexp.MustCompile(`^\d\d\d\d/\d\d/\d\d$`) 35 | 36 | parser := func(_ context.Context, r io.Reader) ([][]string, error) { 37 | wb, err := xls.OpenReader(iowrapper.NewSeeker(r), "utf-8") 38 | if err != nil { 39 | return nil, xerrors.Errorf("failed to open xls file: %w", err) 40 | } 41 | 42 | sheet := wb.GetSheet(0) 43 | if sheet == nil { 44 | return nil, errAMEXStatementNoSheet 45 | } 46 | 47 | records := [][]string{} 48 | 49 | for i := 0; i <= int(sheet.MaxRow); i++ { 50 | row, ok := getRow(sheet, i) 51 | if !ok { 52 | continue 53 | } 54 | 55 | if val := row.Col(row.FirstCol()); !dateRE.Match([]byte(val)) { 56 | continue 57 | } 58 | 59 | record := []string{} 60 | 61 | for colNum := row.FirstCol(); colNum < row.LastCol(); colNum++ { 62 | record = append(record, row.Col(colNum)) 63 | } 64 | 65 | records = append(records, record) 66 | } 67 | 68 | return records, nil 69 | } 70 | 71 | filenameRE := regexp.MustCompile(`/(\d\d\d\d-\d\d)\.xls$`) 72 | 73 | preprocessor := func(ctx context.Context, e bqloader.Event) (context.Context, error) { 74 | match := filenameRE.FindStringSubmatch(e.Name) 75 | if len(match) < 2 { 76 | return ctx, xerrors.Errorf("wrong object path: %s", e.Name) 77 | } 78 | 79 | month, err := time.Parse("2006-01", match[1]) 80 | if err != nil { 81 | return ctx, xerrors.Errorf("failed to parse payment month from object path: %s: %w", match[1], err) 82 | } 83 | 84 | return context.WithValue(ctx, monthKey, month.Format("2006-01-02")), nil 85 | } 86 | 87 | projector := func(ctx context.Context, r []string) ([]string, error) { 88 | if r[0] == "" { 89 | return nil, nil 90 | } 91 | 92 | paymentMonth, ok := ctx.Value(monthKey).(string) 93 | if !ok { 94 | return nil, xerrors.Errorf("failed to get payment month from context: %v", paymentMonth) 95 | } 96 | 97 | // 0: date (ご利用日) 98 | t, err := time.Parse("2006/01/02", r[0]) 99 | if err != nil { 100 | return nil, xerrors.Errorf("failed to parse date: %v", err) 101 | } 102 | r[0] = t.Format("2006-01-02") 103 | 104 | // 1: データ処理日 105 | t, err = time.Parse("2006/01/02", r[1]) 106 | if err != nil { 107 | return nil, xerrors.Errorf("failed to parse date: %v", err) 108 | } 109 | r[1] = t.Format("2006-01-02") 110 | 111 | // 4: 金額 112 | r[4] = CleanNumber(r[4]) 113 | 114 | // 8: payment_month (支払い月) 115 | r = append(r, paymentMonth) 116 | 117 | return r, nil 118 | } 119 | 120 | return &bqloader.Handler{ 121 | Name: name, 122 | Pattern: regexp.MustCompile(pattern), 123 | SkipLeadingRows: 0, 124 | 125 | // Parser: PartialCSVParser(1, 0, "\r\n"), 126 | Parser: parser, 127 | Projector: projector, 128 | Preprocessor: preprocessor, 129 | Notifier: notifier, 130 | 131 | Project: table.Project, 132 | Dataset: table.Dataset, 133 | Table: table.Table, 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Package bqloader is a simple ETL framework running on Cloud Functions 4 | to load data from Cloud Storage into BigQuery. 5 | 6 | Getting started with pre-configured handlers 7 | 8 | See the example to get a full instruction. 9 | https://github.com/nownabe/go-bqloader/tree/main/examples/pre_configured_handlers 10 | 11 | To load some types of CSV formats, you can use pre-configured handlers. 12 | See the full list on GitHub. https://github.com/nownabe/go-bqloader/tree/main/contrib/handlers 13 | 14 | package myfunc 15 | 16 | import ( 17 | "context" 18 | 19 | "go.nownabe.dev/bqloader" 20 | "go.nownabe.dev/bqloader/contrib/handlers" 21 | ) 22 | 23 | var loader bqloader.BQLoader 24 | 25 | func init() { 26 | loader, _ = bqloader.New() 27 | 28 | t := handlers.TableGenerator(os.Getenv("BIGQUERY_PROJECT_ID"), os.Getenv("BIGQUERY_DATASET_ID")) 29 | n := &bqloader.SlackNotifier{ 30 | Token: os.Getenv("SLACK_TOKEN"), 31 | Channel: os.Getenv("SLACK_CHANNEL"), 32 | } 33 | 34 | handlers.MustAddHandlers(context.Background(), loader, 35 | // These build handlers to load CSVs, given four arguments: 36 | // handler name, a pattern to file path on Cloud Storage, a BigQuery table and a notifier. 37 | handlers.SBISumishinNetBankStatement("SBI Bank", `^sbi_bank/`, t("sbi_bank"), n), 38 | handlers.SMBCCardStatement("SMBC Card", `^smbc_card/`, t("smbc_card"), n), 39 | ) 40 | } 41 | 42 | // BQLoad is the entrypoint for Cloud Functions. 43 | func BQLoad(ctx context.Context, e bqloader.Event) error { 44 | return loader.Handle(ctx, e) 45 | } 46 | 47 | Getting started with custom handlers 48 | 49 | See Quickstart example to get a full instruction. 50 | https://github.com/nownabe/go-bqloader/tree/main/examples/quickstart 51 | 52 | For simple transforming and loading CSV, import the package `go.nownabe.dev/bqloader` and write your handler. 53 | 54 | package myfunc 55 | 56 | import ( 57 | "context" 58 | "os" 59 | "regexp" 60 | "strings" 61 | "time" 62 | 63 | "golang.org/x/text/encoding/japanese" 64 | "golang.org/x/xerrors" 65 | 66 | "go.nownabe.dev/bqloader" 67 | ) 68 | 69 | var loader bqloader.BQLoader 70 | 71 | func init() { 72 | loader, _ = bqloader.New() 73 | loader.MustAddHandler(context.Background(), newHandler()) 74 | } 75 | 76 | func newHandler() *bqloader.Handler { 77 | // This projector converts date fields formatted as "2006/01/02" 78 | // at the first column into strings like "2006-01-02" that satisfies 79 | // BigQuery date type. 80 | projector := func(_ context.Context, r []string) ([]string, error) { 81 | t, err := time.Parse("2006/01/02", r[0]) 82 | if err != nil { 83 | return nil, xerrors.Errorf("Column 0 cannot parse as a date: %w", err) 84 | } 85 | 86 | r[0] = t.Format("2006-01-02") 87 | 88 | return r, nil 89 | } 90 | 91 | return &bqloader.Handler{ 92 | Name: "quickstart", // Handler name used in logs and notifications. 93 | Pattern: regexp.MustCompile("^example_bank/"), // This handler processes files matched to this pattern. 94 | Encoding: japanese.ShiftJIS, // Source file encoding. 95 | Parser: bqloader.CSVParser(), // Parser parses source file into records. 96 | Notifier: &bqloader.SlackNotifier{ 97 | Token: os.Getenv("SLACK_TOKEN"), 98 | Channel: os.Getenv("SLACK_CHANNEL"), 99 | }, 100 | Projector: projector, // Projector transforms each row. 101 | SkipLeadingRows: 1, // Skip header row. 102 | 103 | // Destination. 104 | Project: os.Getenv("BIGQUERY_PROJECT_ID"), 105 | Dataset: os.Getenv("BIGQUERY_DATASET_ID"), 106 | Table: os.Getenv("BIGQUERY_TABLE_ID"), 107 | } 108 | } 109 | 110 | // BQLoad is the entrypoint for Cloud Functions. 111 | func BQLoad(ctx context.Context, e bqloader.Event) error { 112 | return loader.Handle(ctx, e) 113 | } 114 | 115 | */ 116 | package bqloader 117 | -------------------------------------------------------------------------------- /handler_test.go: -------------------------------------------------------------------------------- 1 | package bqloader 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | func Test_Handler_WithSkipping(t *testing.T) { 12 | t.Parallel() 13 | 14 | projector := func(_ context.Context, r []string) ([]string, error) { 15 | if r[0] == "" { 16 | return nil, nil 17 | } 18 | 19 | return r, nil 20 | } 21 | 22 | rawCSV := `123,456,789 23 | ,foo bar,123 24 | 234,567,890` 25 | src := bytes.NewBufferString(rawCSV) 26 | 27 | tl := newTestLoader() 28 | 29 | handler := &Handler{ 30 | Name: "test-handler", 31 | Parser: CSVParser(), 32 | Projector: projector, 33 | BatchSize: defaultBatchSize, 34 | Extractor: newTestExtractor(), 35 | Loader: tl, 36 | semaphore: make(chan struct{}, 1), 37 | } 38 | e := Event{Name: "test/name", Bucket: "bucket", source: src} 39 | 40 | err := handler.Handle(context.Background(), e) 41 | if err != nil { 42 | t.Fatalf("Unexpected error: %v", err) 43 | } 44 | 45 | res := tl.(*testLoader) 46 | 47 | if len(res.result) != 2 { 48 | t.Fatalf("Size of result records should be 2, but %d", len(res.result)) 49 | } 50 | 51 | if len(res.result[0]) != 3 { 52 | t.Fatalf("Size of each record be 3, but %d", len(res.result[0])) 53 | } 54 | 55 | if res.result[0][0] != "123" { 56 | t.Errorf(`results[0][0] should be "123", but "%s"`, res.result[0][0]) 57 | } 58 | 59 | if res.result[0][1] != "456" { 60 | t.Errorf(`results[0][1] should be "456", but "%s"`, res.result[0][1]) 61 | } 62 | 63 | if res.result[0][2] != "789" { 64 | t.Errorf(`results[0][2] should be "789", but "%s"`, res.result[0][2]) 65 | } 66 | 67 | if len(res.result[1]) != 3 { 68 | t.Fatalf("Size of each record be 3, but %d", len(res.result[1])) 69 | } 70 | 71 | if res.result[1][0] != "234" { 72 | t.Errorf(`results[1][0] should be "234", but "%s"`, res.result[1][0]) 73 | } 74 | 75 | if res.result[1][1] != "567" { 76 | t.Errorf(`results[1][1] should be "567", but "%s"`, res.result[1][1]) 77 | } 78 | 79 | if res.result[1][2] != "890" { 80 | t.Errorf(`results[1][2] should be "890", but "%s"`, res.result[1][2]) 81 | } 82 | } 83 | 84 | func Test_Handler_WithPreprocessor(t *testing.T) { 85 | t.Parallel() 86 | 87 | projector := func(ctx context.Context, r []string) ([]string, error) { 88 | prefix, ok := ctx.Value("prefix").(string) 89 | if !ok { 90 | return nil, fmt.Errorf("prefix is not found") 91 | } 92 | 93 | row := make([]string, 4) 94 | row[0] = prefix 95 | row[1] = r[0] 96 | row[2] = r[1] 97 | row[3] = r[2] 98 | 99 | return row, nil 100 | } 101 | 102 | preprocessor := func(ctx context.Context, e Event) (context.Context, error) { 103 | prefix := strings.Split(e.Name, "/")[0] 104 | return context.WithValue(ctx, "prefix", prefix), nil 105 | } 106 | 107 | src := bytes.NewBufferString(`123,456,789`) 108 | 109 | tl := newTestLoader() 110 | 111 | handler := &Handler{ 112 | Name: "test-handler", 113 | Parser: CSVParser(), 114 | Projector: projector, 115 | Preprocessor: preprocessor, 116 | BatchSize: defaultBatchSize, 117 | Extractor: newTestExtractor(), 118 | Loader: tl, 119 | semaphore: make(chan struct{}, 1), 120 | } 121 | e := Event{Name: "test/name", Bucket: "bucket", source: src} 122 | 123 | err := handler.Handle(context.Background(), e) 124 | if err != nil { 125 | t.Fatalf("Unexpected error: %v", err) 126 | } 127 | 128 | res := tl.(*testLoader) 129 | 130 | if len(res.result) != 1 { 131 | t.Fatalf("Size of result records should be 1, but %d", len(res.result)) 132 | } 133 | 134 | if len(res.result[0]) != 4 { 135 | t.Fatalf("Size of each record be 4, but %d", len(res.result[0])) 136 | } 137 | 138 | if res.result[0][0] != "test" { 139 | t.Errorf(`results[0][0] should be "test", but "%s"`, res.result[0][0]) 140 | } 141 | 142 | if res.result[0][1] != "123" { 143 | t.Errorf(`results[0][0] should be "123", but "%s"`, res.result[0][1]) 144 | } 145 | 146 | if res.result[0][2] != "456" { 147 | t.Errorf(`results[0][1] should be "456", but "%s"`, res.result[0][2]) 148 | } 149 | 150 | if res.result[0][3] != "789" { 151 | t.Errorf(`results[0][2] should be "789", but "%s"`, res.result[0][3]) 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /contrib/handlers/handlers_test.go: -------------------------------------------------------------------------------- 1 | package handlers_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "testing" 10 | 11 | "go.nownabe.dev/bqloader" 12 | "go.nownabe.dev/bqloader/contrib/handlers" 13 | ) 14 | 15 | type testLoader struct { 16 | result [][]string 17 | } 18 | 19 | func (l *testLoader) Load(ctx context.Context, rs [][]string) error { 20 | l.result = rs 21 | 22 | return nil 23 | } 24 | 25 | type testExtractor struct { 26 | source io.Reader 27 | } 28 | 29 | func (e *testExtractor) Extract(_ context.Context, ev bqloader.Event) (io.Reader, func(), error) { 30 | return e.source, func() {}, nil 31 | } 32 | 33 | func assertEqual(t *testing.T, expected [][]string, actual [][]string) { 34 | t.Helper() 35 | 36 | if len(expected) != len(actual) { 37 | t.Fatalf("expected %d length, but %d", len(expected), len(actual)) 38 | } 39 | 40 | for i := range expected { 41 | if len(expected[i]) != len(actual[i]) { 42 | t.Errorf("expected length of actual[%d] is %d, but %d", i, len(expected[i]), len(actual[i])) 43 | } 44 | 45 | for j := range expected[i] { 46 | if expected[i][j] != actual[i][j] { 47 | t.Errorf("expected actual[%d][%d] is '%s', but '%s'", i, j, expected[i][j], actual[i][j]) 48 | } 49 | } 50 | } 51 | } 52 | 53 | func buildTestHandler( 54 | t *testing.T, 55 | path string, 56 | f func(string, string, handlers.Table, bqloader.Notifier) *bqloader.Handler, 57 | ) (*bqloader.Handler, *testLoader) { 58 | t.Helper() 59 | 60 | body, err := ioutil.ReadFile(path) 61 | if err != nil { 62 | t.Fatalf("failed to read CSV: %v", err) 63 | } 64 | 65 | tl := &testLoader{} 66 | table := handlers.Table{Project: "p", Dataset: "d", Table: "t"} 67 | 68 | h := f("name", "^path_to/", table, nil) 69 | h.SetConcurrency(1) 70 | h.BatchSize = 100 71 | h.Loader = tl 72 | h.Extractor = &testExtractor{source: bytes.NewBuffer(body)} 73 | 74 | return h, tl 75 | } 76 | 77 | func Test_CleanNumber(t *testing.T) { 78 | t.Parallel() 79 | 80 | cases := []struct { 81 | input string 82 | output string 83 | }{ 84 | {input: "12,345", output: "12345"}, 85 | {input: "306.4800USD", output: "306.4800"}, 86 | {input: "257,345円", output: "257345"}, 87 | {input: "-12,345", output: "-12345"}, 88 | {input: "¥12345-", output: "12345"}, 89 | {input: "-", output: ""}, 90 | } 91 | 92 | for _, c := range cases { 93 | c := c 94 | t.Run(c.input, func(t *testing.T) { 95 | t.Parallel() 96 | 97 | o := handlers.CleanNumber(c.input) 98 | if o != c.output { 99 | t.Errorf("CleanNumber(%s) should be %s, but %s", c.input, c.output, o) 100 | } 101 | }) 102 | } 103 | } 104 | 105 | func Test_PartialCSVParser(t *testing.T) { 106 | t.Parallel() 107 | 108 | cases := []struct { 109 | name string 110 | skipHeadRows uint 111 | skipTailRows uint 112 | sep string 113 | body string 114 | expect [][]string 115 | }{ 116 | { 117 | skipHeadRows: 3, 118 | skipTailRows: 3, 119 | sep: "\n", 120 | body: "foo\n\nbar\n1,2,3\n4,5,6\n\nbaz\nqux", 121 | expect: [][]string{{"1", "2", "3"}, {"4", "5", "6"}}, 122 | }, 123 | { 124 | skipHeadRows: 0, 125 | skipTailRows: 3, 126 | sep: "\n", 127 | body: "1,2,3\n4,5,6\n\nbaz\nqux", 128 | expect: [][]string{{"1", "2", "3"}, {"4", "5", "6"}}, 129 | }, 130 | { 131 | skipHeadRows: 3, 132 | skipTailRows: 0, 133 | sep: "\n", 134 | body: "foo\n\nbar\n1,2,3\n4,5,6", 135 | expect: [][]string{{"1", "2", "3"}, {"4", "5", "6"}}, 136 | }, 137 | { 138 | skipHeadRows: 3, 139 | skipTailRows: 3, 140 | sep: "\r\n", 141 | body: "foo\r\n\r\nbar\r\n1,2,3\r\n4,5,6\r\n\r\nbaz\r\nqux", 142 | expect: [][]string{{"1", "2", "3"}, {"4", "5", "6"}}, 143 | }, 144 | } 145 | 146 | ctx := context.Background() 147 | for _, c := range cases { 148 | c := c 149 | t.Run( 150 | fmt.Sprintf("head=%d,tail=%d,sep=%q", c.skipHeadRows, c.skipTailRows, c.sep), 151 | func(t *testing.T) { 152 | t.Parallel() 153 | 154 | f := handlers.PartialCSVParser(c.skipHeadRows, c.skipTailRows, c.sep) 155 | actual, err := f(ctx, bytes.NewReader([]byte(c.body))) 156 | if err != nil { 157 | t.Errorf("Unexpected error: %v", err) 158 | } 159 | 160 | assertEqual(t, c.expect, actual) 161 | }, 162 | ) 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | bqloader 2 | ======== 3 | 4 | [![PkgGoDev](https://pkg.go.dev/badge/go.nownabe.dev/bqloader)](https://pkg.go.dev/go.nownabe.dev/bqloader) 5 | ![Main Branch Workflow](https://github.com/nownabe/go-bqloader/workflows/Main%20Branch%20Workflow/badge.svg) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/nownabe/go-bqloader)](https://goreportcard.com/report/github.com/nownabe/go-bqloader) 7 | [![codecov](https://codecov.io/gh/nownabe/go-bqloader/branch/main/graph/badge.svg)](https://codecov.io/gh/nownabe/go-bqloader) 8 | ![GitHub](https://img.shields.io/github/license/nownabe/go-bqloader) 9 | ![GitHub tag (latest SemVer)](https://img.shields.io/github/v/tag/nownabe/go-bqloader?sort=semver) 10 | 11 | bqloader is a simple ETL framework to load data from Cloud Storage into BigQuery. 12 | 13 | ## Installation 14 | 15 | ```bash 16 | go get -u go.nownabe.dev/bqloader 17 | ``` 18 | 19 | ## Getting Started with Pre-configured Handlers 20 | 21 | See the [example](https://github.com/nownabe/go-bqloader/tree/main/examples/pre_configured_handlers) to get a full instruction. 22 | 23 | To load some types of CSV formats, you can use pre-configured handlers. 24 | See [full list](https://github.com/nownabe/go-bqloader/tree/main/contrib/handlers). 25 | 26 | 27 | ```go 28 | package myfunc 29 | 30 | import ( 31 | "context" 32 | 33 | "go.nownabe.dev/bqloader" 34 | "go.nownabe.dev/bqloader/contrib/handlers" 35 | ) 36 | 37 | var loader bqloader.BQLoader 38 | 39 | func init() { 40 | loader, _ = bqloader.New() 41 | 42 | t := handlers.TableGenerator(os.Getenv("BIGQUERY_PROJECT_ID"), os.Getenv("BIGQUERY_DATASET_ID")) 43 | n := &bqloader.SlackNotifier{ 44 | Token: os.Getenv("SLACK_TOKEN"), 45 | Channel: os.Getenv("SLACK_CHANNEL"), 46 | } 47 | 48 | handlers.MustAddHandlers(context.Background(), loader, 49 | /* 50 | These build handlers to load CSVs, given four arguments: 51 | handler name, a pattern to file path on Cloud Storage, a BigQuery table and a notifier. 52 | */ 53 | handlers.SBISumishinNetBankStatement("SBI Bank", `^sbi_bank/`, t("sbi_bank"), n), 54 | handlers.SMBCCardStatement("SMBC Card", `^smbc_card/`, t("smbc_card"), n), 55 | ) 56 | } 57 | 58 | // BQLoad is the entrypoint for Cloud Functions. 59 | func BQLoad(ctx context.Context, e bqloader.Event) error { 60 | return loader.Handle(ctx, e) 61 | } 62 | ``` 63 | 64 | 65 | ## Getting Started with Custom Handlers 66 | 67 | (See [Quickstart example](https://github.com/nownabe/go-bqloader/tree/main/examples/quickstart) to get a full instruction.) 68 | 69 | To load other CSVs, import the package `go.nownabe.dev/bqloader` and write your custom handler. 70 | 71 | ```go 72 | package myfunc 73 | 74 | import ( 75 | "context" 76 | "os" 77 | "regexp" 78 | "strings" 79 | "time" 80 | 81 | "golang.org/x/text/encoding/japanese" 82 | "golang.org/x/xerrors" 83 | 84 | "go.nownabe.dev/bqloader" 85 | ) 86 | 87 | var loader bqloader.BQLoader 88 | 89 | func init() { 90 | loader, _ = bqloader.New() 91 | loader.MustAddHandler(context.Background(), newHandler()) 92 | } 93 | 94 | func newHandler() *bqloader.Handler { 95 | /* 96 | This projector converts date fields formatted as "2006/01/02" 97 | at the first column into strings like "2006-01-02" that satisfies 98 | BigQuery date type. 99 | */ 100 | projector := func(_ context.Context, r []string) ([]string, error) { 101 | t, err := time.Parse("2006/01/02", r[0]) 102 | if err != nil { 103 | return nil, xerrors.Errorf("Column 0 cannot parse as a date: %w", err) 104 | } 105 | 106 | r[0] = t.Format("2006-01-02") 107 | 108 | return r, nil 109 | } 110 | 111 | return &bqloader.Handler{ 112 | Name: "quickstart", // Handler name used in logs and notifications. 113 | Pattern: regexp.MustCompile("^example_bank/"), // This handler processes files matched to this pattern. 114 | Encoding: japanese.ShiftJIS, // Source file encoding. 115 | Parser: bqloader.CSVParser(), // Parser parses source file into records. 116 | Notifier: &bqloader.SlackNotifier{ 117 | Token: os.Getenv("SLACK_TOKEN"), 118 | Channel: os.Getenv("SLACK_CHANNEL"), 119 | }, 120 | Projector: projector, // Projector transforms each row. 121 | SkipLeadingRows: 1, // Skip header row. 122 | 123 | // Destination. 124 | Project: os.Getenv("BIGQUERY_PROJECT_ID"), 125 | Dataset: os.Getenv("BIGQUERY_DATASET_ID"), 126 | Table: os.Getenv("BIGQUERY_TABLE_ID"), 127 | } 128 | } 129 | 130 | // BQLoad is the entrypoint for Cloud Functions. 131 | func BQLoad(ctx context.Context, e bqloader.Event) error { 132 | return loader.Handle(ctx, e) 133 | } 134 | ``` 135 | 136 | ## Diagram 137 | 138 | ![diagram](https://raw.githubusercontent.com/nownabe/go-bqloader/main/diagram.png) 139 | -------------------------------------------------------------------------------- /bqloader.go: -------------------------------------------------------------------------------- 1 | package bqloader 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | "sync" 8 | "time" 9 | 10 | "cloud.google.com/go/functions/metadata" 11 | "github.com/rs/zerolog" 12 | "golang.org/x/sync/errgroup" 13 | "golang.org/x/xerrors" 14 | ) 15 | 16 | // BQLoader loads data from Cloud Storage to BigQuery table. 17 | type BQLoader interface { 18 | AddHandler(context.Context, *Handler) error 19 | Handle(context.Context, Event) error 20 | MustAddHandler(context.Context, *Handler) 21 | } 22 | 23 | // New build a new Loader. 24 | func New(opts ...Option) (BQLoader, error) { 25 | bq := &bqloader{ 26 | handlers: []*Handler{}, 27 | mu: sync.RWMutex{}, 28 | prettyLogging: false, 29 | logLevel: zerolog.ErrorLevel, 30 | concurrency: 1, 31 | } 32 | 33 | for _, o := range opts { 34 | if err := o.apply(bq); err != nil { 35 | return nil, err 36 | } 37 | } 38 | 39 | var w io.Writer 40 | if bq.prettyLogging { 41 | w = zerolog.ConsoleWriter{Out: os.Stdout} 42 | } else { 43 | w = os.Stdout 44 | } 45 | l := zerolog.New(w).Level(bq.logLevel).With().Timestamp().Logger().Hook(severityHook{}) 46 | bq.logger = &l 47 | 48 | bq.semaphore = make(chan struct{}, bq.concurrency) 49 | 50 | return bq, nil 51 | } 52 | 53 | type bqloader struct { 54 | handlers []*Handler 55 | mu sync.RWMutex 56 | logger *zerolog.Logger 57 | prettyLogging bool 58 | logLevel zerolog.Level 59 | concurrency int 60 | semaphore chan struct{} 61 | } 62 | 63 | func (l *bqloader) AddHandler(ctx context.Context, h *Handler) error { 64 | l.mu.Lock() 65 | defer l.mu.Unlock() 66 | 67 | if h.Extractor == nil { 68 | ex, err := newDefaultExtractor(ctx, h.Project) 69 | if err != nil { 70 | err = xerrors.Errorf("failed to build default extractor for project '%s': %w", h.Project, err) 71 | h.logger(ctx, l.logger).Err(err).Msg(err.Error()) 72 | return err 73 | } 74 | h.Extractor = ex 75 | } 76 | 77 | if h.Loader == nil { 78 | loader, err := newDefaultLoader(ctx, h.Project, h.Dataset, h.Table) 79 | if err != nil { 80 | err = xerrors.Errorf("failed to build default loader for table '%s.%s.%s': %w", 81 | h.Project, h.Dataset, h.Table, err) 82 | h.logger(ctx, l.logger).Err(err).Msg(err.Error()) 83 | return err 84 | } 85 | h.Loader = loader 86 | } 87 | 88 | h.semaphore = l.semaphore 89 | 90 | if h.BatchSize == 0 { 91 | h.BatchSize = defaultBatchSize 92 | } 93 | 94 | l.handlers = append(l.handlers, h) 95 | 96 | return nil 97 | } 98 | 99 | func (l *bqloader) MustAddHandler(ctx context.Context, h *Handler) { 100 | if err := l.AddHandler(ctx, h); err != nil { 101 | panic(err) 102 | } 103 | } 104 | 105 | func (l *bqloader) Handle(ctx context.Context, e Event) error { 106 | ctx = withStartedTime(ctx) 107 | logger := contextualLogger(ctx, e, l.logger) 108 | 109 | logger.Info().Msg("bqloader started to handle an event") 110 | defer func() { 111 | now := time.Now() 112 | e := logger.Info().Time("finished", now) 113 | if t, ok := startedTimeFrom(ctx); ok { 114 | e.TimeDiff("elapsed", now, t) 115 | } 116 | e.Msgf("bqloader finished to handle an envent") 117 | }() 118 | 119 | g, ctx := errgroup.WithContext(logger.WithContext(ctx)) 120 | 121 | for _, h := range l.handlers { 122 | if h.match(e.Name) { 123 | h := h 124 | g.Go(func() error { 125 | return h.Handle(ctx, e) 126 | }) 127 | } 128 | } 129 | 130 | if err := g.Wait(); err != nil { 131 | err = xerrors.Errorf("imcompleted with error: %w", err) 132 | logger.Err(err).Msg(err.Error()) 133 | return err 134 | } 135 | 136 | return nil 137 | } 138 | 139 | /* 140 | severity log field is used as Cloud Logging severity 141 | See https://cloud.google.com/functions/docs/monitoring/logging#processing_special_json_fields_in_messages 142 | */ 143 | type severityHook struct{} 144 | 145 | func (h severityHook) Run(e *zerolog.Event, level zerolog.Level, msg string) { 146 | if level != zerolog.NoLevel { 147 | e.Str("severity", level.String()) 148 | } 149 | } 150 | 151 | func contextualLogger(ctx context.Context, e Event, l *zerolog.Logger) *zerolog.Logger { 152 | lctx := e.logger(l).With() 153 | 154 | t, ok := startedTimeFrom(ctx) 155 | if ok { 156 | lctx = lctx.Time("started", t) 157 | } 158 | 159 | md, err := metadata.FromContext(ctx) 160 | if err == nil { 161 | rd := zerolog.Dict(). 162 | Str("service", md.Resource.Service). 163 | Str("name", md.Resource.Name). 164 | Str("type", md.Resource.Type). 165 | Str("rawPath", md.Resource.RawPath) 166 | 167 | d := zerolog.Dict(). 168 | Str("eventId", md.EventID). 169 | Time("timestamp", md.Timestamp). 170 | Str("eventType", md.EventType). 171 | Dict("resource", rd) 172 | 173 | lctx = lctx.Dict("metadata", d) 174 | } else { 175 | l.Warn().Msg(err.Error()) 176 | } 177 | 178 | rl := lctx.Logger() 179 | return &rl 180 | } 181 | -------------------------------------------------------------------------------- /handler.go: -------------------------------------------------------------------------------- 1 | package bqloader 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | "sync" 7 | "time" 8 | 9 | "github.com/rs/zerolog" 10 | "github.com/rs/zerolog/log" 11 | "golang.org/x/sync/errgroup" 12 | "golang.org/x/text/encoding" 13 | "golang.org/x/text/transform" 14 | "golang.org/x/xerrors" 15 | ) 16 | 17 | const defaultBatchSize = 10000 18 | 19 | // Handler defines how to handle events which match specified pattern. 20 | type Handler struct { 21 | // Name is the handler's name. 22 | Name string 23 | 24 | Pattern *regexp.Regexp 25 | Encoding encoding.Encoding 26 | Parser Parser 27 | Notifier Notifier 28 | Projector Projector 29 | SkipLeadingRows uint 30 | Preprocessor Preprocessor 31 | 32 | // BatchSize specifies how much records are processed in a groutine. 33 | // Default is 10000. 34 | BatchSize int 35 | 36 | // Project specifies GCP project name of destination BigQuery table. 37 | Project string 38 | 39 | // Dataset specifies BigQuery dataset ID of destination table 40 | Dataset string 41 | 42 | // Table specifies BigQuery table ID as destination. 43 | Table string 44 | 45 | Extractor Extractor 46 | Loader Loader 47 | semaphore chan struct{} 48 | } 49 | 50 | // Projector transforms source records into records for destination. 51 | type Projector func(context.Context, []string) ([]string, error) 52 | 53 | // Preprocessor preprocesses event and store data into a map. 54 | type Preprocessor func(context.Context, Event) (context.Context, error) 55 | 56 | func (h *Handler) match(name string) bool { 57 | return h.Pattern != nil && h.Pattern.MatchString(name) 58 | } 59 | 60 | // SetConcurrency sets handler's concurrency directly. 61 | // Normally set concurrency to BQLoader with WithConcurrency option. 62 | func (h *Handler) SetConcurrency(n int) { 63 | h.semaphore = make(chan struct{}, n) 64 | } 65 | 66 | // Handle handles events. 67 | func (h *Handler) Handle(ctx context.Context, e Event) error { 68 | ctx = withHandlerStartedTime(ctx) 69 | l := log.Ctx(ctx) 70 | l = h.logger(ctx, l) 71 | ctx = l.WithContext(ctx) 72 | 73 | l.Info().Msgf("handler %s started to handle an event", h.Name) 74 | defer func() { 75 | now := time.Now() 76 | e := l.Info().Time("handlerFinished", now) 77 | if t, ok := handlerStartedTimeFrom(ctx); ok { 78 | e.TimeDiff("handlerElapsed", now, t) 79 | } 80 | e.Msgf("handler %s finished to handle an event", h.Name) 81 | }() 82 | 83 | err := h.process(ctx, e) 84 | if err != nil { 85 | err = xerrors.Errorf("failed to handle: %w", err) 86 | l.Err(err).Msg(err.Error()) 87 | } 88 | 89 | if h.Notifier != nil { 90 | res := &Result{Event: e, Handler: h, Error: err} 91 | if nerr := h.Notifier.Notify(ctx, res); nerr != nil { 92 | nerr = xerrors.Errorf("failed to notify: %w", nerr) 93 | l.Err(nerr).Msg(nerr.Error()) 94 | } 95 | } 96 | 97 | return err 98 | } 99 | 100 | func (h *Handler) process(ctx context.Context, e Event) error { 101 | ctx, err := h.preprocess(ctx, e) 102 | if err != nil { 103 | return xerrors.Errorf("failed to preprocess: %w", err) 104 | } 105 | 106 | r, closer, err := h.Extractor.Extract(ctx, e) 107 | if err != nil { 108 | return xerrors.Errorf("failed to extract: %w", err) 109 | } 110 | defer closer() 111 | 112 | if h.Encoding != nil { 113 | r = transform.NewReader(r, h.Encoding.NewDecoder()) 114 | } 115 | 116 | source, err := h.Parser(ctx, r) 117 | if err != nil { 118 | return xerrors.Errorf("failed to parse: %w", err) 119 | } 120 | 121 | records, err := h.project(ctx, source[h.SkipLeadingRows:]) 122 | if err != nil { 123 | return xerrors.Errorf("failed to project: %w", err) 124 | } 125 | 126 | if err := h.Loader.Load(ctx, records); err != nil { 127 | return xerrors.Errorf("failed to load: %w", err) 128 | } 129 | 130 | return nil 131 | } 132 | 133 | func (h *Handler) preprocess(ctx context.Context, e Event) (context.Context, error) { 134 | if h.Preprocessor == nil { 135 | return ctx, nil 136 | } 137 | 138 | return h.Preprocessor(ctx, e) 139 | } 140 | 141 | func (h *Handler) project(ctx context.Context, source [][]string) ([][]string, error) { 142 | records := [][]string{} 143 | mu := sync.Mutex{} 144 | eg := errgroup.Group{} 145 | numBatches := h.calcBatches(len(source)) 146 | 147 | for i := 0; i < numBatches; i++ { 148 | startLine := h.BatchSize * i 149 | endLine := h.BatchSize * (i + 1) 150 | if endLine > len(source) { 151 | endLine = len(source) 152 | } 153 | 154 | h.semaphore <- struct{}{} 155 | 156 | eg.Go(func() error { 157 | defer func() { <-h.semaphore }() 158 | 159 | batchRecords := [][]string{} 160 | 161 | for j := startLine; j < endLine; j++ { 162 | record, err := h.Projector(ctx, source[j]) 163 | if err != nil { 164 | return xerrors.Errorf("failed to project row %d (line %d): %w", j, uint(j)+h.SkipLeadingRows, err) 165 | } 166 | 167 | if record != nil { 168 | batchRecords = append(batchRecords, record) 169 | } 170 | } 171 | 172 | mu.Lock() 173 | defer mu.Unlock() 174 | 175 | records = append(records, batchRecords...) 176 | 177 | return nil 178 | }) 179 | } 180 | 181 | if err := eg.Wait(); err != nil { 182 | return nil, xerrors.Errorf("failed to wait errgroup: %w", err) 183 | } 184 | 185 | return records, nil 186 | } 187 | 188 | func (h *Handler) logger(ctx context.Context, l *zerolog.Logger) *zerolog.Logger { 189 | lctx := l.With() 190 | 191 | if t, ok := handlerStartedTimeFrom(ctx); ok { 192 | lctx = lctx.Time("handlerStarted", t) 193 | } 194 | 195 | d := zerolog.Dict(). 196 | Str("name", h.Name). 197 | Uint("skipLeadingRows", h.SkipLeadingRows). 198 | Str("project", h.Project). 199 | Str("dataset", h.Dataset). 200 | Str("table", h.Table) 201 | 202 | if h.Pattern != nil { 203 | d = d.Str("pattern", h.Pattern.String()) 204 | } 205 | 206 | logger := lctx.Dict("handler", d).Logger() 207 | 208 | return &logger 209 | } 210 | 211 | func (h *Handler) calcBatches(length int) int { 212 | r := length / h.BatchSize 213 | if length%h.BatchSize != 0 { 214 | r++ 215 | } 216 | return r 217 | } 218 | -------------------------------------------------------------------------------- /examples/preprocessor/go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 4 | cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= 5 | cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= 6 | cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= 7 | cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= 8 | cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= 9 | cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= 10 | cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= 11 | cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= 12 | cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= 13 | cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= 14 | cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= 15 | cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= 16 | cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= 17 | cloud.google.com/go v0.68.0 h1:AnVkaPGAuWaIY/8a75HlNzZNrHDee6YL4rWkwS+CeyE= 18 | cloud.google.com/go v0.68.0/go.mod h1:91NO4SCDjUfe1zeC0f4/dpckkUNpuNEyqm4X2KLrzNQ= 19 | cloud.google.com/go v0.72.0 h1:eWRCuwubtDrCJG0oSUMgnsbD4CmPFQF2ei4OFbXvwww= 20 | cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= 21 | cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= 22 | cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= 23 | cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= 24 | cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= 25 | cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= 26 | cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= 27 | cloud.google.com/go/bigquery v1.12.0 h1:f6ZZ0IeEp/f6t0HewQb7dA+cj99VW3HOJSJbkWaESIw= 28 | cloud.google.com/go/bigquery v1.12.0/go.mod h1:qg6pXHplLataesM6qKANFGtuxgULUukphxDh7bor+9A= 29 | cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= 30 | cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= 31 | cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= 32 | cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= 33 | cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= 34 | cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= 35 | cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= 36 | cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= 37 | cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= 38 | cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= 39 | cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= 40 | cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4= 41 | cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= 42 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 43 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 44 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 45 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 46 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 47 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 48 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 49 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 50 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 51 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 52 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 53 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 54 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 55 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 56 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 57 | github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= 58 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 59 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 60 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 61 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 62 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 63 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= 64 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 65 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 66 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 67 | github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= 68 | github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 69 | github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 70 | github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 71 | github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= 72 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 73 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 74 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 75 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 76 | github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 77 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= 78 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 79 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 80 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 81 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 82 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 83 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 84 | github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= 85 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 86 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= 87 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 88 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 89 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 90 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 91 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 92 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 93 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 94 | github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 95 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 96 | github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 97 | github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= 98 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 99 | github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= 100 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 101 | github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs= 102 | github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= 103 | github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60= 104 | github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= 105 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 106 | github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 107 | github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 108 | github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 109 | github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 110 | github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 111 | github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 112 | github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 113 | github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= 114 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 115 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 116 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 117 | github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= 118 | github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= 119 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 120 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 121 | github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 122 | github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 123 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 124 | github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= 125 | github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= 126 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 127 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 128 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 129 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 130 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 131 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 132 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 133 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 134 | github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= 135 | github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs= 136 | github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= 137 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 138 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 139 | github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 140 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 141 | github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 142 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 143 | go.nownabe.dev/bqloader v1.2.0 h1:+ofEmIyYt+NNab/R0tANnWKkrkPol7GtrDWrvoIhDsc= 144 | go.nownabe.dev/bqloader v1.2.0/go.mod h1:W6+P3zMw7fTqnFvCfSW1K4krVAxDn2Ex0XEx96Nfxso= 145 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 146 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 147 | go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 148 | go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 149 | go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= 150 | go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 151 | go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= 152 | go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= 153 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 154 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 155 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 156 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 157 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 158 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 159 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 160 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 161 | golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= 162 | golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= 163 | golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 164 | golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 165 | golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 166 | golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= 167 | golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= 168 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 169 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 170 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 171 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 172 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 173 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 174 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 175 | golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 176 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 177 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= 178 | golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 179 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= 180 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 181 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 182 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 183 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 184 | golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= 185 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 186 | golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 187 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 188 | golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= 189 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 190 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 191 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 192 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 193 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 194 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 195 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 196 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 197 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 198 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 199 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 200 | golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 201 | golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 202 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 203 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 204 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 205 | golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 206 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 207 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 208 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 209 | golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 210 | golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 211 | golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 212 | golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 213 | golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 214 | golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 215 | golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 216 | golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 217 | golang.org/x/net v0.0.0-20200927032502-5d4f70055728 h1:5wtQIAulKU5AbLQOkjxl32UufnIOqgBX72pS0AV14H0= 218 | golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 219 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 220 | golang.org/x/net v0.0.0-20201031054903-ff519b6c9102 h1:42cLlJJdEh+ySyeUUbEQ5bsTiq8voBeTuweGVkY6Puw= 221 | golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 222 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 223 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 224 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 225 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 226 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 227 | golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc= 228 | golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= 229 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 230 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 231 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 232 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 233 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 234 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 235 | golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 236 | golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 237 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= 238 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 239 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 240 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 241 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 242 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 243 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 244 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 245 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 246 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 247 | golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 248 | golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 249 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 250 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 251 | golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 252 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 253 | golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 254 | golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 255 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 256 | golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 257 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 258 | golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 259 | golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 260 | golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 261 | golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 262 | golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 263 | golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 264 | golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 265 | golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc= 266 | golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 267 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= 268 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 269 | golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 270 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 271 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 272 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 273 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 274 | golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= 275 | golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 276 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 277 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 278 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 279 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 280 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 281 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 282 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 283 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 284 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 285 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 286 | golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 287 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 288 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 289 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 290 | golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 291 | golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 292 | golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 293 | golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 294 | golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 295 | golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 296 | golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 297 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 298 | golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 299 | golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 300 | golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 301 | golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 302 | golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 303 | golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 304 | golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 305 | golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 306 | golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 307 | golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 308 | golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 309 | golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 310 | golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= 311 | golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= 312 | golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= 313 | golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 314 | golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 315 | golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 316 | golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 317 | golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= 318 | golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= 319 | golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= 320 | golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= 321 | golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= 322 | golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= 323 | golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= 324 | golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= 325 | golang.org/x/tools v0.0.0-20201011145850-ed2f50202694 h1:BANdcOVw3KTuUiyfDp7wrzCpkCe8UP3lowugJngxBTg= 326 | golang.org/x/tools v0.0.0-20201011145850-ed2f50202694/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= 327 | golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd h1:kJP9fbfkpUoA4y03Nxor8be+YbShcXP16fc7G4nlgpw= 328 | golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 329 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 330 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 331 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 332 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 333 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 334 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 335 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= 336 | google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 337 | google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 338 | google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 339 | google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 340 | google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 341 | google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 342 | google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 343 | google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 344 | google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 345 | google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 346 | google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= 347 | google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= 348 | google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= 349 | google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= 350 | google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= 351 | google.golang.org/api v0.32.0 h1:Le77IccnTqEa8ryp9wIpX5W3zYm7Gf9LhOp9PHcwFts= 352 | google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= 353 | google.golang.org/api v0.35.0 h1:TBCmTTxUrRDA1iTctnK/fIeitxIZ+TQuaf0j29fmCGo= 354 | google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= 355 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 356 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 357 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 358 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 359 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 360 | google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= 361 | google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 362 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 363 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 364 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 365 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 366 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 367 | google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 368 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 369 | google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= 370 | google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 371 | google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 372 | google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 373 | google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 374 | google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 375 | google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 376 | google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= 377 | google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 378 | google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 379 | google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 380 | google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 381 | google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 382 | google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 383 | google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 384 | google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 385 | google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= 386 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 387 | google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= 388 | google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 389 | google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 390 | google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 391 | google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 392 | google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 393 | google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 394 | google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 395 | google.golang.org/genproto v0.0.0-20201002142447-3860012362da/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 396 | google.golang.org/genproto v0.0.0-20201009135657-4d944d34d83c h1:YhtyeaRgEDy6amoaijBY0B6uzaztIKKZbUcaO2OIFps= 397 | google.golang.org/genproto v0.0.0-20201009135657-4d944d34d83c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 398 | google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb h1:MoNcrN5yaH+35Ge8RUwFbL7ekwq9ED2fiDpgWKrR29w= 399 | google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 400 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 401 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 402 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 403 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 404 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 405 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 406 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 407 | google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 408 | google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= 409 | google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= 410 | google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 411 | google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 412 | google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 413 | google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0= 414 | google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 415 | google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= 416 | google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= 417 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 418 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 419 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 420 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 421 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 422 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 423 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 424 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 425 | google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= 426 | google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= 427 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 428 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 429 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 430 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 431 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 432 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 433 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 434 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 435 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 436 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 437 | honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= 438 | honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= 439 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 440 | rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= 441 | rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= 442 | -------------------------------------------------------------------------------- /examples/quickstart/go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 4 | cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= 5 | cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= 6 | cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= 7 | cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= 8 | cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= 9 | cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= 10 | cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= 11 | cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= 12 | cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= 13 | cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= 14 | cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= 15 | cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= 16 | cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= 17 | cloud.google.com/go v0.68.0 h1:AnVkaPGAuWaIY/8a75HlNzZNrHDee6YL4rWkwS+CeyE= 18 | cloud.google.com/go v0.68.0/go.mod h1:91NO4SCDjUfe1zeC0f4/dpckkUNpuNEyqm4X2KLrzNQ= 19 | cloud.google.com/go v0.72.0 h1:eWRCuwubtDrCJG0oSUMgnsbD4CmPFQF2ei4OFbXvwww= 20 | cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= 21 | cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= 22 | cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= 23 | cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= 24 | cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= 25 | cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= 26 | cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= 27 | cloud.google.com/go/bigquery v1.12.0 h1:f6ZZ0IeEp/f6t0HewQb7dA+cj99VW3HOJSJbkWaESIw= 28 | cloud.google.com/go/bigquery v1.12.0/go.mod h1:qg6pXHplLataesM6qKANFGtuxgULUukphxDh7bor+9A= 29 | cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= 30 | cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= 31 | cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= 32 | cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= 33 | cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= 34 | cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= 35 | cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= 36 | cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= 37 | cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= 38 | cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= 39 | cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= 40 | cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4= 41 | cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= 42 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 43 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 44 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 45 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 46 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 47 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 48 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 49 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 50 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 51 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 52 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 53 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 54 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 55 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 56 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 57 | github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= 58 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 59 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 60 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 61 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 62 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 63 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= 64 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 65 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 66 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 67 | github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= 68 | github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 69 | github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 70 | github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 71 | github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= 72 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 73 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 74 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 75 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 76 | github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 77 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= 78 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 79 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 80 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 81 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 82 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 83 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 84 | github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= 85 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 86 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= 87 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 88 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 89 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 90 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 91 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 92 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 93 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 94 | github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 95 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 96 | github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 97 | github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= 98 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 99 | github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= 100 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 101 | github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs= 102 | github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= 103 | github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60= 104 | github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= 105 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 106 | github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 107 | github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 108 | github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 109 | github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 110 | github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 111 | github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 112 | github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 113 | github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= 114 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 115 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 116 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 117 | github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= 118 | github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= 119 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 120 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 121 | github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 122 | github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 123 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 124 | github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= 125 | github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= 126 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 127 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 128 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 129 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 130 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 131 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 132 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 133 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 134 | github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= 135 | github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs= 136 | github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= 137 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 138 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 139 | github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 140 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 141 | github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 142 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 143 | go.nownabe.dev/bqloader v1.2.0 h1:+ofEmIyYt+NNab/R0tANnWKkrkPol7GtrDWrvoIhDsc= 144 | go.nownabe.dev/bqloader v1.2.0/go.mod h1:W6+P3zMw7fTqnFvCfSW1K4krVAxDn2Ex0XEx96Nfxso= 145 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 146 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 147 | go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 148 | go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 149 | go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= 150 | go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 151 | go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= 152 | go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= 153 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 154 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 155 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 156 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 157 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 158 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 159 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 160 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 161 | golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= 162 | golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= 163 | golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 164 | golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 165 | golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 166 | golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= 167 | golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= 168 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 169 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 170 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 171 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 172 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 173 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 174 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 175 | golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 176 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 177 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= 178 | golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 179 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= 180 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 181 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 182 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 183 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 184 | golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= 185 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 186 | golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 187 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 188 | golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= 189 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 190 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 191 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 192 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 193 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 194 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 195 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 196 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 197 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 198 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 199 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 200 | golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 201 | golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 202 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 203 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 204 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 205 | golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 206 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 207 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 208 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 209 | golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 210 | golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 211 | golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 212 | golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 213 | golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 214 | golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 215 | golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 216 | golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 217 | golang.org/x/net v0.0.0-20200927032502-5d4f70055728 h1:5wtQIAulKU5AbLQOkjxl32UufnIOqgBX72pS0AV14H0= 218 | golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 219 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 220 | golang.org/x/net v0.0.0-20201031054903-ff519b6c9102 h1:42cLlJJdEh+ySyeUUbEQ5bsTiq8voBeTuweGVkY6Puw= 221 | golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 222 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 223 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 224 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 225 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 226 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 227 | golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc= 228 | golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= 229 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 230 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 231 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 232 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 233 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 234 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 235 | golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 236 | golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 237 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= 238 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 239 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 240 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 241 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 242 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 243 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 244 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 245 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 246 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 247 | golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 248 | golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 249 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 250 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 251 | golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 252 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 253 | golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 254 | golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 255 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 256 | golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 257 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 258 | golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 259 | golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 260 | golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 261 | golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 262 | golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 263 | golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 264 | golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 265 | golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc= 266 | golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 267 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= 268 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 269 | golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 270 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 271 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 272 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 273 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 274 | golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= 275 | golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 276 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 277 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 278 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 279 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 280 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 281 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 282 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 283 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 284 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 285 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 286 | golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 287 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 288 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 289 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 290 | golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 291 | golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 292 | golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 293 | golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 294 | golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 295 | golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 296 | golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 297 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 298 | golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 299 | golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 300 | golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 301 | golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 302 | golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 303 | golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 304 | golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 305 | golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 306 | golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 307 | golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 308 | golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 309 | golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 310 | golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= 311 | golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= 312 | golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= 313 | golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 314 | golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 315 | golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 316 | golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 317 | golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= 318 | golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= 319 | golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= 320 | golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= 321 | golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= 322 | golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= 323 | golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= 324 | golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= 325 | golang.org/x/tools v0.0.0-20201011145850-ed2f50202694 h1:BANdcOVw3KTuUiyfDp7wrzCpkCe8UP3lowugJngxBTg= 326 | golang.org/x/tools v0.0.0-20201011145850-ed2f50202694/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= 327 | golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd h1:kJP9fbfkpUoA4y03Nxor8be+YbShcXP16fc7G4nlgpw= 328 | golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 329 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 330 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 331 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 332 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 333 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 334 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 335 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= 336 | google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 337 | google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 338 | google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 339 | google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 340 | google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 341 | google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 342 | google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 343 | google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 344 | google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 345 | google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 346 | google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= 347 | google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= 348 | google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= 349 | google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= 350 | google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= 351 | google.golang.org/api v0.32.0 h1:Le77IccnTqEa8ryp9wIpX5W3zYm7Gf9LhOp9PHcwFts= 352 | google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= 353 | google.golang.org/api v0.35.0 h1:TBCmTTxUrRDA1iTctnK/fIeitxIZ+TQuaf0j29fmCGo= 354 | google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= 355 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 356 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 357 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 358 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 359 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 360 | google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= 361 | google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 362 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 363 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 364 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 365 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 366 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 367 | google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 368 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 369 | google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= 370 | google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 371 | google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 372 | google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 373 | google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 374 | google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 375 | google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 376 | google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= 377 | google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 378 | google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 379 | google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 380 | google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 381 | google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 382 | google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 383 | google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 384 | google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 385 | google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= 386 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 387 | google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= 388 | google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 389 | google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 390 | google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 391 | google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 392 | google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 393 | google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 394 | google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 395 | google.golang.org/genproto v0.0.0-20201002142447-3860012362da/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 396 | google.golang.org/genproto v0.0.0-20201009135657-4d944d34d83c h1:YhtyeaRgEDy6amoaijBY0B6uzaztIKKZbUcaO2OIFps= 397 | google.golang.org/genproto v0.0.0-20201009135657-4d944d34d83c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 398 | google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb h1:MoNcrN5yaH+35Ge8RUwFbL7ekwq9ED2fiDpgWKrR29w= 399 | google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 400 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 401 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 402 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 403 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 404 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 405 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 406 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 407 | google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 408 | google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= 409 | google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= 410 | google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 411 | google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 412 | google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 413 | google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0= 414 | google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 415 | google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= 416 | google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= 417 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 418 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 419 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 420 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 421 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 422 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 423 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 424 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 425 | google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= 426 | google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= 427 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 428 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 429 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 430 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 431 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 432 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 433 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 434 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 435 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 436 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 437 | honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= 438 | honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= 439 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 440 | rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= 441 | rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= 442 | --------------------------------------------------------------------------------