├── test ├── ls_tmp │ └── .gitignore ├── testevent.json ├── testeventsns.json ├── localstack.yml ├── config.yml.no_key_matcher ├── config.yml.invalid_regexp ├── config.redshift-data.yml ├── config.iam_role.yml ├── event.json ├── config.yml └── notification.json ├── cmd └── rin │ ├── .gitignore │ └── main.go ├── Dockerfile ├── .gitignore ├── docker-compose.yaml ├── .github └── workflows │ ├── test.yml │ └── release.yml ├── .goreleaser.yml ├── Makefile ├── LICENSE ├── lambda.go ├── go.mod ├── event.go ├── event_test.go ├── local_test.go ├── redshift.go ├── README.md ├── config_test.go ├── go.sum ├── rin.go └── config.go /test/ls_tmp/.gitignore: -------------------------------------------------------------------------------- 1 | * -------------------------------------------------------------------------------- /cmd/rin/.gitignore: -------------------------------------------------------------------------------- 1 | rin 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.15.0 2 | LABEL maintainer "fujiwara " 3 | 4 | RUN apk --no-cache add ca-certificates 5 | COPY dist/Rin_linux_amd64_v1/rin /usr/local/bin/rin 6 | WORKDIR / 7 | ENTRYPOINT ["/usr/local/bin/rin"] 8 | -------------------------------------------------------------------------------- /test/testevent.json: -------------------------------------------------------------------------------- 1 | { 2 | "Service": "Amazon S3", 3 | "Event": "s3:TestEvent", 4 | "Time": "2021-03-30T05:58:18.184Z", 5 | "Bucket": "example-bucket", 6 | "RequestId": "C0CMH2QNHV3A1SGQ", 7 | "HostId": "/9JSRrT+j26Hk56OjL/qeDZM76Otmaj3oY9hnTuD/kdyN+WN6vDli3q0LPJzUMRW5y/yVmTOlzQ=" 8 | } 9 | -------------------------------------------------------------------------------- /test/testeventsns.json: -------------------------------------------------------------------------------- 1 | { 2 | "Message": "{\n \"Service\": \"Amazon S3\",\n \"Event\": \"s3:TestEvent\",\n \"Time\": \"2021-03-30T05:58:18.184Z\",\n \"Bucket\": \"example-bucket\",\n \"RequestId\": \"C0CMH2QNHV3A1SGQ\",\n \"HostId\": \"/9JSRrT+j26Hk56OjL/qeDZM76Otmaj3oY9hnTuD/kdyN+WN6vDli3q0LPJzUMRW5y/yVmTOlzQ=\"\n}\n" 3 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | *~ 27 | pkg/* 28 | dist/* 29 | -------------------------------------------------------------------------------- /test/localstack.yml: -------------------------------------------------------------------------------- 1 | queue_name: rin_test 2 | 3 | credentials: 4 | aws_region: ap-northeast-1 5 | 6 | s3: 7 | bucket: rin-test 8 | region: ap-northeast-1 9 | 10 | sql_option: "JSON 'auto' GZIP" 11 | 12 | redshift: 13 | host: localhost 14 | port: 4577 15 | dbname: test 16 | user: root 17 | password: toor 18 | 19 | targets: 20 | - redshift: 21 | table: foo 22 | s3: 23 | key_prefix: test_xxx/foo 24 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | localstack: 5 | image: localstack/localstack 6 | ports: 7 | - "4566:4566" 8 | - "${PORT_WEB_UI-8080}:${PORT_WEB_UI-8080}" 9 | environment: 10 | - SERVICES=s3,sqs,redshift 11 | - DEBUG=${DEBUG- } 12 | - DATA_DIR=${DATA_DIR- } 13 | - PORT_WEB_UI=${PORT_WEB_UI- } 14 | - DOCKER_HOST=unix:///var/run/docker.sock 15 | volumes: 16 | - "${PWD}/test/ls_tmp:/tmp/localstack" 17 | -------------------------------------------------------------------------------- /test/config.yml.no_key_matcher: -------------------------------------------------------------------------------- 1 | queue_name: rin_test 2 | 3 | credentials: 4 | aws_access_key_id: AAA 5 | aws_secret_access_key: SSS 6 | aws_region: ap-northeast-1 7 | 8 | s3: 9 | bucket: test.bucket.test 10 | region: ap-northeast-1 11 | 12 | sql_option: "JSON 'auto' GZIP" 13 | 14 | redshift: 15 | host: localhost 16 | port: 5432 17 | dbname: test 18 | user: test_user 19 | password: test_pass 20 | 21 | targets: 22 | - redshift: 23 | schema: $1 24 | table: $2 25 | s3: 26 | bucket: example.bucket 27 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | on: [push] 3 | jobs: 4 | test: 5 | strategy: 6 | matrix: 7 | go: 8 | - 1.18 9 | - 1.19 10 | name: Build 11 | runs-on: ubuntu-20.04 12 | steps: 13 | - name: Set up Go 14 | uses: actions/setup-go@v3 15 | with: 16 | go-version: ${{ matrix.go }} 17 | id: go 18 | 19 | - name: Check out code into the Go module directory 20 | uses: actions/checkout@v3 21 | 22 | - name: Build & Test 23 | run: | 24 | make test 25 | -------------------------------------------------------------------------------- /test/config.yml.invalid_regexp: -------------------------------------------------------------------------------- 1 | queue_name: rin_test 2 | 3 | credentials: 4 | aws_access_key_id: AAA 5 | aws_secret_access_key: SSS 6 | aws_region: ap-northeast-1 7 | 8 | s3: 9 | bucket: test.bucket.test 10 | region: ap-northeast-1 11 | 12 | sql_option: "JSON 'auto' GZIP" 13 | 14 | redshift: 15 | host: localhost 16 | port: 5432 17 | dbname: test 18 | user: test_user 19 | password: test_pass 20 | 21 | targets: 22 | - redshift: 23 | schema: $1 24 | table: $2 25 | s3: 26 | bucket: example.bucket 27 | key_regexp: test/(s[0-9]+)/(t[0-9]+/ 28 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # This is an example goreleaser.yaml file with some sane defaults. 2 | # Make sure to check the documentation at http://goreleaser.com 3 | before: 4 | hooks: 5 | - go mod download 6 | builds: 7 | - env: 8 | - CGO_ENABLED=0 9 | main: ./cmd/rin/main.go 10 | binary: rin 11 | goos: 12 | - linux 13 | - darwin 14 | goarch: 15 | - arm64 16 | - amd64 17 | archives: 18 | checksum: 19 | name_template: "checksums.txt" 20 | snapshot: 21 | name_template: "{{ .Tag }}-next" 22 | changelog: 23 | sort: asc 24 | filters: 25 | exclude: 26 | - "^docs:" 27 | - "^test:" 28 | release: 29 | prerelease: "true" 30 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | branches: 6 | - "!**/*" 7 | tags: 8 | - "v*" 9 | 10 | permissions: write-all 11 | 12 | jobs: 13 | goreleaser: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v3 18 | - name: Set up Go 19 | uses: actions/setup-go@v3 20 | with: 21 | go-version: 1.19 22 | - name: Run GoReleaser 23 | uses: goreleaser/goreleaser-action@v3 24 | with: 25 | version: latest 26 | args: release --rm-dist 27 | env: 28 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 29 | - name: Release Image 30 | run: | 31 | echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin 32 | make release-image 33 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GIT_VER := $(shell git describe --tags) 2 | DATE := $(shell date +%Y-%m-%dT%H:%M:%S%z) 3 | export GO111MODULE := on 4 | 5 | .PHONY: test test-localstack install clean image release-image 6 | 7 | cmd/rin/rin: *.go cmd/rin/main.go 8 | cd cmd/rin && go build -ldflags "-s -w -X main.version=${GIT_VER} -X main.buildDate=${DATE}" 9 | 10 | install: cmd/rin/rin 11 | install cmd/rin/rin ${GOPATH}/bin 12 | 13 | test-localstack: 14 | docker-compose up -d 15 | TEST_LOCALSTACK=on dockerize -timeout 30s -wait tcp://localhost:4566 go test -v -run Local ./... 16 | 17 | test: 18 | go test -v ./... 19 | 20 | dist/: 21 | goreleaser build --snapshot --rm-dist 22 | 23 | clean: 24 | rm -rf cmd/rin/rin pkg/* test/ls_tmp/* dist/ 25 | 26 | image: dist/ 27 | docker build \ 28 | --tag ghcr.io/fujiwara/rin:$(GIT_VER) \ 29 | . 30 | 31 | release-image: image 32 | docker push ghcr.io/fujiwara/rin:$(GIT_VER) 33 | -------------------------------------------------------------------------------- /test/config.redshift-data.yml: -------------------------------------------------------------------------------- 1 | queue_name: rin_test 2 | 3 | credentials: 4 | aws_region: ap-northeast-1 5 | aws_iam_role: "arn:aws:iam::123456789012:role/rin" 6 | 7 | s3: 8 | bucket: test.bucket.test 9 | region: ap-northeast-1 10 | 11 | sql_option: "JSON 'auto' GZIP" 12 | 13 | redshift: 14 | driver: redshift-data 15 | cluster: mycluster 16 | dbname: test 17 | user: test_user 18 | reconnect_on_error: true 19 | 20 | targets: 21 | - s3: 22 | key_prefix: test/foo/discard 23 | discard: true 24 | 25 | - redshift: 26 | table: foo 27 | s3: 28 | key_prefix: test/foo 29 | 30 | - redshift: 31 | schema: xxx 32 | table: bar_break 33 | s3: 34 | key_prefix: test/bar/break 35 | sql_option: "CSV DELIMITER ',' ESCAPE" 36 | break: true 37 | 38 | - redshift: 39 | schema: xxx 40 | table: bar 41 | s3: 42 | key_prefix: test/bar 43 | sql_option: "CSV DELIMITER ',' ESCAPE" 44 | 45 | - redshift: 46 | schema: $1 47 | table: $2 48 | s3: 49 | bucket: example.bucket 50 | key_regexp: test/(s[0-9]+)/(t[0-9]+)/ 51 | -------------------------------------------------------------------------------- /test/config.iam_role.yml: -------------------------------------------------------------------------------- 1 | queue_name: rin_test 2 | 3 | credentials: 4 | aws_region: ap-northeast-1 5 | aws_iam_role: "arn:aws:iam::123456789012:role/rin" 6 | 7 | s3: 8 | bucket: test.bucket.test 9 | region: ap-northeast-1 10 | 11 | sql_option: "JSON 'auto' GZIP" 12 | 13 | redshift: 14 | host: localhost 15 | port: 5432 16 | dbname: test 17 | user: test_user 18 | password: test_pass 19 | reconnect_on_error: true 20 | 21 | targets: 22 | - s3: 23 | key_prefix: test/foo/discard 24 | discard: true 25 | 26 | - redshift: 27 | table: foo 28 | s3: 29 | key_prefix: test/foo 30 | 31 | - redshift: 32 | schema: xxx 33 | table: bar_break 34 | s3: 35 | key_prefix: test/bar/break 36 | sql_option: "CSV DELIMITER ',' ESCAPE" 37 | break: true 38 | 39 | - redshift: 40 | schema: xxx 41 | table: bar 42 | s3: 43 | key_prefix: test/bar 44 | sql_option: "CSV DELIMITER ',' ESCAPE" 45 | 46 | - redshift: 47 | schema: $1 48 | table: $2 49 | s3: 50 | bucket: example.bucket 51 | key_regexp: test/(s[0-9]+)/(t[0-9]+)/ 52 | -------------------------------------------------------------------------------- /test/event.json: -------------------------------------------------------------------------------- 1 | { 2 | "Records": [ 3 | { 4 | "eventVersion": "2.0", 5 | "eventSource": "aws:s3", 6 | "awsRegion": "ap-northeast-1", 7 | "eventTime": "2015-04-21T04:55:48.282Z", 8 | "eventName": "ObjectCreated:Put", 9 | "userIdentity": { 10 | "principalId": "AWS:AIDAITB24YMP65EXRRFHC" 11 | }, 12 | "requestParameters": { 13 | "sourceIPAddress": "10.115.144.24" 14 | }, 15 | "responseElements": { 16 | "x-amz-request-id": "C223B09A2672B58C", 17 | "x-amz-id-2": "lwnmR96s31UoVCw5ozvg+jV+heZKoheJ+KBoWinmnfl1RzxVUn48R+Baha1vUyW0" 18 | }, 19 | "s3": { 20 | "s3SchemaVersion": "1.0", 21 | "configurationId": "test", 22 | "bucket": { 23 | "name": "test.bucket.test", 24 | "ownerIdentity": { 25 | "principalId": "A3RIPTMLB7ZZQI" 26 | }, 27 | "arn": "arn:aws:s3:::test.bucket.test" 28 | }, 29 | "object": { 30 | "key": "foo/bar.json", 31 | "size": 443, 32 | "eTag": "86fcdfb65af50a994cf63ddd280cea0d" 33 | } 34 | } 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 FUJIWARA Shunichiro 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /test/config.yml: -------------------------------------------------------------------------------- 1 | queue_name: rin_test 2 | 3 | credentials: 4 | aws_access_key_id: AAA 5 | aws_secret_access_key: '{{ must_env "AWS_SECRET_ACCESS_KEY" }}' 6 | aws_region: ap-northeast-1 7 | 8 | s3: 9 | bucket: test.bucket.test 10 | region: ap-northeast-1 11 | 12 | sql_option: "JSON 'auto' GZIP" 13 | 14 | redshift: 15 | host: localhost 16 | port: 5432 17 | dbname: test 18 | user: test_user 19 | password: test_pass 20 | reconnect_on_error: true 21 | 22 | targets: 23 | - s3: 24 | key_prefix: test/foo/discard 25 | discard: true 26 | 27 | - redshift: 28 | table: foo 29 | s3: 30 | key_prefix: test/foo 31 | 32 | - redshift: 33 | schema: xxx 34 | table: bar_break 35 | s3: 36 | key_prefix: test/bar/break 37 | sql_option: "CSV DELIMITER ',' ESCAPE" 38 | break: true 39 | 40 | - redshift: 41 | schema: xxx 42 | table: bar 43 | s3: 44 | key_prefix: test/bar 45 | sql_option: "CSV DELIMITER ',' ESCAPE" 46 | break: true 47 | 48 | - redshift: 49 | schema: $1 50 | table: $2 51 | s3: 52 | bucket: example.bucket 53 | key_regexp: test/(s[0-9]+)/(t[0-9]+)/ 54 | -------------------------------------------------------------------------------- /test/notification.json: -------------------------------------------------------------------------------- 1 | { 2 | "Type" : "Notification", 3 | "MessageId" : "22b80b92-fdea-4c2c-8f9d-bdfb0c7bf324", 4 | "TopicArn" : "arn:aws:sns:us-west-2:123456789012:MyTopic", 5 | "Subject" : "My First Message", 6 | "Message" : "{\"Records\":[{\"eventVersion\":\"2.0\",\"eventSource\":\"aws:s3\",\"awsRegion\":\"ap-northeast-1\",\"eventTime\":\"2015-04-21T04:55:48.282Z\",\"eventName\":\"ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"AWS:AIDAITB24YMP65EXRRFHC\"},\"requestParameters\":{\"sourceIPAddress\":\"10.115.144.24\"},\"responseElements\":{\"x-amz-request-id\":\"C223B09A2672B58C\",\"x-amz-id-2\":\"lwnmR96s31UoVCw5ozvg+jV+heZKoheJ+KBoWinmnfl1RzxVUn48R+Baha1vUyW0\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"test\",\"bucket\":{\"name\":\"test.bucket.test\",\"ownerIdentity\":{\"principalId\":\"A3RIPTMLB7ZZQI\"},\"arn\":\"arn:aws:s3:::test.bucket.test\"},\"object\":{\"key\":\"test/foo/bar.json\",\"size\":443,\"eTag\":\"86fcdfb65af50a994cf63ddd280cea0d\"}}}]}", 7 | "Timestamp" : "2012-05-02T00:54:06.655Z", 8 | "SignatureVersion" : "1", 9 | "Signature" : "EXAMPLEw6JRNwm1LFQL4ICB0bnXrdB8ClRMTQFGBqwLpGbM78tJ4etTwC5zU7O3tS6tGpey3ejedNdOJ+1fkIp9F2/LmNVKb5aFlYq+9rk9ZiPph5YlLmWsDcyC5T+Sy9/umic5S0UQc2PEtgdpVBahwNOdMW4JPwk0kAJJztnc=", 10 | "SigningCertURL" : "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-f3ecfb7224c7233fe7bb5f59f96de52f.pem", 11 | "UnsubscribeURL" : "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-west-2:123456789012:MyTopic:c9135db0-26c4-47ec-8998-413945fb5a96" 12 | } 13 | -------------------------------------------------------------------------------- /lambda.go: -------------------------------------------------------------------------------- 1 | package rin 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "log" 7 | "sync" 8 | 9 | "github.com/aws/aws-lambda-go/events" 10 | "github.com/aws/aws-lambda-go/lambda" 11 | ) 12 | 13 | type SQSBatchResponse struct { 14 | BatchItemFailures []BatchItemFailureItem `json:"batchItemFailures,omitempty"` 15 | } 16 | 17 | type BatchItemFailureItem struct { 18 | ItemIdentifier string `json:"itemIdentifier"` 19 | } 20 | 21 | func runLambdaHandler(opt *Option) error { 22 | if opt.BatchMode { 23 | log.Printf("[info] starting lambda handler SQS batch mode") 24 | lambda.Start(newLambdaSQSBatchHandler(opt)) 25 | } else { 26 | log.Printf("[info] starting lambda handler SQS event mode") 27 | lambda.Start(lambdaSQSEventHandler) 28 | } 29 | return nil 30 | } 31 | 32 | func lambdaSQSEventHandler(ctx context.Context, event *events.SQSEvent) (*SQSBatchResponse, error) { 33 | resp := &SQSBatchResponse{ 34 | BatchItemFailures: nil, 35 | } 36 | for _, record := range event.Records { 37 | if record.MessageId == "" { 38 | return nil, errors.New("sqs message id is empty") 39 | } 40 | if err := processEvent(ctx, record.MessageId, record.Body); err != nil { 41 | resp.BatchItemFailures = append(resp.BatchItemFailures, BatchItemFailureItem{ 42 | ItemIdentifier: record.MessageId, 43 | }) 44 | } 45 | } 46 | return resp, nil 47 | } 48 | 49 | func newLambdaSQSBatchHandler(opt *Option) func(ctx context.Context) error { 50 | return func(ctx context.Context) error { 51 | var wg sync.WaitGroup 52 | wg.Add(1) 53 | err := sqsWorker(ctx, &wg, opt) 54 | wg.Wait() 55 | if e, ok := err.(MaxExecutionTimeReachedError); ok { 56 | log.Printf("[info] %s", e.Error()) 57 | return nil 58 | } 59 | return err 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/fujiwara/Rin 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/aws/aws-lambda-go v1.34.1 7 | github.com/aws/aws-sdk-go-v2 v1.17.1 8 | github.com/aws/aws-sdk-go-v2/config v1.18.0 9 | github.com/aws/aws-sdk-go-v2/credentials v1.13.0 10 | github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.30 11 | github.com/aws/aws-sdk-go-v2/service/redshift v1.26.7 12 | github.com/aws/aws-sdk-go-v2/service/s3 v1.27.8 13 | github.com/aws/aws-sdk-go-v2/service/sqs v1.19.7 14 | github.com/hashicorp/logutils v1.0.0 15 | github.com/kayac/go-config v0.6.0 16 | github.com/lib/pq v1.10.6 17 | github.com/mashiike/redshift-data-sql-driver v0.1.0 18 | ) 19 | 20 | require ( 21 | github.com/BurntSushi/toml v1.2.0 // indirect 22 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.6 // indirect 23 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect 24 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect 25 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect 26 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect 27 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.11 // indirect 28 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.7 // indirect 29 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.15 // indirect 30 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect 31 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.14 // indirect 32 | github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.16.13 // indirect 33 | github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect 34 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect 35 | github.com/aws/aws-sdk-go-v2/service/sts v1.17.2 // indirect 36 | github.com/aws/smithy-go v1.13.4 // indirect 37 | github.com/jmespath/go-jmespath v0.4.0 // indirect 38 | github.com/pkg/errors v0.9.1 // indirect 39 | gopkg.in/yaml.v2 v2.4.0 // indirect 40 | ) 41 | -------------------------------------------------------------------------------- /cmd/rin/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "os" 8 | "strings" 9 | 10 | rin "github.com/fujiwara/Rin" 11 | "github.com/hashicorp/logutils" 12 | ) 13 | 14 | var ( 15 | version string 16 | buildDate string 17 | ) 18 | 19 | func main() { 20 | var ( 21 | config string 22 | showVersion bool 23 | debug bool 24 | dryRun bool 25 | ) 26 | opt := &rin.Option{} 27 | flag.StringVar(&config, "config", "config.yaml", "config file path") 28 | flag.StringVar(&config, "c", "config.yaml", "config file path") 29 | flag.BoolVar(&debug, "debug", false, "enable debug logging") 30 | flag.BoolVar(&debug, "d", false, "enable debug logging") 31 | flag.BoolVar(&showVersion, "version", false, "show version") 32 | flag.BoolVar(&showVersion, "v", false, "show version") 33 | flag.BoolVar(&opt.BatchMode, "batch", false, "batch mode") 34 | flag.BoolVar(&opt.BatchMode, "b", false, "batch mode") 35 | flag.DurationVar(&opt.MaxExecutionTime, "max-execution-time", 0, "max execution time") 36 | flag.BoolVar(&dryRun, "dry-run", false, "dry run mode (load configuration only)") 37 | flag.VisitAll(func(f *flag.Flag) { 38 | if len(f.Name) <= 1 { 39 | return 40 | } 41 | envName := strings.ToUpper("RIN_" + strings.Replace(f.Name, "-", "_", -1)) 42 | if s := os.Getenv(envName); s != "" { 43 | f.Value.Set(s) 44 | } 45 | }) 46 | flag.Parse() 47 | 48 | if showVersion { 49 | fmt.Println("version:", version) 50 | fmt.Println("build:", buildDate) 51 | return 52 | } 53 | 54 | minLevel := "info" 55 | if debug { 56 | minLevel = "debug" 57 | } 58 | filter := &logutils.LevelFilter{ 59 | Levels: []logutils.LogLevel{"debug", "info", "warn", "error"}, 60 | MinLevel: logutils.LogLevel(minLevel), 61 | Writer: os.Stderr, 62 | } 63 | log.SetOutput(filter) 64 | log.Println("[info] rin version:", version) 65 | log.Println("[info] option:", opt.String()) 66 | 67 | run := rin.Run 68 | if dryRun { 69 | run = rin.DryRun 70 | } 71 | if err := run(config, opt); err != nil { 72 | log.Println("[error]", err) 73 | os.Exit(1) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /event.go: -------------------------------------------------------------------------------- 1 | package rin 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/url" 7 | "strings" 8 | ) 9 | 10 | func ParseEvent(b []byte) (Event, error) { 11 | var e Event 12 | 13 | // If event comes to sqs through sns, we need to unmarshal the sns event first 14 | var snsE SnsEvent 15 | if err := json.Unmarshal(b, &snsE); err != nil { 16 | return e, err 17 | } 18 | if snsE.Message != nil { 19 | b = []byte(*snsE.Message) 20 | } 21 | 22 | // Unmarshall s3 event 23 | if err := json.Unmarshal(b, &e); err != nil { 24 | return e, err 25 | } 26 | if len(e.Records) == 0 && e.IsTestEvent() { 27 | return e, nil 28 | } 29 | for _, r := range e.Records { 30 | if !strings.Contains(r.S3.Object.Key, "%") { 31 | continue 32 | } 33 | if _key, err := url.QueryUnescape(r.S3.Object.Key); err == nil { 34 | r.S3.Object.Key = _key 35 | } 36 | } 37 | return e, nil 38 | } 39 | 40 | type SnsEvent struct { 41 | Message *string 42 | } 43 | 44 | type Event struct { 45 | Records []*EventRecord `json:"Records"` 46 | Event string 47 | Bucket string 48 | } 49 | 50 | func (e Event) IsTestEvent() bool { 51 | return e.Event == "s3:TestEvent" 52 | } 53 | 54 | func (e Event) String() string { 55 | if e.IsTestEvent() { 56 | return fmt.Sprintf("%s for %s", e.Event, e.Bucket) 57 | } 58 | 59 | s := make([]string, len(e.Records)) 60 | for i, r := range e.Records { 61 | s[i] = r.String() 62 | } 63 | return strings.Join(s, ", ") 64 | } 65 | 66 | type EventRecord struct { 67 | EventVersion string `json:"eventVersion"` 68 | EventName string `json:"eventName"` 69 | EventSource string `json:"eventSource"` 70 | EventTime string `json:"eventTime"` 71 | AWSRegion string `json:"awsRegion"` 72 | S3 S3Event `json:"s3"` 73 | } 74 | 75 | func (r EventRecord) String() string { 76 | return r.EventName + " " + fmt.Sprintf(S3URITemplate, r.S3.Bucket.Name, r.S3.Object.Key) 77 | } 78 | 79 | type S3Event struct { 80 | S3SchemaVersion string `json:"s3SchemaVersion"` 81 | ConfigurationID string `json:"configurationId"` 82 | Bucket S3Bucket `json:"bucket"` 83 | Object S3Object `json:"object"` 84 | } 85 | 86 | type S3Bucket struct { 87 | Name string `json:"name"` 88 | ARN string `json:"arn"` 89 | } 90 | 91 | type S3Object struct { 92 | Key string `json:"key"` 93 | Size int64 `json:"size"` 94 | ETag string `json:"eTag"` 95 | } 96 | -------------------------------------------------------------------------------- /event_test.go: -------------------------------------------------------------------------------- 1 | package rin_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | rin "github.com/fujiwara/Rin" 9 | ) 10 | 11 | var eventStr = `{ 12 | "Records": [ 13 | { 14 | "eventVersion": "2.0", 15 | "eventSource": "aws:s3", 16 | "awsRegion": "ap-northeast-1", 17 | "eventTime": "2015-04-21T04:55:48.282Z", 18 | "eventName": "ObjectCreated:Put", 19 | "userIdentity": { 20 | "principalId": "AWS:AIDAITB24YMP65EXRRFHC" 21 | }, 22 | "requestParameters": { 23 | "sourceIPAddress": "10.115.144.24" 24 | }, 25 | "responseElements": { 26 | "x-amz-request-id": "C223B09A2672B58C", 27 | "x-amz-id-2": "lwnmR96s31UoVCw5ozvg+jV+heZKoheJ+KBoWinmnfl1RzxVUn48R+Baha1vUyW0" 28 | }, 29 | "s3": { 30 | "s3SchemaVersion": "1.0", 31 | "configurationId": "test", 32 | "bucket": { 33 | "name": "test.bucket.test", 34 | "ownerIdentity": { 35 | "principalId": "A3RIPTMLB7ZZQI" 36 | }, 37 | "arn": "arn:aws:s3:::test.bucket.test" 38 | }, 39 | "object": { 40 | "key": "foo/bar%3Dbaz.json", 41 | "size": 443, 42 | "eTag": "86fcdfb65af50a994cf63ddd280cea0d" 43 | } 44 | } 45 | } 46 | ] 47 | }` 48 | 49 | func TestParseEvent(t *testing.T) { 50 | var event rin.Event 51 | event, err := rin.ParseEvent([]byte(eventStr)) 52 | if err != nil { 53 | t.Error("json decode error", err) 54 | } 55 | if event.IsTestEvent() { 56 | t.Error("must not be a test event") 57 | } 58 | r := event.Records[0] 59 | if r.EventName != "ObjectCreated:Put" { 60 | t.Error("unexpected EventName", r.EventName) 61 | } 62 | if r.EventSource != "aws:s3" { 63 | t.Error("unexpected EventSource", r.EventSource) 64 | } 65 | if r.AWSRegion != "ap-northeast-1" { 66 | t.Error("unexpected AWSRegion", r.AWSRegion) 67 | } 68 | if r.S3.Bucket.Name != "test.bucket.test" { 69 | t.Error("unexpected bucket name", r.S3.Bucket.Name) 70 | } 71 | if r.S3.Object.Key != "foo/bar=baz.json" { 72 | t.Error("unexpected key", r.S3.Object.Key) 73 | } 74 | } 75 | 76 | func TestParseTestEvent(t *testing.T) { 77 | f, err := os.Open("test/testevent.json") 78 | if err != nil { 79 | t.Error(err) 80 | } 81 | b, _ := ioutil.ReadAll(f) 82 | f.Close() 83 | 84 | event, err := rin.ParseEvent(b) 85 | if err != nil { 86 | t.Error("json decode error", err) 87 | } 88 | if !event.IsTestEvent() { 89 | t.Errorf("not a test event %s", string(b)) 90 | } 91 | if event.String() != "s3:TestEvent for example-bucket" { 92 | t.Errorf("unexpected string %s", event.String()) 93 | } 94 | } 95 | 96 | func TestParseTestEventSns(t *testing.T) { 97 | f, err := os.Open("test/testeventsns.json") 98 | if err != nil { 99 | t.Error(err) 100 | } 101 | b, _ := ioutil.ReadAll(f) 102 | f.Close() 103 | 104 | event, err := rin.ParseEvent(b) 105 | if err != nil { 106 | t.Error("json decode error", err) 107 | } 108 | if !event.IsTestEvent() { 109 | t.Errorf("not a test event %s", string(b)) 110 | } 111 | if event.String() != "s3:TestEvent for example-bucket" { 112 | t.Errorf("unexpected string %s", event.String()) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /local_test.go: -------------------------------------------------------------------------------- 1 | package rin_test 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/aws/aws-sdk-go-v2/aws" 10 | "github.com/aws/aws-sdk-go-v2/credentials" 11 | "github.com/aws/aws-sdk-go-v2/service/s3" 12 | "github.com/aws/aws-sdk-go-v2/service/s3/types" 13 | "github.com/aws/aws-sdk-go-v2/service/sqs" 14 | 15 | rin "github.com/fujiwara/Rin" 16 | ) 17 | 18 | const ( 19 | SQSEndpoint = "http://localhost:4566" 20 | S3Endpoint = "http://localhost:4566" 21 | RedshiftEndpoint = "http://localhost:4566" 22 | ) 23 | 24 | var message = `{ 25 | "Records":[ 26 | { 27 | "eventName":"PutObject", 28 | "s3":{ 29 | "s3SchemaVersion":"1.0", 30 | "bucket":{ 31 | "name":"rin-test" 32 | }, 33 | "object":{ 34 | "key":"test/foo/1" 35 | } 36 | } 37 | } 38 | ] 39 | }` 40 | 41 | var sessions = &rin.SessionStore{ 42 | SQS: &aws.Config{ 43 | Credentials: credentials.StaticCredentialsProvider{ 44 | Value: aws.Credentials{ 45 | AccessKeyID: "foo", 46 | SecretAccessKey: "var", 47 | SessionToken: "", 48 | }, 49 | }, 50 | Region: "ap-northeast-1", 51 | EndpointResolverWithOptions: aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { 52 | return aws.Endpoint{ 53 | PartitionID: "aws", 54 | URL: SQSEndpoint, 55 | SigningRegion: "ap-northeast-1", 56 | }, nil 57 | }), 58 | }, 59 | S3: &aws.Config{ 60 | Credentials: credentials.StaticCredentialsProvider{ 61 | Value: aws.Credentials{ 62 | AccessKeyID: "foo", 63 | SecretAccessKey: "var", 64 | SessionToken: "", 65 | }, 66 | }, 67 | Region: "ap-northeast-1", 68 | EndpointResolverWithOptions: aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { 69 | return aws.Endpoint{ 70 | PartitionID: "aws", 71 | URL: S3Endpoint, 72 | SigningRegion: "ap-northeast-1", 73 | }, nil 74 | }), 75 | }, 76 | S3OptFns: []func(o *s3.Options){ 77 | func(o *s3.Options) { 78 | o.UsePathStyle = true 79 | }, 80 | }, 81 | } 82 | 83 | func TestLocalStack(t *testing.T) { 84 | if os.Getenv("TEST_LOCALSTACK") == "" { 85 | return 86 | } 87 | d1 := setupSQS(t) 88 | defer d1() 89 | 90 | d2 := setupS3(t) 91 | defer d2() 92 | 93 | rin.Sessions = sessions 94 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 95 | defer cancel() 96 | 97 | if err := rin.RunWithContext(ctx, "s3://rin-test/localstack.yml", &rin.Option{BatchMode: false}); err != nil { 98 | t.Error(err) 99 | } 100 | } 101 | 102 | func setupS3(t *testing.T) func() { 103 | svc := s3.NewFromConfig(*sessions.S3, sessions.S3OptFns...) 104 | 105 | bucket := aws.String("rin-test") 106 | key := aws.String("localstack.yml") 107 | 108 | _, err := svc.CreateBucket(context.Background(), &s3.CreateBucketInput{ 109 | Bucket: bucket, 110 | CreateBucketConfiguration: &types.CreateBucketConfiguration{ 111 | LocationConstraint: types.BucketLocationConstraintApNortheast1, 112 | }, 113 | }) 114 | if err != nil { 115 | t.Error(err) 116 | } 117 | 118 | f, err := os.Open("test/localstack.yml") 119 | if err != nil { 120 | t.Error(err) 121 | } 122 | _, err = svc.PutObject(context.Background(), &s3.PutObjectInput{ 123 | Bucket: bucket, 124 | Key: key, 125 | Body: f, 126 | }) 127 | if err != nil { 128 | t.Error(err) 129 | } 130 | 131 | return func() { 132 | svc.DeleteObject(context.Background(), &s3.DeleteObjectInput{ 133 | Bucket: bucket, 134 | Key: key, 135 | }) 136 | svc.DeleteBucket(context.Background(), &s3.DeleteBucketInput{ 137 | Bucket: bucket, 138 | }) 139 | } 140 | } 141 | 142 | func setupSQS(t *testing.T) func() { 143 | svc := sqs.NewFromConfig(*sessions.SQS, sessions.SQSOptFns...) 144 | r, err := svc.CreateQueue(context.Background(), &sqs.CreateQueueInput{ 145 | QueueName: aws.String("rin_test"), 146 | }) 147 | if err != nil { 148 | t.Error(err) 149 | } 150 | 151 | for i := 0; i < 2; i++ { 152 | if _, err := svc.SendMessage(context.Background(), &sqs.SendMessageInput{ 153 | MessageBody: aws.String(message), 154 | QueueUrl: r.QueueUrl, 155 | }); err != nil { 156 | t.Error(err) 157 | } 158 | } 159 | return func() { 160 | svc.DeleteQueue(context.Background(), &sqs.DeleteQueueInput{ 161 | QueueUrl: r.QueueUrl, 162 | }) 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /redshift.go: -------------------------------------------------------------------------------- 1 | package rin 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "log" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/aws/aws-sdk-go-v2/aws" 11 | "github.com/aws/aws-sdk-go-v2/service/redshift" 12 | _ "github.com/lib/pq" 13 | _ "github.com/mashiike/redshift-data-sql-driver" 14 | ) 15 | 16 | var ( 17 | DBPool = make(map[string]*sql.DB, 0) 18 | DBPoolMutex sync.Mutex 19 | redshiftSvc *redshift.Client 20 | ) 21 | 22 | func BoolValue(b *bool) bool { 23 | if b != nil { 24 | return *b 25 | } 26 | return false 27 | } 28 | 29 | func Import(ctx context.Context, event Event) (int, error) { 30 | var processed int 31 | for _, record := range event.Records { 32 | TARGETS: 33 | for _, target := range config.Targets { 34 | if ok, cap := target.MatchEventRecord(record); ok { 35 | if target.Discard { 36 | processed++ 37 | break TARGETS 38 | } 39 | err := target.ImportRedshift(ctx, record, cap) 40 | if err != nil { 41 | if BoolValue(config.Redshift.ReconnectOnError) { 42 | target.DisconnectToRedshift() 43 | } 44 | return processed, err 45 | } else { 46 | processed++ 47 | } 48 | if target.Break { 49 | break TARGETS 50 | } 51 | } 52 | } 53 | } 54 | return processed, nil 55 | } 56 | 57 | func (target *Target) DisconnectToRedshift() { 58 | r := target.Redshift 59 | dsn := r.DSN() 60 | log.Println("[info] Disconnect to Redshift", r.VisibleDSN()) 61 | 62 | DBPoolMutex.Lock() 63 | defer DBPoolMutex.Unlock() 64 | 65 | if db := DBPool[dsn]; db != nil { 66 | db.Close() 67 | } 68 | delete(DBPool, dsn) 69 | } 70 | 71 | func (target *Target) ConnectToRedshift(ctx context.Context) (*sql.DB, error) { 72 | r := target.Redshift 73 | dsn := r.DSN() 74 | 75 | DBPoolMutex.Lock() 76 | defer DBPoolMutex.Unlock() 77 | 78 | if db := DBPool[dsn]; db != nil { 79 | if db.Ping() == nil { 80 | return db, nil 81 | } else { 82 | delete(DBPool, dsn) 83 | } 84 | } 85 | log.Println("[info] Connect to Redshift", r.VisibleDSN()) 86 | 87 | var user, password = r.User, r.Password 88 | // redshift-data driver creates a temporary credentials by itself 89 | if password == "" && r.Driver != DriverRedshiftData { 90 | if redshiftSvc == nil { 91 | redshiftSvc = redshift.NewFromConfig(*Sessions.Redshift, Sessions.RedshiftOptFns...) 92 | } 93 | id := strings.SplitN(r.Host, ".", 2)[0] 94 | log.Printf("[info] Getting cluster credentials for %s user %s", r.Host, r.User) 95 | res, err := redshiftSvc.GetClusterCredentials(ctx, &redshift.GetClusterCredentialsInput{ 96 | ClusterIdentifier: aws.String(id), 97 | DbUser: aws.String(r.User), 98 | }) 99 | if err != nil { 100 | return nil, err 101 | } 102 | user, password = *res.DbUser, *res.DbPassword 103 | log.Printf("[debug] Got user %s password %s", user, password) 104 | } 105 | 106 | db, err := sql.Open(r.Driver, r.DSNWith(user, password)) 107 | if err != nil { 108 | return nil, err 109 | } 110 | DBPool[dsn] = db 111 | return db, nil 112 | } 113 | 114 | func (target *Target) ImportRedshift(ctx context.Context, record *EventRecord, cap *[]string) error { 115 | if config.Redshift.UseTransaction() { 116 | return target.importRedshiftWithTx(ctx, record, cap) 117 | } else { 118 | return target.importRedshiftWithoutTx(ctx, record, cap) 119 | } 120 | } 121 | 122 | func (target *Target) importRedshiftWithTx(ctx context.Context, record *EventRecord, cap *[]string) error { 123 | log.Printf("[info] Import to target %s from record %s", target, record) 124 | db, err := target.ConnectToRedshift(ctx) 125 | if err != nil { 126 | return err 127 | } 128 | txn, err := db.Begin() 129 | if err != nil { 130 | return err 131 | } 132 | defer txn.Rollback() 133 | 134 | query, err := target.BuildCopySQL(record.S3.Object.Key, config.Credentials, cap) 135 | if err != nil { 136 | return err 137 | } 138 | log.Println("[debug] SQL:", query) 139 | 140 | stmt, err := txn.Prepare(query) 141 | if err != nil { 142 | return err 143 | } 144 | defer stmt.Close() 145 | _, err = stmt.Exec() 146 | if err != nil { 147 | return err 148 | } 149 | 150 | err = txn.Commit() 151 | if err != nil { 152 | return err 153 | } 154 | return nil 155 | } 156 | 157 | func (target *Target) importRedshiftWithoutTx(ctx context.Context, record *EventRecord, cap *[]string) error { 158 | log.Printf("[info] Import to target %s from record %s", target, record) 159 | db, err := target.ConnectToRedshift(ctx) 160 | if err != nil { 161 | return err 162 | } 163 | 164 | query, err := target.BuildCopySQL(record.S3.Object.Key, config.Credentials, cap) 165 | if err != nil { 166 | return err 167 | } 168 | log.Println("[debug] SQL:", query) 169 | 170 | if _, err := db.Exec(query); err != nil { 171 | return err 172 | } 173 | 174 | return nil 175 | } 176 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rin 2 | 3 | Rin is a Redshift data Importer by SQS messaging. 4 | 5 | ## Architecture 6 | 7 | 1. (Someone) creates a S3 object. 8 | 2. [S3 event notifications](https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) will send to a message to SQS. 9 | 3. Rin will fetch messages from SQS, and publish a "COPY" query to Redshift. 10 | 11 | ## Installation 12 | 13 | ### Binary packages 14 | 15 | [Releases](https://github.com/fujiwara/Rin/releases) 16 | 17 | ### Homebrew 18 | 19 | ```console 20 | $ brew install fujiwara/tap/rin 21 | ``` 22 | 23 | ### Docker 24 | 25 | [GitHub Packages](https://github.com/users/fujiwara/packages/container/package/rin) 26 | 27 | ```console 28 | $ docker pull ghcr.io/fujiwara/rin:v1.1.3 29 | ``` 30 | 31 | ## Configuration 32 | 33 | [Configuring Amazon S3 Event Notifications](https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html). 34 | 35 | 1. Create SQS queue. 36 | 2. Attach SQS access policy to the queue. [Example Walkthrough 1:](https://docs.aws.amazon.com/AmazonS3/latest/dev/ways-to-add-notification-config-to-bucket.html) 37 | 3. [Enable Event Notifications](http://docs.aws.amazon.com/AmazonS3/latest/UG/SettingBucketNotifications.html) on a S3 bucket. 38 | 4. Run `rin` process with configuration for using the SQS and S3. 39 | 40 | ### config.yaml 41 | 42 | ```yaml 43 | queue_name: my_queue_name # SQS queue name 44 | 45 | credentials: 46 | aws_region: ap-northeast-1 47 | 48 | redshift: 49 | host: localhost 50 | port: 5439 51 | dbname: test 52 | user: test_user 53 | password: '{{ must_env "REDSHIFT_PASSWORD" }}' 54 | schema: public 55 | reconnect_on_error: true # disconnect Redshift on error occurred 56 | 57 | s3: 58 | bucket: test.bucket.test 59 | region: ap-northeast-1 60 | 61 | sql_option: "JSON 'auto' GZIP" # COPY SQL option 62 | 63 | # define import target mappings 64 | targets: 65 | - s3: 66 | key_prefix: test/foo/ignore 67 | discard: true # Do not import and do not try following targets. Matches only. 68 | 69 | - redshift: 70 | table: foo 71 | s3: 72 | key_prefix: test/foo 73 | 74 | - redshift: 75 | schema: xxx 76 | table: bar 77 | s3: 78 | key_prefix: test/bar 79 | break: true # Do not try following targets. 80 | 81 | - redshift: 82 | schema: $1 # expand by key_regexp captured value. 83 | table: $2 84 | s3: 85 | key_regexp: test/schema-([a-z]+)/table-([a-z]+)/ 86 | 87 | - redshift: 88 | host: redshift.example.com # override default section in this target 89 | port: 5439 90 | dbname: example 91 | user: example_user 92 | password: example_pass 93 | schema: public 94 | table: example 95 | s3: 96 | bucket: redshift.example.com 97 | region: ap-northeast-1 98 | key_prefix: logs/example/ 99 | sql_option: "CSV DELIMITER ',' ESCAPE" 100 | ``` 101 | 102 | A configuration file is parsed by [kayac/go-config](https://github.com/kayac/go-config). 103 | 104 | go-config expands environment variables using syntax `{{ env "FOO" }}` or `{{ must_env "FOO" }}` in a configuration file. 105 | 106 | When the password for Redshift is empty, Rin will try call [GetClusterCredentials API](https://docs.aws.amazon.com/redshift/latest/APIReference/API_GetClusterCredentials.html) to get a temporary password for the cluster. 107 | 108 | #### Credentials 109 | 110 | Rin requires credentials for SQS and Redshift. 111 | 112 | 1. `credentials.aws_access_key_id` and `credentials.aws_secret_access_key` 113 | - used for SQS and Redshift(COPY query and Data API access). 114 | 2. `credentials.aws_iam_role` 115 | - used for Redshift COPY query only. 116 | - for SQS and Redshift Data API, Rin will try to get a instance credentials. 117 | 118 | ## Run 119 | 120 | ### daemon mode 121 | 122 | Rin waits new SQS messages and processing it continually. 123 | 124 | ``` 125 | $ rin -config config.yaml [-debug] 126 | ``` 127 | 128 | `-config` also accepts HTTP/S3/File URL to specify the location of configuration file. 129 | For example, 130 | 131 | ``` 132 | $ rin -config s3://rin-config.my-bucket/config.yaml 133 | ``` 134 | 135 | ### batch mode 136 | 137 | Rin process new SQS messages and exit. 138 | 139 | ``` 140 | $ rin -config config.yaml -batch [-debug] 141 | ``` 142 | 143 | ## Set max execution time 144 | 145 | A CLI option `-max-execution-time` is set max execution time for running SQS worker and batch process. 146 | 147 | ## SQL Drivers 148 | 149 | Rin has two ways to connect to Redshift. 150 | 151 | ### `postgres` driver 152 | 153 | `postgres` driver is the default. Rin connects to Redshift with PostgreSQL protocol over TCP in the VPC network. 154 | 155 | `host`, `port`, `user` and `password` fields are required in the `redshift` section. 156 | 157 | ```yaml 158 | redshift: 159 | driver: postgres # default 160 | host: localhost 161 | port: 5439 162 | user: test_user 163 | password: '{{ must_env "REDSHIFT_PASSWORD" }}' 164 | ``` 165 | 166 | ### `redshift-data` driver 167 | 168 | `redshift-data` driver connects to Redshift via [Redshift Data API](https://docs.aws.amazon.com/redshift/latest/mgmt/data-api.html). 169 | 170 | Redshift Data API does not require a VPC network. 171 | 172 | With provisoned cluster, `driver`, `cluster` and `user` are required. 173 | 174 | ```yaml 175 | redshift: 176 | driver: redshift-data 177 | cluster: your-cluster-name 178 | user: test_user 179 | ``` 180 | 181 | With Redshift serverless, `driver`, `workgroup` are required. 182 | 183 | ```yaml 184 | redshift: 185 | driver: redshift-data 186 | workgroup: your-workgroup-name 187 | ``` 188 | 189 | See also [github.com/mashiike/redshift-data-sql-driver](https://github.com/mashiike/redshift-data-sql-driver). 190 | -------------------------------------------------------------------------------- /config_test.go: -------------------------------------------------------------------------------- 1 | package rin_test 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | rin "github.com/fujiwara/Rin" 9 | ) 10 | 11 | var BrokenConfig = []string{ 12 | "test/config.yml.invalid_regexp", 13 | "test/config.yml.no_key_matcher", 14 | "test/config.yml.not_found", 15 | } 16 | 17 | type testExpected struct { 18 | targets []testTarget 19 | Driver string 20 | DSN string 21 | VisibleDSN string 22 | } 23 | 24 | type testTarget struct { 25 | Bucket string 26 | Key string 27 | SQL string 28 | } 29 | 30 | var Expected = testExpected{ 31 | DSN: "postgres://test_user:test_pass@localhost:5432/test", 32 | VisibleDSN: "redshift://test_user:****@localhost:5432/test", 33 | Driver: "postgres", 34 | targets: []testTarget{ 35 | { 36 | "test.bucket.test", 37 | "test/foo/xxx.json", 38 | `/* Rin */ COPY "foo" FROM 's3://test.bucket.test/test/foo/xxx.json' CREDENTIALS 'aws_access_key_id=AAA;aws_secret_access_key=SSS' REGION 'ap-northeast-1' JSON 'auto' GZIP`, 39 | }, 40 | { 41 | "test.bucket.test", 42 | "test/foo/discard/xxx.json", 43 | "", 44 | }, 45 | { 46 | "test.bucket.test", 47 | "test/bar/break", 48 | `/* Rin */ COPY "xxx"."bar_break" FROM 's3://test.bucket.test/test/bar/break' CREDENTIALS 'aws_access_key_id=AAA;aws_secret_access_key=SSS' REGION 'ap-northeast-1' CSV DELIMITER ',' ESCAPE`, 49 | }, 50 | { 51 | "test.bucket.test", 52 | "test/bar/y's.csv", 53 | `/* Rin */ COPY "xxx"."bar" FROM 's3://test.bucket.test/test/bar/y''s.csv' CREDENTIALS 'aws_access_key_id=AAA;aws_secret_access_key=SSS' REGION 'ap-northeast-1' CSV DELIMITER ',' ESCAPE`, 54 | }, 55 | { 56 | "example.bucket", 57 | "test/s1/t256/aaa.json", 58 | `/* Rin */ COPY "s1"."t256" FROM 's3://example.bucket/test/s1/t256/aaa.json' CREDENTIALS 'aws_access_key_id=AAA;aws_secret_access_key=SSS' REGION 'ap-northeast-1' JSON 'auto' GZIP`, 59 | }, 60 | }, 61 | } 62 | 63 | var ExpectedIAMRole = testExpected{ 64 | DSN: "postgres://test_user:test_pass@localhost:5432/test", 65 | VisibleDSN: "redshift://test_user:****@localhost:5432/test", 66 | Driver: "postgres", 67 | targets: []testTarget{ 68 | { 69 | "test.bucket.test", 70 | "test/foo/xxx.json", 71 | `/* Rin */ COPY "foo" FROM 's3://test.bucket.test/test/foo/xxx.json' CREDENTIALS 'aws_iam_role=arn:aws:iam::123456789012:role/rin' REGION 'ap-northeast-1' JSON 'auto' GZIP`, 72 | }, 73 | { 74 | "test.bucket.test", 75 | "test/foo/discard/xxx.json", 76 | "", 77 | }, 78 | { 79 | "test.bucket.test", 80 | "test/bar/break", 81 | `/* Rin */ COPY "xxx"."bar_break" FROM 's3://test.bucket.test/test/bar/break' CREDENTIALS 'aws_iam_role=arn:aws:iam::123456789012:role/rin' REGION 'ap-northeast-1' CSV DELIMITER ',' ESCAPE`, 82 | }, 83 | { 84 | "test.bucket.test", 85 | "test/bar/y's.csv", 86 | `/* Rin */ COPY "xxx"."bar" FROM 's3://test.bucket.test/test/bar/y''s.csv' CREDENTIALS 'aws_iam_role=arn:aws:iam::123456789012:role/rin' REGION 'ap-northeast-1' CSV DELIMITER ',' ESCAPE`, 87 | }, 88 | { 89 | "example.bucket", 90 | "test/s1/t256/aaa.json", 91 | `/* Rin */ COPY "s1"."t256" FROM 's3://example.bucket/test/s1/t256/aaa.json' CREDENTIALS 'aws_iam_role=arn:aws:iam::123456789012:role/rin' REGION 'ap-northeast-1' JSON 'auto' GZIP`, 92 | }, 93 | }, 94 | } 95 | 96 | var ExpectedRedshiftData = testExpected{ 97 | DSN: "test_user@cluster(mycluster)/test", 98 | VisibleDSN: "redshift-data://test_user@cluster(mycluster)/test", 99 | Driver: "redshift-data", 100 | targets: []testTarget{ 101 | { 102 | "test.bucket.test", 103 | "test/foo/xxx.json", 104 | `/* Rin */ COPY "foo" FROM 's3://test.bucket.test/test/foo/xxx.json' CREDENTIALS 'aws_iam_role=arn:aws:iam::123456789012:role/rin' REGION 'ap-northeast-1' JSON 'auto' GZIP`, 105 | }, 106 | { 107 | "test.bucket.test", 108 | "test/foo/discard/xxx.json", 109 | "", 110 | }, 111 | { 112 | "test.bucket.test", 113 | "test/bar/break", 114 | `/* Rin */ COPY "xxx"."bar_break" FROM 's3://test.bucket.test/test/bar/break' CREDENTIALS 'aws_iam_role=arn:aws:iam::123456789012:role/rin' REGION 'ap-northeast-1' CSV DELIMITER ',' ESCAPE`, 115 | }, 116 | { 117 | "test.bucket.test", 118 | "test/bar/y's.csv", 119 | `/* Rin */ COPY "xxx"."bar" FROM 's3://test.bucket.test/test/bar/y''s.csv' CREDENTIALS 'aws_iam_role=arn:aws:iam::123456789012:role/rin' REGION 'ap-northeast-1' CSV DELIMITER ',' ESCAPE`, 120 | }, 121 | { 122 | "example.bucket", 123 | "test/s1/t256/aaa.json", 124 | `/* Rin */ COPY "s1"."t256" FROM 's3://example.bucket/test/s1/t256/aaa.json' CREDENTIALS 'aws_iam_role=arn:aws:iam::123456789012:role/rin' REGION 'ap-northeast-1' JSON 'auto' GZIP`, 125 | }, 126 | }, 127 | } 128 | 129 | func TestLoadConfigError(t *testing.T) { 130 | ctx := context.Background() 131 | for _, f := range BrokenConfig { 132 | _, err := rin.LoadConfig(ctx, f) 133 | if err == nil { 134 | t.Errorf("LoadConfig(%s) must be failed", f) 135 | } 136 | t.Log(err) 137 | } 138 | } 139 | 140 | func TestLoadConfig(t *testing.T) { 141 | os.Setenv("AWS_SECRET_ACCESS_KEY", "SSS") 142 | testConfig(t, "test/config.yml", Expected) 143 | testConfig(t, "test/config.iam_role.yml", ExpectedIAMRole) 144 | testConfig(t, "test/config.redshift-data.yml", ExpectedRedshiftData) 145 | } 146 | 147 | func testConfig(t *testing.T, name string, expected testExpected) { 148 | ctx := context.Background() 149 | config, err := rin.LoadConfig(ctx, name) 150 | if err != nil { 151 | t.Fatalf("load config failed: %s", err) 152 | } 153 | //t.Log("global.sql_option", config.SQLOption) 154 | if len(config.Targets) != 5 { 155 | t.Error("invalid targets len", len(config.Targets)) 156 | } 157 | 158 | if expected.DSN != config.Redshift.DSN() { 159 | t.Errorf("invalid DSN expected %s got %s", expected.DSN, config.Redshift.DSN()) 160 | } 161 | if expected.VisibleDSN != config.Redshift.VisibleDSN() { 162 | t.Errorf("invalid VisibleDSN expected %s got %s", expected.VisibleDSN, config.Redshift.VisibleDSN()) 163 | } 164 | if expected.Driver != config.Redshift.Driver { 165 | t.Errorf("invalid Driver expected %s got %s", expected.Driver, config.Redshift.Driver) 166 | } 167 | 168 | for _, e := range expected.targets { 169 | var sql string 170 | var err error 171 | for i, target := range config.Targets { 172 | ok, cap := target.Match(e.Bucket, e.Key) 173 | if !ok { 174 | continue 175 | } 176 | if target.Discard { 177 | t.Log("discard", e.Key, "target", i) 178 | break 179 | } else { 180 | t.Log("build", e.Key, "target", i) 181 | sql, err = target.BuildCopySQL(e.Key, config.Credentials, cap) 182 | if err != nil { 183 | t.Error(err) 184 | } 185 | //t.Log(sql) 186 | } 187 | if target.Break { 188 | t.Log("break", e.Key, "target", i) 189 | break 190 | } 191 | if !rin.BoolValue(target.Redshift.ReconnectOnError) { 192 | t.Error("reconnect_on_error must be true") 193 | } 194 | } 195 | if sql != e.SQL { 196 | t.Errorf("unexpected SQL:\nExpected:%s\nGot:%s", e.SQL, sql) 197 | } 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= 3 | github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= 4 | github.com/aws/aws-lambda-go v1.34.1 h1:M3a/uFYBjii+tDcOJ0wL/WyFi2550FHoECdPf27zvOs= 5 | github.com/aws/aws-lambda-go v1.34.1/go.mod h1:jwFe2KmMsHmffA1X2R09hH6lFzJQxzI8qK17ewzbQMM= 6 | github.com/aws/aws-sdk-go-v2 v1.16.13/go.mod h1:xSyvSnzh0KLs5H4HJGeIEsNYemUWdNIl0b/rP6SIsLU= 7 | github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk= 8 | github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= 9 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.6 h1:PPefqpze5qW/eqdgK5RqtOTQi5GhXpSxitbGqImAQ1I= 10 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.6/go.mod h1:bcLsAUI2iDC8zC52XQvczR/zpaC1q/wP32p3wwvqGVo= 11 | github.com/aws/aws-sdk-go-v2/config v1.17.4/go.mod h1:ul+ru+huVpfduF9XRmGUq82T8T3K+nIFQuF6F+L+548= 12 | github.com/aws/aws-sdk-go-v2/config v1.18.0 h1:ULASZmfhKR/QE9UeZ7mzYjUzsnIydy/K1YMT6uH1KC0= 13 | github.com/aws/aws-sdk-go-v2/config v1.18.0/go.mod h1:H13DRX9Nv5tAcQvPABrE3dm5XnLp1RC7fVSM3OWiLvA= 14 | github.com/aws/aws-sdk-go-v2/credentials v1.12.17/go.mod h1:jd1mvJulXY7ccHvcSiJceYhv06yWIIRkJnwWEA4IX+g= 15 | github.com/aws/aws-sdk-go-v2/credentials v1.13.0 h1:W5f73j1qurASap+jdScUo4aGzSXxaC7wq1i7CiwhvU8= 16 | github.com/aws/aws-sdk-go-v2/credentials v1.13.0/go.mod h1:prZpUfBu1KZLBLVX482Sq4DpDXGugAre08TPEc21GUg= 17 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.14/go.mod h1:5CU57SyF5jZLSIw4OOll0PG83ThXwNdkRFOc0EltD/0= 18 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs= 19 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8= 20 | github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.30 h1:Rtd+R7uWtQg5+bZ72x1g1ENjQykhFKnayo6Lv/QpxFU= 21 | github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.30/go.mod h1:Fbi0PULkPycJg44P9rwhQUGknk8Fl6DUTXcCaSZ6FeI= 22 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.20/go.mod h1:gdZ5gRUaxThXIZyZQ8MTtgYBk2jbHgp05BO3GcD9Cwc= 23 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E= 24 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= 25 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.14/go.mod h1:GEV9jaDPIgayiU+uevxwozcvUOjc+P4aHE2BeSjm2vE= 26 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 h1:oRHDrwCTVT8ZXi4sr9Ld+EXk7N/KGssOr2ygNeojEhw= 27 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= 28 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.21/go.mod h1:Q0pktZjvRZk77TBto6yAvUAi7fcse1bdcMctBDVGgBw= 29 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 h1:Mza+vlnZr+fPKFKRq/lKGVvM6B/8ZZmNdEopOwSQLms= 30 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26/go.mod h1:Y2OJ+P+MC1u1VKnavT+PshiEuGPyh/7DqxoDNij4/bg= 31 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.11 h1:zFriLANEIFWl/TQvPqhRASnU8Xr9fzshPL0OY7e1DpM= 32 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.11/go.mod h1:EAtoA46xWR2I0fROMCsb0lgC4kYfgaK9EBrCv9hIHYM= 33 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.7 h1:f0l2kujaZ0UyqwfKdtPaYQs8vzFmLbtPhWDNYeEY4ho= 34 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.7/go.mod h1:aGaU7sKr91r4yZCi+4fWpsDepAzy8A6u/1enpD3K6mM= 35 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.15 h1:xw0EMeNfAdmiFX3Ix9OOdqW/S2GPeV3WAYKHr2qR/W4= 36 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.15/go.mod h1:SvmZIJp6fx7Yua+4hhigLm5kVRDWo56Cj+j8FvVl6M8= 37 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.14/go.mod h1:8qOLjqMzY/S1kh3myDXA1yxK5eD4uN8aOJgKpgvc4OM= 38 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 h1:GE25AWCdNUPh9AOJzI9KIJnja7IwUc1WyUqz/JTyJ/I= 39 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19/go.mod h1:02CP6iuYP+IVnBX5HULVdSAku/85eHB2Y9EsFhrkEwU= 40 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.14 h1:sGFyMilgKmgg8TsGMUXApIvIrbc9SZs2sFrbdugL21c= 41 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.14/go.mod h1:QWqlQbLB0GYO6hDDUwPKr2VKr7C6lpCdOzs92IVYQmk= 42 | github.com/aws/aws-sdk-go-v2/service/redshift v1.26.7 h1:ZcKfXp8TaF2tIPokg1zwXSdExvBj5uY974oE515jvE4= 43 | github.com/aws/aws-sdk-go-v2/service/redshift v1.26.7/go.mod h1:Y1KwXk8Pdg6mY/zFKAgnCp+tv9OvGcQFPPzPZCTXW7E= 44 | github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.16.13 h1:hVrup9EZ/QgcSn6viaHnNTGvr+F82fUekDrvmtoKIPQ= 45 | github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.16.13/go.mod h1:7f6XcpJ1hH7ai5yby7gE03CaIXqGvlaWGilNnE/wzpc= 46 | github.com/aws/aws-sdk-go-v2/service/s3 v1.27.8 h1:zYpocIndjdPRURWkq/Rschy8WpC+vL0f74z+lJhEpJk= 47 | github.com/aws/aws-sdk-go-v2/service/s3 v1.27.8/go.mod h1:aljgUlqAplymnhQNEcyx/fjUmQtOXCsS6Ry+ySpCcA8= 48 | github.com/aws/aws-sdk-go-v2/service/sqs v1.19.7 h1:7Ui029eK+i+6JILQXUYG6lzRdWUq8pbbJvkegFR6Soc= 49 | github.com/aws/aws-sdk-go-v2/service/sqs v1.19.7/go.mod h1:vMdSMmI0ajtCjxN4pTocddojOpPSQWBH6L0VsuQbLyQ= 50 | github.com/aws/aws-sdk-go-v2/service/sso v1.11.20/go.mod h1:hPsROgDdgY/NQ1gPt7VJWG0GjSnalDC0DkkMfGEw2gc= 51 | github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 h1:GFZitO48N/7EsFDt8fMa5iYdmWqkUDDB3Eje6z3kbG0= 52 | github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vboR0FeJ89OkEy1M9mWbK2ifCI= 53 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.2/go.mod h1:5cxfDYtY2mDOlmesy4yycb6lwyy1U/iAUOHKhQLKw/E= 54 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZoe5gvi9wjudnxvozYFJo= 55 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI= 56 | github.com/aws/aws-sdk-go-v2/service/sts v1.16.16/go.mod h1:Y9iBgT1w2vHtYzJEkwD6FqILjDSsvbxcW/+wIYxyse4= 57 | github.com/aws/aws-sdk-go-v2/service/sts v1.17.2 h1:tpwEMRdMf2UsplengAOnmSIRdvAxf75oUFR+blBr92I= 58 | github.com/aws/aws-sdk-go-v2/service/sts v1.17.2/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= 59 | github.com/aws/smithy-go v1.13.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= 60 | github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk= 61 | github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= 62 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 63 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 64 | github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 65 | github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= 66 | github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 67 | github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= 68 | github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= 69 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 70 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 71 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 72 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 73 | github.com/kayac/go-config v0.6.0 h1:Y4l9tsWrUCvT1id8tbO4aT4SdGxbYqd8lqSe5l1GrK0= 74 | github.com/kayac/go-config v0.6.0/go.mod h1:5C4ZN+sMjYpEX0bi+AcgF6g0hZYVdzZiV16TEyzAzfk= 75 | github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= 76 | github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 77 | github.com/mashiike/redshift-data-sql-driver v0.1.0 h1:ANVUFHt7qXvpjPUKt5b+RT3yJ2TkcOhn4rUaJMbVJr0= 78 | github.com/mashiike/redshift-data-sql-driver v0.1.0/go.mod h1:ETccRF1+n++o8HP4eG/IrSeA+hlywV+qgF9y0oF2D1I= 79 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 80 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 81 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 82 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 83 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 84 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 85 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 86 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 87 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 88 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 89 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 90 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 91 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 92 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 93 | -------------------------------------------------------------------------------- /rin.go: -------------------------------------------------------------------------------- 1 | package rin 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/signal" 9 | "strings" 10 | "sync" 11 | "syscall" 12 | "time" 13 | 14 | "github.com/aws/aws-sdk-go-v2/aws" 15 | awsConfig "github.com/aws/aws-sdk-go-v2/config" 16 | "github.com/aws/aws-sdk-go-v2/credentials" 17 | "github.com/aws/aws-sdk-go-v2/service/redshift" 18 | "github.com/aws/aws-sdk-go-v2/service/redshiftdata" 19 | "github.com/aws/aws-sdk-go-v2/service/s3" 20 | "github.com/aws/aws-sdk-go-v2/service/sqs" 21 | redshiftdatasqldriver "github.com/mashiike/redshift-data-sql-driver" 22 | ) 23 | 24 | var config *Config 25 | var MaxDeleteRetry = 8 26 | var Sessions *SessionStore 27 | 28 | type Option struct { 29 | MaxExecutionTime time.Duration `json:"max_execution_time"` 30 | BatchMode bool `json:"batch_mode"` 31 | } 32 | 33 | func (o *Option) String() string { 34 | return strings.Join([]string{ 35 | "MaxExecutionTime: " + o.MaxExecutionTime.String(), 36 | fmt.Sprintf("BatchMode: %v", o.BatchMode), 37 | }, ", ") 38 | } 39 | 40 | func init() { 41 | Sessions = &SessionStore{} 42 | redshiftdatasqldriver.RedshiftDataClientConstructor = func(ctx context.Context, cfg *redshiftdatasqldriver.RedshiftDataConfig) (redshiftdatasqldriver.RedshiftDataClient, error) { 43 | return redshiftdata.NewFromConfig(*Sessions.Redshift, cfg.RedshiftDataOptFns...), nil 44 | } 45 | } 46 | 47 | type SessionStore struct { 48 | SQS *aws.Config 49 | SQSOptFns []func(*sqs.Options) 50 | Redshift *aws.Config 51 | RedshiftOptFns []func(*redshift.Options) 52 | S3 *aws.Config 53 | S3OptFns []func(*s3.Options) 54 | } 55 | 56 | var TrapSignals = []os.Signal{ 57 | syscall.SIGHUP, 58 | syscall.SIGINT, 59 | syscall.SIGTERM, 60 | syscall.SIGQUIT, 61 | } 62 | 63 | type NoMessageError struct { 64 | s string 65 | } 66 | 67 | func (e NoMessageError) Error() string { 68 | return e.s 69 | } 70 | 71 | type MaxExecutionTimeReachedError struct{} 72 | 73 | func (e MaxExecutionTimeReachedError) Error() string { 74 | return "max execution time reached" 75 | } 76 | 77 | func DryRun(configFile string, opt *Option) error { 78 | ctx := context.Background() 79 | var err error 80 | log.Println("[info] Loading config:", configFile) 81 | config, err = LoadConfig(ctx, configFile) 82 | if err != nil { 83 | return err 84 | } 85 | for _, target := range config.Targets { 86 | log.Println("[info] Define target", target.String()) 87 | } 88 | return nil 89 | } 90 | 91 | func Run(configFile string, opt *Option) error { 92 | return RunWithContext(context.Background(), configFile, opt) 93 | } 94 | 95 | func RunWithContext(ctx context.Context, configFile string, opt *Option) error { 96 | var err error 97 | log.Println("[info] Loading config:", configFile) 98 | config, err = LoadConfig(ctx, configFile) 99 | if err != nil { 100 | return err 101 | } 102 | for _, target := range config.Targets { 103 | log.Println("[info] Define target", target.String()) 104 | } 105 | 106 | if Sessions.SQS == nil { 107 | opts := []func(*awsConfig.LoadOptions) error{ 108 | awsConfig.WithRegion(config.Credentials.AWS_REGION), 109 | } 110 | if config.Credentials.AWS_ACCESS_KEY_ID != "" { 111 | opts = append(opts, awsConfig.WithCredentialsProvider(credentials.StaticCredentialsProvider{ 112 | Value: aws.Credentials{ 113 | AccessKeyID: config.Credentials.AWS_ACCESS_KEY_ID, 114 | SecretAccessKey: config.Credentials.AWS_SECRET_ACCESS_KEY, 115 | Source: "from Rin config", 116 | }, 117 | })) 118 | } 119 | c, err := awsConfig.LoadDefaultConfig(ctx, opts...) 120 | if err != nil { 121 | return err 122 | } 123 | Sessions.SQS = &c 124 | Sessions.SQSOptFns = make([]func(*sqs.Options), 0) 125 | Sessions.Redshift = &c 126 | Sessions.RedshiftOptFns = make([]func(*redshift.Options), 0) 127 | Sessions.S3 = &c 128 | Sessions.S3OptFns = make([]func(*s3.Options), 0) 129 | } 130 | 131 | if isLambda() { 132 | return runLambdaHandler(opt) 133 | } 134 | 135 | signalCh := make(chan os.Signal, 1) 136 | signal.Notify(signalCh, TrapSignals...) 137 | ctx, cancel := context.WithCancel(ctx) 138 | defer cancel() 139 | var wg sync.WaitGroup 140 | wg.Add(2) // signal handler + sqsWorker 141 | 142 | // wait for signal 143 | go func() { 144 | defer wg.Done() 145 | select { 146 | case sig := <-signalCh: 147 | log.Printf("[info] Got signal: %s(%d)", sig, sig) 148 | log.Println("[info] Shutting down worker...") 149 | cancel() 150 | case <-ctx.Done(): 151 | } 152 | }() 153 | 154 | // run worker 155 | err = sqsWorker(ctx, &wg, opt) 156 | if e, ok := err.(MaxExecutionTimeReachedError); ok { 157 | log.Printf("[info] %s", e.Error()) 158 | cancel() 159 | } 160 | 161 | wg.Wait() 162 | log.Println("[info] Shutdown.") 163 | if ctx.Err() == context.Canceled { 164 | // normally exit 165 | return nil 166 | } 167 | return err 168 | } 169 | 170 | func isLambda() bool { 171 | return strings.HasPrefix(os.Getenv("AWS_EXECUTION_ENV"), "AWS_Lambda") || os.Getenv("AWS_LAMBDA_RUNTIME_API") != "" 172 | } 173 | 174 | func sqsWorker(ctx context.Context, wg *sync.WaitGroup, opt *Option) error { 175 | svc := sqs.NewFromConfig(*Sessions.SQS, Sessions.SQSOptFns...) 176 | var mode string 177 | if opt.BatchMode { 178 | mode = "Batch" 179 | } else { 180 | mode = "Worker" 181 | } 182 | log.Printf("[info] Starting up SQS %s", mode) 183 | defer log.Printf("[info] Shutdown SQS %s", mode) 184 | defer wg.Done() 185 | 186 | log.Println("[info] Connect to SQS:", config.QueueName) 187 | res, err := svc.GetQueueUrl(ctx, &sqs.GetQueueUrlInput{ 188 | QueueName: aws.String(config.QueueName), 189 | }) 190 | if err != nil { 191 | return err 192 | } 193 | var timeout <-chan time.Time 194 | if opt.MaxExecutionTime > 0 { 195 | timeout = time.NewTimer(opt.MaxExecutionTime).C 196 | } else { 197 | timeout = make(chan time.Time, 1) // never timeout 198 | } 199 | for { 200 | select { 201 | case <-timeout: 202 | return MaxExecutionTimeReachedError{} 203 | case <-ctx.Done(): 204 | return nil 205 | default: 206 | } 207 | if err := handleMessage(ctx, svc, res.QueueUrl); err != nil { 208 | if e, ok := err.(NoMessageError); ok { 209 | if opt.BatchMode { 210 | log.Printf("[info] %s. Exit.", e.Error()) 211 | break 212 | } 213 | time.Sleep(100 * time.Millisecond) 214 | } 215 | } 216 | } 217 | return nil 218 | } 219 | 220 | func handleMessage(ctx context.Context, svc *sqs.Client, queueUrl *string) error { 221 | var completed = false 222 | res, err := svc.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{ 223 | MaxNumberOfMessages: 1, 224 | QueueUrl: queueUrl, 225 | }) 226 | if err != nil { 227 | return err 228 | } 229 | if len(res.Messages) == 0 { 230 | return NoMessageError{"No messages"} 231 | } 232 | msg := res.Messages[0] 233 | msgId := *msg.MessageId 234 | log.Printf("[info] [%s] Starting process message.", msgId) 235 | log.Printf("[debug] [%s] handle: %s", msgId, *msg.ReceiptHandle) 236 | log.Printf("[debug] [%s] body: %s", msgId, *msg.Body) 237 | 238 | defer func() { 239 | if !completed { 240 | log.Printf("[info] [%s] Aborted message. ReceiptHandle: %s", msgId, *msg.ReceiptHandle) 241 | } 242 | }() 243 | 244 | if err := processEvent(ctx, msgId, *msg.Body); err != nil { 245 | return err 246 | } 247 | 248 | ctxDelete, cancel := context.WithTimeout(ctx, 60*time.Second) 249 | defer cancel() 250 | _, err = svc.DeleteMessage(ctxDelete, &sqs.DeleteMessageInput{ 251 | QueueUrl: queueUrl, 252 | ReceiptHandle: msg.ReceiptHandle, 253 | }) 254 | if err != nil { 255 | log.Printf("[warn] [%s] Can't delete message. %s", msgId, err) 256 | // retry 257 | for i := 1; i <= MaxDeleteRetry; i++ { 258 | log.Printf("[info] [%s] Retry to delete after %d sec.", msgId, i*i) 259 | time.Sleep(time.Duration(i*i) * time.Second) 260 | _, err = svc.DeleteMessage(context.Background(), &sqs.DeleteMessageInput{ 261 | QueueUrl: queueUrl, 262 | ReceiptHandle: msg.ReceiptHandle, 263 | }) 264 | if err == nil { 265 | log.Printf("[info] [%s] Message was deleted successfuly.", msgId) 266 | break 267 | } 268 | log.Printf("[warn] [%s] Can't delete message. %s", msgId, err) 269 | if i == MaxDeleteRetry { 270 | log.Printf("[error] [%s] Max retry count reached. Giving up.", msgId) 271 | } 272 | } 273 | } 274 | 275 | completed = true 276 | log.Printf("[info] [%s] Completed message.", msgId) 277 | return nil 278 | } 279 | 280 | func processEvent(ctx context.Context, msgId string, body string) error { 281 | event, err := ParseEvent([]byte(body)) 282 | if err != nil { 283 | log.Printf("[error] [%s] Can't parse event from Body. %s", msgId, err) 284 | return err 285 | } 286 | if event.IsTestEvent() { 287 | log.Printf("[info] [%s] Skipping %s", msgId, event.String()) 288 | } else { 289 | log.Printf("[info] [%s] Importing event: %s", msgId, event) 290 | n, err := Import(ctx, event) 291 | if err != nil { 292 | log.Printf("[error] [%s] Import failed. %s", msgId, err) 293 | return err 294 | } 295 | if n == 0 { 296 | log.Printf("[warn] [%s] All events were not matched for any targets. Ignored.", msgId) 297 | } else { 298 | log.Printf("[info] [%s] %d actions completed.", msgId, n) 299 | } 300 | } 301 | return nil 302 | } 303 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package rin 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "net/url" 10 | "regexp" 11 | "strconv" 12 | "strings" 13 | 14 | "github.com/aws/aws-sdk-go-v2/aws" 15 | awsConfig "github.com/aws/aws-sdk-go-v2/config" 16 | "github.com/aws/aws-sdk-go-v2/feature/s3/manager" 17 | "github.com/aws/aws-sdk-go-v2/service/s3" 18 | "github.com/lib/pq" 19 | 20 | goconfig "github.com/kayac/go-config" 21 | ) 22 | 23 | const ( 24 | S3URITemplate = "s3://%s/%s" 25 | SQLTemplate = "/* Rin */ COPY %s FROM %s CREDENTIALS '%s' REGION '%s' %s" 26 | // Prefix SQL comment "/* Rin */". Because a query which start with "COPY", pq expect a PostgreSQL COPY command response, but a Redshift response is different it. 27 | 28 | DriverPostgres = "postgres" 29 | DriverRedshiftData = "redshift-data" 30 | ) 31 | 32 | func quoteValue(v string) string { 33 | return "'" + strings.Replace(v, "'", "''", -1) + "'" 34 | } 35 | 36 | type Config struct { 37 | QueueName string `yaml:"queue_name"` 38 | Targets []*Target `yaml:"targets"` 39 | Credentials Credentials `yaml:"credentials"` 40 | Redshift *Redshift `yaml:"redshift"` 41 | S3 *S3 `yaml:"s3"` 42 | SQLOption string `yaml:"sql_option"` 43 | } 44 | 45 | type Credentials struct { 46 | AWS_ACCESS_KEY_ID string `yaml:"aws_access_key_id"` 47 | AWS_SECRET_ACCESS_KEY string `yaml:"aws_secret_access_key"` 48 | AWS_REGION string `yaml:"aws_region"` 49 | AWS_IAM_ROLE string `yaml:"aws_iam_role"` 50 | } 51 | 52 | func (c Credentials) RedshiftCredential() string { 53 | if c.AWS_IAM_ROLE != "" { 54 | return fmt.Sprintf("aws_iam_role=%s", c.AWS_IAM_ROLE) 55 | } else { 56 | return fmt.Sprintf("aws_access_key_id=%s;aws_secret_access_key=%s", c.AWS_ACCESS_KEY_ID, c.AWS_SECRET_ACCESS_KEY) 57 | } 58 | } 59 | 60 | type Target struct { 61 | Redshift *Redshift `yaml:"redshift"` 62 | S3 *S3 `yaml:"s3"` 63 | SQLOption string `yaml:"sql_option"` 64 | Break bool `yaml:"break"` 65 | Discard bool `yaml:"discard"` 66 | 67 | keyMatcher func(string) (bool, *[]string) 68 | } 69 | 70 | type SQLParam struct { 71 | Table string 72 | Option string 73 | } 74 | 75 | func (t *Target) String() string { 76 | var s string 77 | if t.Discard { 78 | s = strings.Join([]string{t.S3.String(), "Discard"}, " => ") 79 | } else { 80 | s = strings.Join([]string{t.S3.String(), t.Redshift.String()}, " => ") 81 | } 82 | if t.Break { 83 | s = s + " => Break" 84 | } 85 | return s 86 | } 87 | 88 | func (t *Target) Match(bucket, key string) (bool, *[]string) { 89 | if bucket != t.S3.Bucket { 90 | return false, nil 91 | } 92 | return t.keyMatcher(key) 93 | } 94 | 95 | func (t *Target) MatchEventRecord(r *EventRecord) (bool, *[]string) { 96 | return t.Match(r.S3.Bucket.Name, r.S3.Object.Key) 97 | } 98 | 99 | func (t *Target) buildKeyMatcher() error { 100 | if prefix := t.S3.KeyPrefix; prefix != "" { 101 | t.keyMatcher = func(key string) (bool, *[]string) { 102 | if strings.HasPrefix(key, prefix) { 103 | capture := []string{key} 104 | return true, &capture 105 | } else { 106 | return false, nil 107 | } 108 | } 109 | } else if r := t.S3.KeyRegexp; r != "" { 110 | reg, err := regexp.Compile(r) 111 | if err != nil { 112 | return err 113 | } 114 | t.keyMatcher = func(key string) (bool, *[]string) { 115 | capture := reg.FindStringSubmatch(key) 116 | if len(capture) == 0 { 117 | return false, nil 118 | } else { 119 | return true, &capture 120 | } 121 | } 122 | } else { 123 | return fmt.Errorf("target.key_prefix or key_regexp is not defined") 124 | } 125 | return nil 126 | } 127 | 128 | func expandPlaceHolder(s string, capture *[]string) string { 129 | for i, v := range *capture { 130 | s = strings.Replace(s, "$"+strconv.Itoa(i), v, -1) 131 | } 132 | return s 133 | } 134 | 135 | func (t *Target) BuildCopySQL(key string, cred Credentials, capture *[]string) (string, error) { 136 | var table string 137 | _table := expandPlaceHolder(t.Redshift.Table, capture) 138 | if t.Redshift.Schema == "" { 139 | table = pq.QuoteIdentifier(_table) 140 | } else { 141 | _schema := expandPlaceHolder(t.Redshift.Schema, capture) 142 | table = pq.QuoteIdentifier(_schema) + "." + pq.QuoteIdentifier(_table) 143 | } 144 | query := fmt.Sprintf( 145 | SQLTemplate, 146 | table, 147 | quoteValue(fmt.Sprintf(S3URITemplate, t.S3.Bucket, key)), 148 | cred.RedshiftCredential(), 149 | t.S3.Region, 150 | t.SQLOption, 151 | ) 152 | return query, nil 153 | } 154 | 155 | type S3 struct { 156 | Region string `yaml:"region"` 157 | Bucket string `yaml:"bucket"` 158 | KeyPrefix string `yaml:"key_prefix"` 159 | KeyRegexp string `yaml:"key_regexp"` 160 | } 161 | 162 | func (s3 S3) String() string { 163 | if s3.KeyPrefix != "" { 164 | return fmt.Sprintf(S3URITemplate, s3.Bucket, s3.KeyPrefix) 165 | } else { 166 | return fmt.Sprintf(S3URITemplate, s3.Bucket, s3.KeyRegexp) 167 | } 168 | } 169 | 170 | type Redshift struct { 171 | Driver string `yaml:"driver"` 172 | 173 | // for postgres driver 174 | Host string `yaml:"host"` 175 | Port int `yaml:"port"` 176 | 177 | // for redshift-data driver provisioned 178 | Cluster string `yaml:"cluster"` 179 | 180 | // for redshift-data driver serverless 181 | Workgroup string `yaml:"workgroup"` 182 | 183 | DBName string `yaml:"dbname"` 184 | User string `yaml:"user"` 185 | Password string `yaml:"password"` 186 | Schema string `yaml:"schema"` 187 | Table string `yaml:"table"` 188 | ReconnectOnError *bool `yaml:"reconnect_on_error"` 189 | } 190 | 191 | func (r Redshift) UseTransaction() bool { 192 | // redshift-data driver does not support transaction 193 | // https://github.com/mashiike/redshift-data-sql-driver#unsupported-features 194 | return r.Driver == DriverPostgres 195 | } 196 | 197 | func (r Redshift) DSN() string { 198 | return r.DSNWith(r.User, r.Password) 199 | } 200 | 201 | func (r Redshift) DSNWith(user string, password string) string { 202 | switch r.Driver { 203 | case DriverPostgres: 204 | var p string 205 | if password != "" { 206 | p = url.QueryEscape(password) 207 | } else { 208 | p = "****" 209 | } 210 | return fmt.Sprintf("postgres://%s:%s@%s:%d/%s", 211 | url.QueryEscape(user), p, 212 | url.QueryEscape(r.Host), r.Port, url.QueryEscape(r.DBName), 213 | ) 214 | case DriverRedshiftData: 215 | if r.Workgroup != "" { 216 | return fmt.Sprintf("workgroup(%s)/%s", 217 | url.QueryEscape(r.Workgroup), 218 | url.QueryEscape(r.DBName), 219 | ) 220 | } else if r.Cluster != "" { 221 | return fmt.Sprintf("%s@cluster(%s)/%s", 222 | url.QueryEscape(user), 223 | url.QueryEscape(r.Cluster), 224 | url.QueryEscape(r.DBName), 225 | ) 226 | } 227 | panic("redshift-data driver requires workgroup or cluster") 228 | default: 229 | panic("unknown driver: " + r.Driver) 230 | } 231 | } 232 | 233 | func (r Redshift) VisibleDSN() string { 234 | if r.Driver == DriverPostgres { 235 | return strings.Replace(r.DSNWith(r.User, ""), "postgres://", "redshift://", 1) 236 | } else { 237 | return r.Driver + "://" + r.DSN() 238 | } 239 | } 240 | 241 | func (r Redshift) String() string { 242 | if r.Schema == "" { 243 | return fmt.Sprintf("%s/public.%s", r.VisibleDSN(), r.Table) 244 | } else { 245 | return fmt.Sprintf("%s/%s.%s", r.VisibleDSN(), r.Schema, r.Table) 246 | } 247 | } 248 | 249 | func loadSrcFrom(ctx context.Context, path string) ([]byte, error) { 250 | u, err := url.Parse(path) 251 | if err != nil { 252 | // not a URL. load as a file path 253 | return ioutil.ReadFile(path) 254 | } 255 | switch u.Scheme { 256 | case "http", "https": 257 | return fetchHTTP(ctx, u) 258 | case "s3": 259 | return fetchS3(ctx, u) 260 | case "file", "": 261 | return ioutil.ReadFile(u.Path) 262 | default: 263 | return nil, fmt.Errorf("scheme %s is not supported", u.Scheme) 264 | } 265 | } 266 | 267 | func fetchHTTP(ctx context.Context, u *url.URL) ([]byte, error) { 268 | log.Println("[info] fetching from", u) 269 | req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil) 270 | if err != nil { 271 | return nil, err 272 | } 273 | resp, err := http.DefaultClient.Do(req) 274 | if err != nil { 275 | return nil, err 276 | } 277 | defer resp.Body.Close() 278 | return ioutil.ReadAll(resp.Body) 279 | } 280 | 281 | func fetchS3(ctx context.Context, u *url.URL) ([]byte, error) { 282 | log.Println("[info] fetching from", u) 283 | var s3Svc *s3.Client 284 | if Sessions.S3 == nil { 285 | awsCfg, err := awsConfig.LoadDefaultConfig(ctx) 286 | if err != nil { 287 | return nil, fmt.Errorf("failed to load default aws config, %w", err) 288 | } 289 | s3Svc = s3.NewFromConfig(awsCfg) 290 | } else { 291 | s3Svc = s3.NewFromConfig(*Sessions.S3, Sessions.S3OptFns...) 292 | } 293 | bucket := u.Host 294 | key := strings.TrimLeft(u.Path, "/") 295 | headObject, err := s3Svc.HeadObject(ctx, &s3.HeadObjectInput{ 296 | Bucket: aws.String(bucket), 297 | Key: aws.String(key), 298 | }) 299 | if err != nil { 300 | return nil, fmt.Errorf("failed to head object from S3, %w", err) 301 | } 302 | buf := make([]byte, int(headObject.ContentLength)) 303 | w := manager.NewWriteAtBuffer(buf) 304 | downloader := manager.NewDownloader(s3Svc) 305 | _, err = downloader.Download(ctx, w, &s3.GetObjectInput{ 306 | Bucket: aws.String(bucket), 307 | Key: aws.String(key), 308 | }) 309 | if err != nil { 310 | return nil, fmt.Errorf("failed to fetch from S3, %s", err) 311 | } 312 | return buf, nil 313 | } 314 | 315 | func LoadConfig(ctx context.Context, path string) (*Config, error) { 316 | src, err := loadSrcFrom(ctx, path) 317 | if err != nil { 318 | return nil, err 319 | } 320 | var c Config = Config{ 321 | Redshift: &Redshift{ 322 | Driver: DriverPostgres, // default 323 | }, 324 | } 325 | err = goconfig.LoadWithEnvBytes(&c, src) 326 | if err != nil { 327 | return nil, err 328 | } 329 | err = (&c).merge() 330 | if err != nil { 331 | return nil, err 332 | } 333 | return &c, (&c).validate() 334 | } 335 | 336 | func (c *Config) validate() error { 337 | switch c.Redshift.Driver { 338 | case DriverPostgres, DriverRedshiftData: // ok 339 | log.Println("[debug] redshift.driver is", c.Redshift.Driver) 340 | default: 341 | return fmt.Errorf("invalid redshift.driver must be %s or %s", DriverPostgres, DriverRedshiftData) 342 | } 343 | if c.QueueName == "" { 344 | if !isLambda() { 345 | return fmt.Errorf("queue_name required") 346 | } 347 | } 348 | if len(c.Targets) == 0 { 349 | return fmt.Errorf("no targets defined") 350 | } 351 | return nil 352 | } 353 | 354 | func (c *Config) merge() error { 355 | cr := c.Redshift 356 | cs := c.S3 357 | for _, t := range c.Targets { 358 | if t.SQLOption == "" { 359 | t.SQLOption = c.SQLOption 360 | } 361 | tr := t.Redshift 362 | if tr == nil { 363 | t.Redshift = cr 364 | } else { 365 | if tr.Host == "" { 366 | tr.Host = cr.Host 367 | } 368 | if tr.Port == 0 { 369 | tr.Port = cr.Port 370 | } 371 | if tr.DBName == "" { 372 | tr.DBName = cr.DBName 373 | } 374 | if tr.User == "" { 375 | tr.User = cr.User 376 | } 377 | if tr.Password == "" { 378 | tr.Password = cr.Password 379 | } 380 | if tr.Schema == "" { 381 | tr.Schema = cr.Schema 382 | } 383 | if tr.Table == "" { 384 | tr.Table = cr.Table 385 | } 386 | if tr.ReconnectOnError == nil { 387 | tr.ReconnectOnError = cr.ReconnectOnError 388 | } 389 | if tr.Driver == "" { 390 | tr.Driver = cr.Driver 391 | } 392 | if tr.Workgroup == "" { 393 | tr.Workgroup = cr.Workgroup 394 | } 395 | if tr.Cluster == "" { 396 | tr.Cluster = cr.Cluster 397 | } 398 | } 399 | 400 | ts := t.S3 401 | if ts == nil { 402 | t.S3 = cs 403 | } else { 404 | if ts.Bucket == "" { 405 | ts.Bucket = cs.Bucket 406 | } 407 | if ts.Region == "" { 408 | ts.Region = cs.Region 409 | } 410 | if ts.KeyPrefix == "" { 411 | ts.KeyPrefix = cs.KeyPrefix 412 | } 413 | if ts.KeyRegexp == "" { 414 | ts.KeyRegexp = cs.KeyRegexp 415 | } 416 | } 417 | err := t.buildKeyMatcher() 418 | if err != nil { 419 | return err 420 | } 421 | } 422 | return nil 423 | } 424 | --------------------------------------------------------------------------------