├── .github └── workflows │ └── test.yml ├── .gitignore ├── Makefile ├── README.md ├── go.mod ├── go.sum ├── main.go └── zipserver ├── .gitignore ├── archive.go ├── archive_test.go ├── config.go ├── config_test.go ├── copy_handler.go ├── extract_handler.go ├── extract_handler_test.go ├── gcs_storage.go ├── gcs_storage_test.go ├── list_handler.go ├── lock_table.go ├── lock_table_test.go ├── mem_storage.go ├── metrics.go ├── metrics_test.go ├── readers.go ├── readers_test.go ├── s3_storage.go ├── serve_zip.go ├── server.go ├── slurp_handler.go ├── specifications.go └── storage.go /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: "test" 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | 11 | - name: Set up Go 12 | uses: actions/setup-go@v4 13 | with: 14 | go-version: 1.19 15 | 16 | - name: Build 17 | run: go build -v -o zipserver.bin 18 | 19 | - name: Test 20 | run: go test -v -race ./... 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | test_files/ 3 | zipserver.json 4 | *.pem -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | .PHONY: install test 3 | 4 | build: 5 | go build -o bin/zipserver 6 | 7 | install: 8 | go install github.com/itchio/zipserver 9 | 10 | test: 11 | go test -v github.com/itchio/zipserver/zipserver 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![test](https://github.com/itchio/zipserver/actions/workflows/test.yml/badge.svg)](https://github.com/itchio/zipserver/actions/workflows/test.yml) 2 | 3 | # zipserver 4 | 5 | Zip server is an HTTP service that takes a key to a zip file on Google Cloud 6 | storage, extracts it, then reuploads the individual files to a specified 7 | prefix. It can restrict extraction of the zip file based on individual file 8 | size, total file size, or number of files. 9 | 10 | 11 | ## Usage 12 | 13 | Install 14 | 15 | ```bash 16 | go install github.com/itchio/zipserver@latest 17 | 18 | zipserver -help 19 | ``` 20 | 21 | Create a config file: 22 | 23 | `zipserver.json`: 24 | 25 | ```json 26 | { 27 | "PrivateKeyPath": "path/to/service/key.pem", 28 | "ClientEmail": "111111111111@developer.gserviceaccount.com" 29 | } 30 | ``` 31 | 32 | More config settings can be found in `config.go` 33 | 34 | Run: 35 | 36 | ```bash 37 | $GOPATH/bin/zipserver 38 | ``` 39 | 40 | Extract a zip file: 41 | 42 | ```bash 43 | curl http://localhost:8090/extract?key=zips/my_file.zip&prefix=extracted 44 | ``` 45 | 46 | 47 | ## Slurping 48 | 49 | You can tell the zip server to download a file from a URL. This can be used to 50 | load a zip file you want to extract later. 51 | 52 | ```bash 53 | curl http://localhost:8090/slurp?key=myfile.zip&url=http://leafo.net/file.zip 54 | ``` 55 | 56 | ## GCS authentication and permissions 57 | 58 | The key file in your config should be the PEM-encoded private key for a 59 | service account which has permissions to view and create objects on your 60 | chosen GCS bucket. 61 | 62 | The bucket needs correct access settings: 63 | 64 | - Public access must be enabled, not prevented. 65 | - Access control should be set to fine-grained ("legacy ACL"), not uniform. 66 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/itchio/zipserver 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go v1.55.5 7 | github.com/go-errors/errors v1.5.1 8 | github.com/stretchr/testify v1.10.0 9 | golang.org/x/oauth2 v0.24.0 10 | ) 11 | 12 | require ( 13 | cloud.google.com/go v0.65.0 // indirect 14 | github.com/davecgh/go-spew v1.1.1 // indirect 15 | github.com/jmespath/go-jmespath v0.4.0 // indirect 16 | github.com/pmezard/go-difflib v1.0.0 // indirect 17 | gopkg.in/yaml.v3 v3.0.1 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 4 | cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= 5 | cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= 6 | cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= 7 | cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= 8 | cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= 9 | cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= 10 | cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= 11 | cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= 12 | cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= 13 | cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= 14 | cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= 15 | cloud.google.com/go v0.65.0 h1:Dg9iHVQfrhq82rUNu9ZxUDrJLaxFUe/HlCVaLyRruq8= 16 | cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= 17 | cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= 18 | cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= 19 | cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= 20 | cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= 21 | cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= 22 | cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= 23 | cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= 24 | cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= 25 | cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= 26 | cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= 27 | cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= 28 | cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= 29 | cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= 30 | cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= 31 | cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= 32 | cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= 33 | cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= 34 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 35 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 36 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 37 | github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= 38 | github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= 39 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 40 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 41 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 42 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 43 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 44 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 45 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 46 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 47 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 48 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 49 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 50 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 51 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 52 | github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= 53 | github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= 54 | github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= 55 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 56 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 57 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 58 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 59 | github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 60 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 61 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 62 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 63 | github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= 64 | github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 65 | github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 66 | github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= 67 | github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= 68 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 69 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 70 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 71 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 72 | github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 73 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= 74 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 75 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 76 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 77 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 78 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 79 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 80 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 81 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 82 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 83 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 84 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 85 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 86 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 87 | github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 88 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 89 | github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 90 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 91 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 92 | github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= 93 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 94 | github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 95 | github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 96 | github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 97 | github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 98 | github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 99 | github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= 100 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 101 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 102 | github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= 103 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 104 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 105 | github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 106 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 107 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 108 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 109 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 110 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 111 | github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= 112 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 113 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 114 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 115 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 116 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 117 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 118 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 119 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 120 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 121 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 122 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 123 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 124 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 125 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 126 | github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 127 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 128 | github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 129 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 130 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 131 | go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 132 | go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 133 | go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 134 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 135 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 136 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 137 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 138 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 139 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 140 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 141 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 142 | golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= 143 | golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= 144 | golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 145 | golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 146 | golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 147 | golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= 148 | golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= 149 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 150 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 151 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 152 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 153 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 154 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 155 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 156 | golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 157 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 158 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= 159 | golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 160 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 161 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 162 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 163 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 164 | golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= 165 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 166 | golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 167 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 168 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 169 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 170 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 171 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 172 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 173 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 174 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 175 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 176 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 177 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 178 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 179 | golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 180 | golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 181 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 182 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 183 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 184 | golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 185 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 186 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 187 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 188 | golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 189 | golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 190 | golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 191 | golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 192 | golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 193 | golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 194 | golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 195 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 196 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 197 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 198 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 199 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 200 | golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= 201 | golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= 202 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 203 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 204 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 205 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 206 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 207 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 208 | golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 209 | golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 210 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 211 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 212 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 213 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 214 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 215 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 216 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 217 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 218 | golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 219 | golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 220 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 221 | golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 222 | golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 223 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 224 | golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 225 | golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 226 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 227 | golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 228 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 229 | golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 230 | golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 231 | golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 232 | golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 233 | golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 234 | golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 235 | golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 236 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 237 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 238 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 239 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 240 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 241 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 242 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 243 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 244 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 245 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 246 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 247 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 248 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 249 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 250 | golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 251 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 252 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 253 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 254 | golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 255 | golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 256 | golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 257 | golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 258 | golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 259 | golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 260 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 261 | golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 262 | golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 263 | golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 264 | golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 265 | golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 266 | golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 267 | golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 268 | golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 269 | golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 270 | golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 271 | golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 272 | golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 273 | golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= 274 | golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= 275 | golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= 276 | golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 277 | golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 278 | golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 279 | golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 280 | golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= 281 | golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= 282 | golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= 283 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 284 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 285 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 286 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 287 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 288 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= 289 | google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 290 | google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 291 | google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 292 | google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 293 | google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 294 | google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 295 | google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 296 | google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 297 | google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 298 | google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= 299 | google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= 300 | google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= 301 | google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= 302 | google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= 303 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 304 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 305 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 306 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 307 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 308 | google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 309 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 310 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 311 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 312 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 313 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 314 | google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 315 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 316 | google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= 317 | google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 318 | google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 319 | google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 320 | google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 321 | google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 322 | google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 323 | google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= 324 | google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 325 | google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 326 | google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 327 | google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 328 | google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 329 | google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 330 | google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 331 | google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 332 | google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= 333 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 334 | google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= 335 | google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 336 | google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 337 | google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= 338 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 339 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 340 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 341 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 342 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 343 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 344 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 345 | google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 346 | google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= 347 | google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= 348 | google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 349 | google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= 350 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 351 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 352 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 353 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 354 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 355 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 356 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 357 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 358 | google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= 359 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 360 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 361 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 362 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 363 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 364 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 365 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 366 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 367 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 368 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 369 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 370 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 371 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 372 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 373 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 374 | honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= 375 | honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= 376 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 377 | rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= 378 | rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= 379 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "math/rand" 10 | "time" 11 | 12 | "github.com/go-errors/errors" 13 | "github.com/itchio/zipserver/zipserver" 14 | ) 15 | 16 | var _ fmt.Formatter 17 | 18 | var ( 19 | configFname string 20 | listenTo string 21 | dumpConfig bool 22 | serve string 23 | extract string 24 | ) 25 | 26 | func init() { 27 | flag.StringVar(&configFname, "config", zipserver.DefaultConfigFname, "Path to json config file") 28 | flag.StringVar(&listenTo, "listen", "127.0.0.1:8090", "Address to listen to") 29 | flag.BoolVar(&dumpConfig, "dump", false, "Dump the parsed config and exit") 30 | flag.StringVar(&serve, "serve", "", "Serve a given zip from a local HTTP server") 31 | flag.StringVar(&extract, "extract", "", "Extract zip file to random name on GCS (requires a config with bucket)") 32 | } 33 | 34 | func must(err error) { 35 | if err == nil { 36 | return 37 | } 38 | 39 | if se, ok := err.(*errors.Error); ok { 40 | log.Fatal(se.ErrorStack()) 41 | } else { 42 | log.Fatal(err.Error()) 43 | } 44 | } 45 | 46 | func main() { 47 | flag.Parse() 48 | 49 | config, err := zipserver.LoadConfig(configFname) 50 | must(err) 51 | 52 | if dumpConfig { 53 | fmt.Println(config) 54 | return 55 | } 56 | 57 | if serve != "" { 58 | must(zipserver.ServeZip(config, serve)) 59 | return 60 | } 61 | 62 | if extract != "" { 63 | archiver := zipserver.NewArchiver(config) 64 | limits := zipserver.DefaultExtractLimits(config) 65 | 66 | log.Println("Extraction threads:", limits.ExtractionThreads) 67 | log.Println("Bucket:", config.Bucket) 68 | 69 | rand.Seed(time.Now().UTC().UnixNano()) 70 | 71 | var letters = []rune("rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz") 72 | 73 | var randChars = make([]rune, 20) 74 | for i := range randChars { 75 | randChars[i] = letters[rand.Intn(len(letters))] 76 | } 77 | 78 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(config.JobTimeout)) 79 | defer cancel() 80 | 81 | files, err := archiver.UploadZipFromFile(ctx, extract, string(randChars), limits) 82 | if err != nil { 83 | log.Fatal(err.Error()) 84 | return 85 | } 86 | 87 | blob, _ := json.Marshal(struct { 88 | Success bool 89 | ExtractedFiles []zipserver.ExtractedFile 90 | }{true, files}) 91 | 92 | fmt.Println(string(blob)) 93 | return 94 | } 95 | 96 | err = zipserver.StartZipServer(listenTo, config) 97 | must(err) 98 | } 99 | -------------------------------------------------------------------------------- /zipserver/.gitignore: -------------------------------------------------------------------------------- 1 | /zip_tmp 2 | -------------------------------------------------------------------------------- /zipserver/archive.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "crypto/md5" 7 | "encoding/hex" 8 | "fmt" 9 | "io" 10 | "log" 11 | "mime" 12 | "net/http" 13 | "os" 14 | "path" 15 | "strings" 16 | "time" 17 | 18 | "archive/zip" 19 | 20 | errors "github.com/go-errors/errors" 21 | ) 22 | 23 | var ( 24 | tmpDir = "zip_tmp" 25 | ) 26 | 27 | func init() { 28 | mime.AddExtensionType(".unityweb", "application/octet-stream") 29 | mime.AddExtensionType(".wasm", "application/wasm") 30 | mime.AddExtensionType(".data", "application/octet-stream") // modern unity data file 31 | mime.AddExtensionType(".ico", "image/x-icon") // prevent image/vnd.microsoft.icon 32 | } 33 | 34 | // Archiver holds together the storage along with configuration values 35 | // (credentials, limits etc.) 36 | type Archiver struct { 37 | Storage 38 | *Config 39 | } 40 | 41 | // ExtractedFile represents a file extracted from a .zip into a GCS bucket 42 | type ExtractedFile struct { 43 | Key string 44 | Size uint64 45 | } 46 | 47 | // NewArchiver creates a new archiver from the given config 48 | func NewArchiver(config *Config) *Archiver { 49 | storage, err := NewGcsStorage(config) 50 | 51 | if storage == nil { 52 | log.Fatal("Failed to create storage:", err) 53 | } 54 | 55 | return &Archiver{storage, config} 56 | } 57 | 58 | func fetchZipFilename(bucket, key string) string { 59 | hasher := md5.New() 60 | hasher.Write([]byte(key)) 61 | return bucket + "_" + hex.EncodeToString(hasher.Sum(nil)) + ".zip" 62 | } 63 | 64 | func (a *Archiver) fetchZip(ctx context.Context, key string) (string, error) { 65 | os.MkdirAll(tmpDir, os.ModeDir|0777) 66 | 67 | fname := fetchZipFilename(a.Bucket, key) 68 | fname = path.Join(tmpDir, fname) 69 | 70 | src, _, err := a.Storage.GetFile(ctx, a.Bucket, key) 71 | if err != nil { 72 | return "", errors.Wrap(err, 0) 73 | } 74 | 75 | defer src.Close() 76 | 77 | dest, err := os.Create(fname) 78 | if err != nil { 79 | return "", errors.Wrap(err, 0) 80 | } 81 | _, err = os.Stat(fname) 82 | 83 | defer func() { 84 | dest.Close() 85 | // Clean up if io.Copy below errs. 86 | if err != nil { 87 | os.Remove(fname) 88 | } 89 | }() 90 | 91 | _, err = io.Copy(dest, src) 92 | if err != nil { 93 | return "", errors.Wrap(err, 0) 94 | } 95 | 96 | return fname, nil 97 | } 98 | 99 | // delete all files that have been uploaded so far 100 | func (a *Archiver) abortUpload(files []ExtractedFile) error { 101 | for _, file := range files { 102 | // FIXME: code quality - what if we fail here? any retry strategies? 103 | ctx := context.Background() 104 | a.Storage.DeleteFile(ctx, a.Bucket, file.Key) 105 | } 106 | 107 | return nil 108 | } 109 | 110 | func shouldIgnoreFile(fname string) bool { 111 | if strings.HasSuffix(fname, "/") { 112 | return true 113 | } 114 | 115 | if strings.Contains(fname, "..") { 116 | return true 117 | } 118 | 119 | if strings.Contains(fname, "__MACOSX/") { 120 | return true 121 | } 122 | 123 | if strings.Contains(fname, ".git/") { 124 | return true 125 | } 126 | 127 | if path.IsAbs(fname) { 128 | return true 129 | } 130 | 131 | return false 132 | } 133 | 134 | // UploadFileTask contains the information needed to extract a single file from a .zip 135 | type UploadFileTask struct { 136 | File *zip.File 137 | Key string 138 | } 139 | 140 | // UploadFileResult is successful is Error is nil - in that case, it contains the 141 | // GCS key the file was uploaded under, and the number of bytes written for that file. 142 | type UploadFileResult struct { 143 | Error error 144 | Key string 145 | Size uint64 146 | } 147 | 148 | func uploadWorker( 149 | ctx context.Context, 150 | a *Archiver, 151 | tasks <-chan UploadFileTask, 152 | results chan<- UploadFileResult, 153 | done chan struct{}, 154 | ) { 155 | defer func() { done <- struct{}{} }() 156 | 157 | for task := range tasks { 158 | file := task.File 159 | key := task.Key 160 | 161 | ctx, cancel := context.WithTimeout(ctx, time.Duration(a.Config.FilePutTimeout)) 162 | resource, err := a.extractAndUploadOne(ctx, key, file) 163 | cancel() // Free resources now instead of deferring till func returns 164 | 165 | if err != nil { 166 | log.Print("Failed sending " + key + ": " + err.Error()) 167 | results <- UploadFileResult{err, key, 0} 168 | return 169 | } 170 | 171 | results <- UploadFileResult{nil, resource.key, resource.size} 172 | } 173 | } 174 | 175 | // extracts and sends all files to prefix 176 | func (a *Archiver) sendZipExtracted( 177 | ctx context.Context, 178 | prefix, fname string, 179 | limits *ExtractLimits, 180 | ) ([]ExtractedFile, error) { 181 | zipReader, err := zip.OpenReader(fname) 182 | if err != nil { 183 | return nil, errors.Wrap(err, 0) 184 | } 185 | 186 | defer zipReader.Close() 187 | 188 | if len(zipReader.File) > limits.MaxNumFiles { 189 | err := fmt.Errorf("Too many files in zip (%v > %v)", 190 | len(zipReader.File), limits.MaxNumFiles) 191 | return nil, errors.Wrap(err, 0) 192 | } 193 | 194 | extractedFiles := []ExtractedFile{} 195 | 196 | fileCount := 0 197 | var byteCount uint64 198 | 199 | fileList := []*zip.File{} 200 | 201 | for _, file := range zipReader.File { 202 | if shouldIgnoreFile(file.Name) { 203 | log.Printf("Ignoring file %s", file.Name) 204 | continue 205 | } 206 | 207 | if len(file.Name) > limits.MaxFileNameLength { 208 | err := fmt.Errorf("Zip contains file paths that are too long") 209 | return nil, errors.Wrap(err, 0) 210 | } 211 | 212 | if file.UncompressedSize64 > limits.MaxFileSize { 213 | err := fmt.Errorf("Zip contains file that is too large (%s)", file.Name) 214 | return nil, errors.Wrap(err, 0) 215 | } 216 | 217 | byteCount += file.UncompressedSize64 218 | 219 | if byteCount > limits.MaxTotalSize { 220 | err := fmt.Errorf("Extracted zip too large (max %v bytes)", limits.MaxTotalSize) 221 | return nil, errors.Wrap(err, 0) 222 | } 223 | 224 | fileList = append(fileList, file) 225 | } 226 | 227 | tasks := make(chan UploadFileTask) 228 | results := make(chan UploadFileResult) 229 | done := make(chan struct{}, limits.ExtractionThreads) 230 | 231 | // Context can be canceled by caller or when an individual task fails. 232 | ctx, cancel := context.WithCancel(ctx) 233 | defer cancel() 234 | 235 | for i := 0; i < limits.ExtractionThreads; i++ { 236 | go uploadWorker(ctx, a, tasks, results, done) 237 | } 238 | 239 | activeWorkers := limits.ExtractionThreads 240 | 241 | go func() { 242 | defer func() { close(tasks) }() 243 | for _, file := range fileList { 244 | key := path.Join(prefix, file.Name) 245 | task := UploadFileTask{file, key} 246 | select { 247 | case tasks <- task: 248 | case <-ctx.Done(): 249 | // Something went wrong! 250 | log.Println("Remaining tasks were canceled") 251 | return 252 | } 253 | } 254 | }() 255 | 256 | var extractError error 257 | 258 | for activeWorkers > 0 { 259 | select { 260 | case result := <-results: 261 | if result.Error != nil { 262 | extractError = result.Error 263 | cancel() 264 | } else { 265 | extractedFiles = append(extractedFiles, ExtractedFile{result.Key, result.Size}) 266 | fileCount++ 267 | } 268 | case <-done: 269 | activeWorkers-- 270 | } 271 | } 272 | 273 | close(results) 274 | 275 | if extractError != nil { 276 | log.Printf("Upload error: %s", extractError.Error()) 277 | a.abortUpload(extractedFiles) 278 | return nil, extractError 279 | } 280 | 281 | log.Printf("Sent %d files", fileCount) 282 | return extractedFiles, nil 283 | } 284 | 285 | // sends an individual file from a zip 286 | // Caller should set the job timeout in ctx. 287 | func (a *Archiver) extractAndUploadOne(ctx context.Context, key string, file *zip.File) (*ResourceSpec, error) { 288 | readerCloser, err := file.Open() 289 | if err != nil { 290 | return nil, err 291 | } 292 | defer readerCloser.Close() 293 | 294 | var reader io.Reader = readerCloser 295 | 296 | resource := &ResourceSpec{ 297 | key: key, 298 | } 299 | 300 | // try determining MIME by extension 301 | mimeType := mime.TypeByExtension(path.Ext(key)) 302 | 303 | var buffer bytes.Buffer 304 | _, err = io.Copy(&buffer, io.LimitReader(reader, 512)) 305 | 306 | if err != nil { 307 | return nil, errors.Wrap(err, 0) 308 | } 309 | 310 | contentMimeType := http.DetectContentType(buffer.Bytes()) 311 | // join the bytes read and the original reader 312 | reader = io.MultiReader(&buffer, reader) 313 | 314 | if contentMimeType == "application/x-gzip" || contentMimeType == "application/gzip" { 315 | resource.contentEncoding = "gzip" 316 | 317 | // try to see if there's a real extension hidden beneath 318 | if strings.HasSuffix(key, ".gz") { 319 | realMimeType := mime.TypeByExtension(path.Ext(strings.TrimSuffix(key, ".gz"))) 320 | 321 | if realMimeType != "" { 322 | mimeType = realMimeType 323 | } 324 | } 325 | 326 | } else if strings.HasSuffix(key, ".br") { 327 | // there is no way to detect a brotli stream by content, so we assume if it ends if .br then it's brotli 328 | // this path is used for Unity 2020 webgl games built with brotli compression 329 | resource.contentEncoding = "br" 330 | realMimeType := mime.TypeByExtension(path.Ext(strings.TrimSuffix(key, ".br"))) 331 | 332 | if realMimeType != "" { 333 | mimeType = realMimeType 334 | } 335 | } else if mimeType == "" { 336 | // fall back to the extension detected from content, eg. someone uploaded a .png with wrong extension 337 | mimeType = contentMimeType 338 | } 339 | 340 | if mimeType == "" { 341 | // default mime type 342 | mimeType = "application/octet-stream" 343 | } 344 | resource.contentType = mimeType 345 | 346 | resource.applyRewriteRules() 347 | 348 | log.Printf("Sending: %s", resource) 349 | 350 | limited := limitedReader(reader, file.UncompressedSize64, &resource.size) 351 | 352 | err = a.Storage.PutFileWithSetup(ctx, a.Bucket, resource.key, limited, resource.setupRequest) 353 | if err != nil { 354 | return resource, errors.Wrap(err, 0) 355 | } 356 | 357 | globalMetrics.TotalExtractedFiles.Add(1) 358 | 359 | return resource, nil 360 | } 361 | 362 | // ExtractZip downloads the zip at `key` to a temporary file, 363 | // then extracts its contents and uploads each item to `prefix` 364 | // Caller should set the job timeout in ctx. 365 | func (a *Archiver) ExtractZip( 366 | ctx context.Context, 367 | key, prefix string, 368 | limits *ExtractLimits, 369 | ) ([]ExtractedFile, error) { 370 | fname, err := a.fetchZip(ctx, key) 371 | if err != nil { 372 | return nil, err 373 | } 374 | 375 | defer os.Remove(fname) 376 | prefix = path.Join(a.ExtractPrefix, prefix) 377 | return a.sendZipExtracted(ctx, prefix, fname, limits) 378 | } 379 | 380 | // Caller should set the job timeout in ctx. 381 | func (a *Archiver) UploadZipFromFile( 382 | ctx context.Context, 383 | fname, prefix string, 384 | limits *ExtractLimits, 385 | ) ([]ExtractedFile, error) { 386 | prefix = path.Join("_zipserver", prefix) 387 | return a.sendZipExtracted(ctx, prefix, fname, limits) 388 | } 389 | -------------------------------------------------------------------------------- /zipserver/archive_test.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "context" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "io/fs" 11 | "math/rand" 12 | "net/http" 13 | "os" 14 | "path/filepath" 15 | "strconv" 16 | "strings" 17 | "testing" 18 | "time" 19 | 20 | "github.com/stretchr/testify/assert" 21 | "github.com/stretchr/testify/require" 22 | ) 23 | 24 | func testLimits() *ExtractLimits { 25 | return &ExtractLimits{ 26 | MaxFileSize: 1024 * 1024 * 200, 27 | MaxTotalSize: 1024 * 1024 * 500, 28 | MaxNumFiles: 100, 29 | MaxFileNameLength: 80, 30 | ExtractionThreads: 4, 31 | } 32 | } 33 | 34 | func emptyConfig() *Config { 35 | return &Config{ 36 | Bucket: "testbucket", 37 | ExtractionThreads: 8, 38 | JobTimeout: Duration(10 * time.Second), 39 | FileGetTimeout: Duration(10 * time.Second), 40 | FilePutTimeout: Duration(10 * time.Second), 41 | } 42 | } 43 | 44 | func Test_ExtractOnGCS(t *testing.T) { 45 | withGoogleCloudStorage(t, func(storage Storage, config *Config) { 46 | ctx := context.Background() 47 | archiver := &Archiver{storage, config} 48 | 49 | r, err := os.Open("/home/leafo/code/go/etlua.zip") 50 | assert.NoError(t, err) 51 | defer r.Close() 52 | 53 | err = storage.PutFile(ctx, config.Bucket, "zipserver_test/test.zip", r, "application/zip") 54 | assert.NoError(t, err) 55 | 56 | _, err = archiver.ExtractZip(ctx, "zipserver_test/test.zip", "zipserver_test/extract", testLimits()) 57 | assert.NoError(t, err) 58 | }) 59 | } 60 | 61 | type zipEntry struct { 62 | name string 63 | outName string 64 | data []byte 65 | expectedMimeType string 66 | expectedContentEncoding string 67 | ignored bool 68 | } 69 | 70 | type zipLayout struct { 71 | entries []zipEntry 72 | } 73 | 74 | func (zl *zipLayout) Write(t *testing.T, zw *zip.Writer) { 75 | for _, entry := range zl.entries { 76 | writer, err := zw.CreateHeader(&zip.FileHeader{ 77 | Name: entry.name, 78 | UncompressedSize64: uint64(len(entry.data)), 79 | }) 80 | assert.NoError(t, err) 81 | 82 | _, err = io.Copy(writer, bytes.NewReader(entry.data)) 83 | assert.NoError(t, err) 84 | } 85 | } 86 | 87 | func (zl *zipLayout) Check(t *testing.T, storage *MemStorage, bucket, prefix string) { 88 | ctx := context.Background() 89 | 90 | for _, entry := range zl.entries { 91 | func() { 92 | name := entry.name 93 | if entry.outName != "" { 94 | name = entry.outName 95 | } 96 | 97 | path := fmt.Sprintf("%s/%s", prefix, name) 98 | reader, _, err := storage.GetFile(ctx, bucket, path) 99 | if entry.ignored { 100 | assert.Error(t, err) 101 | assert.True(t, strings.Contains(err.Error(), "object not found")) 102 | return 103 | } 104 | 105 | assert.NoError(t, err) 106 | 107 | defer reader.Close() 108 | 109 | data, err := io.ReadAll(reader) 110 | assert.NoError(t, err) 111 | assert.EqualValues(t, data, entry.data) 112 | 113 | h, err := storage.getHeaders(bucket, path) 114 | assert.NoError(t, err) 115 | assert.EqualValues(t, entry.expectedMimeType, h.Get("content-type")) 116 | assert.EqualValues(t, "public-read", h.Get("x-goog-acl")) 117 | 118 | if entry.expectedContentEncoding != "" { 119 | assert.EqualValues(t, entry.expectedContentEncoding, h.Get("content-encoding")) 120 | } 121 | }() 122 | } 123 | } 124 | 125 | func Test_ExtractInMemory(t *testing.T) { 126 | config := emptyConfig() 127 | 128 | ctx := context.Background() 129 | 130 | storage, err := NewMemStorage() 131 | assert.NoError(t, err) 132 | 133 | archiver := &Archiver{storage, config} 134 | prefix := "zipserver_test/mem_test_extracted" 135 | zipPath := "mem_test.zip" 136 | 137 | _, err = archiver.ExtractZip(ctx, zipPath, prefix, testLimits()) 138 | assert.Error(t, err) 139 | 140 | withZip := func(zl *zipLayout, cb func(zl *zipLayout)) { 141 | var buf bytes.Buffer 142 | 143 | zw := zip.NewWriter(&buf) 144 | 145 | zl.Write(t, zw) 146 | 147 | err = zw.Close() 148 | assert.NoError(t, err) 149 | 150 | err = storage.PutFile(ctx, config.Bucket, zipPath, bytes.NewReader(buf.Bytes()), "application/octet-stream") 151 | assert.NoError(t, err) 152 | 153 | cb(zl) 154 | } 155 | 156 | withZip(&zipLayout{ 157 | entries: []zipEntry{ 158 | zipEntry{ 159 | name: "file.txt", 160 | data: []byte("Hello there"), 161 | expectedMimeType: "text/plain; charset=utf-8", 162 | }, 163 | zipEntry{ 164 | name: "garbage.bin", 165 | data: bytes.Repeat([]byte{3, 1, 5, 3, 2, 6, 1, 2, 5, 3, 4, 6, 2}, 20), 166 | expectedMimeType: "application/octet-stream", 167 | }, 168 | zipEntry{ 169 | name: "something.gz", 170 | data: []byte{0x1F, 0x8B, 0x08, 1, 5, 2, 4, 9, 3, 1, 2, 5}, 171 | expectedMimeType: "application/gzip", 172 | }, 173 | zipEntry{ 174 | name: "something.unityweb", 175 | data: []byte{0x1F, 0x8B, 0x08, 9, 1, 5, 2, 3, 5, 2, 6, 4, 4}, 176 | expectedMimeType: "application/octet-stream", 177 | expectedContentEncoding: "gzip", 178 | }, 179 | zipEntry{ 180 | name: "gamedata.memgz", 181 | outName: "gamedata.mem", 182 | data: []byte{0x1F, 0x8B, 0x08, 1, 5, 2, 3, 1, 2, 1, 2}, 183 | expectedMimeType: "application/octet-stream", 184 | expectedContentEncoding: "gzip", 185 | }, 186 | zipEntry{ 187 | name: "gamedata.jsgz", 188 | outName: "gamedata.js", 189 | data: []byte{0x1F, 0x8B, 0x08, 3, 7, 3, 4, 12, 53, 26, 34}, 190 | expectedMimeType: "application/octet-stream", 191 | expectedContentEncoding: "gzip", 192 | }, 193 | zipEntry{ 194 | name: "gamedata.asm.jsgz", 195 | outName: "gamedata.asm.js", 196 | data: []byte{0x1F, 0x8B, 0x08, 62, 34, 128, 37, 10, 39, 82}, 197 | expectedMimeType: "application/octet-stream", 198 | expectedContentEncoding: "gzip", 199 | }, 200 | zipEntry{ 201 | name: "gamedata.datagz", 202 | outName: "gamedata.data", 203 | data: []byte{0x1F, 0x8B, 0x08, 8, 5, 23, 1, 25, 38}, 204 | expectedMimeType: "application/octet-stream", 205 | expectedContentEncoding: "gzip", 206 | }, 207 | zipEntry{ 208 | name: "__MACOSX/hello", 209 | data: []byte{}, 210 | ignored: true, 211 | }, 212 | zipEntry{ 213 | name: "/woops/hi/im/absolute", 214 | data: []byte{}, 215 | ignored: true, 216 | }, 217 | zipEntry{ 218 | name: "oh/hey/im/a/dir/", 219 | data: []byte{}, 220 | ignored: true, 221 | }, 222 | zipEntry{ 223 | name: "im/trying/to/escape/../../../../../../etc/hosts", 224 | data: []byte{}, 225 | ignored: true, 226 | }, 227 | }, 228 | }, func(zl *zipLayout) { 229 | _, err := archiver.ExtractZip(ctx, zipPath, prefix, testLimits()) 230 | assert.NoError(t, err) 231 | 232 | zl.Check(t, storage, config.Bucket, prefix) 233 | }) 234 | 235 | withZip(&zipLayout{ 236 | entries: []zipEntry{ 237 | zipEntry{ 238 | name: strings.Repeat("x", 101), 239 | data: []byte("uh oh"), 240 | expectedMimeType: "text/plain; charset=utf-8", 241 | }, 242 | }, 243 | }, func(zl *zipLayout) { 244 | limits := testLimits() 245 | limits.MaxFileNameLength = 100 246 | 247 | _, err := archiver.ExtractZip(ctx, zipPath, prefix, limits) 248 | assert.Error(t, err) 249 | assert.True(t, strings.Contains(err.Error(), "paths that are too long")) 250 | }) 251 | 252 | withZip(&zipLayout{ 253 | entries: []zipEntry{ 254 | zipEntry{ 255 | name: "x", 256 | data: bytes.Repeat([]byte("oh no"), 100), 257 | expectedMimeType: "text/plain; charset=utf-8", 258 | }, 259 | }, 260 | }, func(zl *zipLayout) { 261 | limits := testLimits() 262 | limits.MaxFileSize = 499 263 | 264 | _, err := archiver.ExtractZip(ctx, zipPath, prefix, limits) 265 | assert.Error(t, err) 266 | assert.True(t, strings.Contains(err.Error(), "file that is too large")) 267 | }) 268 | 269 | withZip(&zipLayout{ 270 | entries: []zipEntry{ 271 | zipEntry{ 272 | name: "1", 273 | data: []byte("uh oh"), 274 | expectedMimeType: "text/plain; charset=utf-8", 275 | }, 276 | zipEntry{ 277 | name: "2", 278 | data: []byte("uh oh"), 279 | expectedMimeType: "text/plain; charset=utf-8", 280 | }, 281 | zipEntry{ 282 | name: "3", 283 | data: []byte("uh oh"), 284 | expectedMimeType: "text/plain; charset=utf-8", 285 | }, 286 | zipEntry{ 287 | name: "4", 288 | data: []byte("uh oh"), 289 | expectedMimeType: "text/plain; charset=utf-8", 290 | }, 291 | }, 292 | }, func(zl *zipLayout) { 293 | limits := testLimits() 294 | limits.MaxNumFiles = 3 295 | 296 | _, err := archiver.ExtractZip(ctx, zipPath, prefix, limits) 297 | assert.Error(t, err) 298 | assert.True(t, strings.Contains(err.Error(), "Too many files")) 299 | }) 300 | 301 | withZip(&zipLayout{ 302 | entries: []zipEntry{ 303 | zipEntry{ 304 | name: "1", 305 | data: []byte("uh oh"), 306 | expectedMimeType: "text/plain; charset=utf-8", 307 | }, 308 | zipEntry{ 309 | name: "2", 310 | data: []byte("uh oh"), 311 | expectedMimeType: "text/plain; charset=utf-8", 312 | }, 313 | zipEntry{ 314 | name: "3", 315 | data: []byte("uh oh"), 316 | expectedMimeType: "text/plain; charset=utf-8", 317 | }, 318 | zipEntry{ 319 | name: "4", 320 | data: []byte("uh oh"), 321 | expectedMimeType: "text/plain; charset=utf-8", 322 | }, 323 | }, 324 | }, func(zl *zipLayout) { 325 | limits := testLimits() 326 | limits.MaxTotalSize = 6 327 | 328 | _, err := archiver.ExtractZip(ctx, zipPath, prefix, limits) 329 | assert.Error(t, err) 330 | assert.True(t, strings.Contains(err.Error(), "zip too large")) 331 | }) 332 | 333 | // reset storage for this next test 334 | storage, err = NewMemStorage() 335 | assert.NoError(t, err) 336 | storage.planForFailure(config.Bucket, fmt.Sprintf("%s/%s", prefix, "3")) 337 | storage.putDelay = 200 * time.Millisecond 338 | archiver = &Archiver{storage, config} 339 | 340 | withZip(&zipLayout{ 341 | entries: []zipEntry{ 342 | zipEntry{ 343 | name: "1", 344 | data: []byte("uh oh"), 345 | expectedMimeType: "text/plain; charset=utf-8", 346 | }, 347 | zipEntry{ 348 | name: "2", 349 | data: []byte("uh oh"), 350 | expectedMimeType: "text/plain; charset=utf-8", 351 | }, 352 | zipEntry{ 353 | name: "3", 354 | data: []byte("uh oh"), 355 | expectedMimeType: "text/plain; charset=utf-8", 356 | }, 357 | zipEntry{ 358 | name: "4", 359 | data: []byte("uh oh"), 360 | expectedMimeType: "text/plain; charset=utf-8", 361 | }, 362 | }, 363 | }, func(zl *zipLayout) { 364 | limits := testLimits() 365 | 366 | _, err := archiver.ExtractZip(ctx, zipPath, prefix, limits) 367 | assert.Error(t, err) 368 | assert.True(t, strings.Contains(err.Error(), "intentional failure")) 369 | 370 | assert.EqualValues(t, 1, len(storage.objects), "make sure all objects have been cleaned up") 371 | for k := range storage.objects { 372 | assert.EqualValues(t, k, storage.objectPath(config.Bucket, zipPath), "make sure the only remaining object is the zip") 373 | } 374 | }) 375 | } 376 | 377 | // TestFetchZipFailing simulates a download failing after the ouptut file has been created, 378 | // and makes sure the incomplete file is removed. 379 | func TestFetchZipFailing(t *testing.T) { 380 | rand.Seed(time.Now().Unix()) 381 | bucket := "bucket" + strconv.Itoa(rand.Int()) 382 | key := "key" + strconv.Itoa(rand.Int()) 383 | path := fetchZipFilename(bucket, key) 384 | path = filepath.Join(tmpDir, path) 385 | require.False(t, fileExists(path), "test output file existed ahead of time") 386 | t.Logf("temp file: %s", path) 387 | 388 | a := &Archiver{ 389 | Storage: &mockFailingStorage{t, path}, 390 | Config: &Config{ 391 | Bucket: bucket, 392 | }, 393 | } 394 | 395 | ctx := context.Background() 396 | _, err := a.fetchZip(ctx, key) 397 | assert.EqualError(t, err, "intentional failure") 398 | assert.False(t, fileExists(path), "file should have been removed") 399 | } 400 | 401 | func fileExists(path string) bool { 402 | _, err := os.Stat(path) 403 | if err == nil { 404 | return true 405 | } 406 | if errors.Is(err, fs.ErrNotExist) { 407 | return false 408 | } 409 | panic("unexpected error from stat: " + err.Error()) 410 | } 411 | 412 | type mockFailingStorage struct { 413 | t *testing.T 414 | path string 415 | } 416 | 417 | func (m *mockFailingStorage) GetFile(_ context.Context, _, _ string) (io.ReadCloser, http.Header, error) { 418 | return &mockFailingReadCloser{m.t, m.path}, nil, nil 419 | } 420 | 421 | func (m *mockFailingStorage) PutFile(_ context.Context, _, _ string, contents io.Reader, _ string) error { 422 | return nil 423 | } 424 | 425 | func (m *mockFailingStorage) PutFileWithSetup(_ context.Context, _, _ string, contents io.Reader, _ StorageSetupFunc) error { 426 | return nil 427 | } 428 | 429 | func (m *mockFailingStorage) DeleteFile(_ context.Context, _, _ string) error { 430 | return nil 431 | } 432 | 433 | type mockFailingReadCloser struct { 434 | t *testing.T 435 | path string 436 | } 437 | 438 | func (m *mockFailingReadCloser) Read(p []byte) (int, error) { 439 | assert.True(m.t, fileExists(m.path), "file should have been created") 440 | return 0, errors.New("intentional failure") 441 | } 442 | 443 | func (m *mockFailingReadCloser) Close() error { 444 | return nil 445 | } 446 | -------------------------------------------------------------------------------- /zipserver/config.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | errors "github.com/go-errors/errors" 10 | ) 11 | 12 | // DefaultConfigFname is the default name for zipserver's config file 13 | var DefaultConfigFname = "zipserver.json" 14 | 15 | // ExtractLimits describes various limits we enforce when extracting zips, 16 | // mostly related to the number of files, their sizes, and the lengths of their paths 17 | type ExtractLimits struct { 18 | MaxFileSize uint64 19 | MaxTotalSize uint64 20 | MaxNumFiles int 21 | MaxFileNameLength int 22 | ExtractionThreads int 23 | } 24 | 25 | type StorageType int 26 | 27 | const ( 28 | GCS StorageType = iota // Google Cloud Storage 29 | S3 // Amazon S3 Storage 30 | ) 31 | 32 | var storageTypeString = map[string]StorageType{ 33 | "GCS": GCS, 34 | "S3": S3, 35 | } 36 | 37 | var storageTypeInt = map[StorageType]string{ 38 | GCS: "GCS", 39 | S3: "S3", 40 | } 41 | 42 | func (s *StorageType) MarshalJSON() ([]byte, error) { 43 | return json.Marshal(storageTypeInt[*s]) 44 | } 45 | 46 | func (s *StorageType) UnmarshalJSON(data []byte) error { 47 | var str string 48 | if err := json.Unmarshal(data, &str); err != nil { 49 | return err 50 | } 51 | val, ok := storageTypeString[str] 52 | if !ok { 53 | return errors.New("Invalid StorageType value") 54 | } 55 | *s = val 56 | return nil 57 | } 58 | 59 | // StorageTarget represents a storage configuration that can be written to, either GCS or S3 60 | type StorageConfig struct { 61 | Name string 62 | 63 | Type StorageType 64 | 65 | // NOTE: GCS not implemented for storage config yet 66 | GCSPrivateKeyPath string `json:",omitempty"` 67 | GCSClientEmail string `json:",omitempty"` 68 | 69 | S3AccessKeyID string `json:",omitempty"` 70 | S3SecretKey string `json:",omitempty"` 71 | S3Endpoint string `json:",omitempty"` 72 | S3Region string `json:",omitempty"` 73 | 74 | Bucket string `json:",omitempty"` 75 | } 76 | 77 | // TODO: eventually this should be a factory that can return different storage types 78 | func (sc *StorageConfig) NewStorageClient() (*S3Storage, error) { 79 | switch sc.Type { 80 | case S3: 81 | return NewS3Storage(sc) 82 | case GCS: 83 | return nil, fmt.Errorf("GCS storage type is not supported yet") 84 | default: 85 | return nil, fmt.Errorf("unsupported storage type") 86 | } 87 | } 88 | 89 | func (s *StorageConfig) Validate() error { 90 | if s.Name == "" { 91 | return errors.New("Config error: Name field missing") 92 | } 93 | 94 | missingFieldError := func(field string) error { 95 | return errors.New(fmt.Sprintf("Config error: [Storage %s] %s field missing", s.Name, field)) 96 | } 97 | 98 | if s.Type == GCS { 99 | if s.GCSPrivateKeyPath == "" { 100 | return missingFieldError("GCSPrivateKeyPath") 101 | } 102 | 103 | if s.GCSClientEmail == "" { 104 | return missingFieldError("GCSClientEmail") 105 | } 106 | } else if s.Type == S3 { 107 | // access key and secret key are optional for S3, since they can be loaded from env 108 | if s.S3Endpoint == "" { 109 | return missingFieldError("S3Endpoint") 110 | } 111 | 112 | if s.S3Region == "" { 113 | return missingFieldError("S3Region") 114 | } 115 | } 116 | 117 | if s.Bucket == "" { 118 | return missingFieldError("Bucket") 119 | } 120 | 121 | return nil 122 | } 123 | 124 | // Config contains both storage configuration and the enforced extraction limits 125 | type Config struct { 126 | PrivateKeyPath string 127 | ClientEmail string 128 | Bucket string 129 | ExtractPrefix string 130 | MetricsHost string `json:",omitempty"` 131 | 132 | MaxFileSize uint64 133 | MaxTotalSize uint64 134 | MaxNumFiles int 135 | MaxFileNameLength int 136 | ExtractionThreads int 137 | 138 | JobTimeout Duration `json:",omitempty"` // Time to complete entire extract or upload job 139 | FileGetTimeout Duration `json:",omitempty"` // Time to download a single object 140 | FilePutTimeout Duration `json:",omitempty"` // Time to upload a single object 141 | AsyncNotificationTimeout Duration `json:",omitempty"` // Time to complete webhook request 142 | 143 | // Places that can be written to 144 | StorageTargets []StorageConfig `json:",omitempty"` 145 | } 146 | 147 | // GetStorageTargetByName returns the storage target with the given name from the config. 148 | // If no such target exists, it returns nil. 149 | func (c *Config) GetStorageTargetByName(name string) *StorageConfig { 150 | for i, target := range c.StorageTargets { 151 | if target.Name == name { 152 | return &c.StorageTargets[i] 153 | } 154 | } 155 | return nil 156 | } 157 | 158 | var defaultConfig = Config{ 159 | MaxFileSize: 1024 * 1024 * 200, 160 | MaxTotalSize: 1024 * 1024 * 500, 161 | MaxNumFiles: 100, 162 | MaxFileNameLength: 80, 163 | ExtractionThreads: 4, 164 | 165 | JobTimeout: Duration(5 * time.Minute), 166 | FileGetTimeout: Duration(1 * time.Minute), 167 | FilePutTimeout: Duration(1 * time.Minute), 168 | AsyncNotificationTimeout: Duration(5 * time.Second), 169 | } 170 | 171 | // Duration adds JSON (de)serialization to time.Duration. 172 | // This should be fixed in Go 2. 173 | // https://github.com/golang/go/issues/10275 174 | type Duration time.Duration 175 | 176 | func (d Duration) MarshalJSON() ([]byte, error) { 177 | return json.Marshal(time.Duration(d).String()) 178 | } 179 | 180 | func (d *Duration) UnmarshalJSON(b []byte) error { 181 | var s string 182 | if err := json.Unmarshal(b, &s); err != nil { 183 | return err 184 | } 185 | dur, err := time.ParseDuration(s) 186 | if err != nil { 187 | return err 188 | } 189 | *d = Duration(dur) 190 | return nil 191 | } 192 | 193 | // LoadConfig reads a config file into a config struct 194 | func LoadConfig(fname string) (*Config, error) { 195 | jsonBlob, err := os.ReadFile(fname) 196 | if err != nil { 197 | return nil, errors.Wrap(err, 0) 198 | } 199 | 200 | config := defaultConfig 201 | err = json.Unmarshal(jsonBlob, &config) 202 | if err != nil { 203 | return nil, fmt.Errorf("Failed parsing config file %s: %s", fname, err.Error()) 204 | } 205 | 206 | if config.PrivateKeyPath == "" { 207 | return nil, errors.New("Config error: PrivateKeyPath field missing") 208 | } 209 | 210 | if config.ClientEmail == "" { 211 | return nil, errors.New("Config error: ClientEmail field missing") 212 | } 213 | 214 | if config.Bucket == "" { 215 | return nil, errors.New("Config error: Bucket field missing") 216 | } 217 | 218 | if config.ExtractPrefix == "" { 219 | return nil, errors.New("Config error: ExtractPrefix field missing") 220 | } 221 | 222 | // validate storage targets 223 | for _, target := range config.StorageTargets { 224 | if err := target.Validate(); err != nil { 225 | return nil, err 226 | } 227 | } 228 | 229 | return &config, nil 230 | } 231 | 232 | func (c *Config) String() string { 233 | bytes, err := json.MarshalIndent(c, "", " ") 234 | if err != nil { 235 | return fmt.Sprintf("Error: could not stringify config: %s", err.Error()) 236 | } 237 | 238 | return string(bytes) 239 | } 240 | 241 | // DefaultExtractLimits returns only extract limits from a config struct 242 | func DefaultExtractLimits(config *Config) *ExtractLimits { 243 | return &ExtractLimits{ 244 | MaxFileSize: config.MaxFileSize, 245 | MaxTotalSize: config.MaxTotalSize, 246 | MaxNumFiles: config.MaxNumFiles, 247 | MaxFileNameLength: config.MaxFileNameLength, 248 | ExtractionThreads: config.ExtractionThreads, 249 | } 250 | } 251 | -------------------------------------------------------------------------------- /zipserver/config_test.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "os" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func Test_Config(t *testing.T) { 14 | tmpFile, err := os.CreateTemp("", "zipserver-config") 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | 19 | defer os.Remove(tmpFile.Name()) 20 | 21 | writeConfigBytes := func(bytes []byte) { 22 | _, err := tmpFile.Seek(0, io.SeekStart) 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | 27 | _, err = tmpFile.Write(bytes) 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | } 32 | 33 | writeConfig := func(c *Config) { 34 | bytes, err := json.Marshal(c) 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | writeConfigBytes(bytes) 39 | } 40 | 41 | assertConfigError := func() { 42 | _, err = LoadConfig(tmpFile.Name()) 43 | assert.Error(t, err) 44 | } 45 | 46 | writeConfigBytes([]byte("{")) 47 | assertConfigError() 48 | 49 | writeConfig(&Config{}) 50 | assertConfigError() 51 | 52 | writeConfig(&Config{ 53 | PrivateKeyPath: "/foo/bar.pem", 54 | }) 55 | assertConfigError() 56 | 57 | writeConfig(&Config{ 58 | PrivateKeyPath: "/foo/bar.pem", 59 | ClientEmail: "foobar@example.org", 60 | }) 61 | assertConfigError() 62 | 63 | writeConfig(&Config{ 64 | PrivateKeyPath: "/foo/bar.pem", 65 | ClientEmail: "foobar@example.org", 66 | Bucket: "chicken", 67 | }) 68 | assertConfigError() 69 | 70 | writeConfig(&Config{ 71 | PrivateKeyPath: "/foo/bar.pem", 72 | ClientEmail: "foobar@example.org", 73 | Bucket: "chicken", 74 | ExtractPrefix: "saca", 75 | MaxFileSize: 92, 76 | }) 77 | 78 | c, err := LoadConfig(tmpFile.Name()) 79 | assert.NoError(t, err) 80 | 81 | assert.EqualValues(t, "/foo/bar.pem", c.PrivateKeyPath) 82 | assert.EqualValues(t, 92, c.MaxFileSize) 83 | assert.Equal(t, 5*time.Minute, time.Duration(c.JobTimeout)) 84 | assert.Equal(t, 1*time.Minute, time.Duration(c.FileGetTimeout)) 85 | assert.Equal(t, 1*time.Minute, time.Duration(c.FilePutTimeout)) 86 | assert.Equal(t, 5*time.Second, time.Duration(c.AsyncNotificationTimeout)) 87 | 88 | assert.True(t, c.String() != "") 89 | } 90 | -------------------------------------------------------------------------------- /zipserver/copy_handler.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net/http" 10 | "net/url" 11 | "time" 12 | ) 13 | 14 | var copyLockTable = NewLockTable() 15 | 16 | func formatBytes(b float64) string { 17 | const unit = 1024 18 | if b < unit { 19 | return fmt.Sprintf("%.2f B", b) 20 | } 21 | div, exp := float64(unit), 0 22 | for n := b / unit; n >= unit; n /= unit { 23 | div *= unit 24 | exp++ 25 | } 26 | return fmt.Sprintf("%.2f %cB", b/div, "kMGTPE"[exp]) 27 | } 28 | 29 | // notify the callback URL of task completion 30 | func notifyCallback(callbackURL string, resValues url.Values) error { 31 | notifyCtx, notifyCancel := context.WithTimeout(context.Background(), time.Duration(globalConfig.AsyncNotificationTimeout)) 32 | defer notifyCancel() 33 | 34 | outBody := bytes.NewBufferString(resValues.Encode()) 35 | req, err := http.NewRequestWithContext(notifyCtx, http.MethodPost, callbackURL, outBody) 36 | if err != nil { 37 | log.Print("Failed to create callback request: ", err) 38 | return err 39 | } 40 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded") 41 | 42 | response, err := http.DefaultClient.Do(req) 43 | if err != nil { 44 | log.Print("Failed to deliver callback: ", err) 45 | return err 46 | } 47 | 48 | if response.StatusCode != http.StatusOK { 49 | log.Printf("Callback returned unexpected code: %d %s", response.StatusCode, callbackURL) 50 | bodyBytes, _ := io.ReadAll(response.Body) 51 | bodyString := string(bodyBytes) 52 | log.Print(bodyString) 53 | } 54 | 55 | response.Body.Close() 56 | 57 | return nil 58 | } 59 | 60 | // notify the callback URL that an error happened 61 | func notifyError(callbackURL string, err error) error { 62 | globalMetrics.TotalErrors.Add(1) 63 | 64 | message := url.Values{} 65 | message.Add("Success", "false") 66 | message.Add("Error", err.Error()) 67 | return notifyCallback(callbackURL, message) 68 | } 69 | 70 | // The copy handler will asynchronously copy a file from primary storage to the 71 | // storage specified by target 72 | func copyHandler(w http.ResponseWriter, r *http.Request) error { 73 | params := r.URL.Query() 74 | key, err := getParam(params, "key") 75 | if err != nil { 76 | return err 77 | } 78 | 79 | callbackURL, err := getParam(params, "callback") 80 | if err != nil { 81 | return err 82 | } 83 | 84 | targetName, err := getParam(params, "target") 85 | if err != nil { 86 | return err 87 | } 88 | 89 | storageTargetConfig := globalConfig.GetStorageTargetByName(targetName) 90 | if storageTargetConfig == nil { 91 | return fmt.Errorf("Invalid target: %s", targetName) 92 | } 93 | 94 | expectedBucket, _ := getParam(params, "bucket") 95 | targetBucket := storageTargetConfig.Bucket 96 | 97 | if expectedBucket != "" && expectedBucket != targetBucket { 98 | return fmt.Errorf("Expected bucket does not match target bucket: %s != %s", expectedBucket, targetBucket) 99 | } 100 | 101 | lockKey := fmt.Sprintf("%s:%s", targetName, key) 102 | 103 | hasLock := copyLockTable.tryLockKey(lockKey) 104 | 105 | if !hasLock { 106 | // already being extracted in another handler, ask consumer to wait 107 | return writeJSONMessage(w, struct{ Processing bool }{true}) 108 | } 109 | 110 | go (func() { 111 | defer copyLockTable.releaseKey(lockKey) 112 | 113 | jobCtx, cancel := context.WithTimeout(context.Background(), time.Duration(globalConfig.JobTimeout)) 114 | defer cancel() 115 | 116 | storage, err := NewGcsStorage(globalConfig) 117 | 118 | if storage == nil { 119 | notifyError(callbackURL, fmt.Errorf("Failed to create source storage: %v", err)) 120 | return 121 | } 122 | 123 | targetStorage, err := storageTargetConfig.NewStorageClient() 124 | 125 | if err != nil { 126 | notifyError(callbackURL, fmt.Errorf("Failed to create target storage: %v", err)) 127 | return 128 | } 129 | 130 | startTime := time.Now() 131 | 132 | reader, headers, err := storage.GetFile(jobCtx, globalConfig.Bucket, key) 133 | 134 | if err != nil { 135 | log.Print("Failed to get file: ", err) 136 | notifyError(callbackURL, err) 137 | return 138 | } 139 | 140 | defer reader.Close() 141 | 142 | mReader := newMeasuredReader(reader) 143 | 144 | uploadHeaders := http.Header{} 145 | 146 | contentType := headers.Get("Content-Type") 147 | if contentType == "" { 148 | contentType = "application/octet-stream" 149 | } 150 | 151 | uploadHeaders.Set("Content-Type", contentType) 152 | 153 | contentDisposition := headers.Get("Content-Disposition") 154 | if contentDisposition != "" { 155 | uploadHeaders.Set("Content-Disposition", contentDisposition) 156 | } 157 | 158 | log.Print("Starting transfer: [", targetName, "] ", targetBucket, "/", key, " ", uploadHeaders) 159 | checksumMd5, err := targetStorage.PutFile(jobCtx, targetBucket, key, mReader, uploadHeaders) 160 | 161 | if err != nil { 162 | log.Print("Failed to copy file: ", err) 163 | notifyError(callbackURL, err) 164 | return 165 | } 166 | 167 | globalMetrics.TotalCopiedFiles.Add(1) 168 | log.Print("Transfer complete: [", targetName, "] ", targetBucket, "/", key, 169 | ", bytes read: ", formatBytes(float64(mReader.BytesRead)), 170 | ", duration: ", mReader.Duration.Seconds(), 171 | ", speed: ", formatBytes(mReader.TransferSpeed()), "/s") 172 | 173 | resValues := url.Values{} 174 | resValues.Add("Success", "true") 175 | resValues.Add("Key", key) 176 | resValues.Add("Duration", fmt.Sprintf("%.4fs", time.Since(startTime).Seconds())) 177 | resValues.Add("Size", fmt.Sprintf("%d", mReader.BytesRead)) 178 | resValues.Add("Md5", checksumMd5) 179 | 180 | notifyCallback(callbackURL, resValues) 181 | })() 182 | 183 | return writeJSONMessage(w, struct { 184 | Processing bool 185 | Async bool 186 | }{true, true}) 187 | } 188 | -------------------------------------------------------------------------------- /zipserver/extract_handler.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | "net/url" 11 | "time" 12 | ) 13 | 14 | // mutex for keys currently being extracted 15 | var extractLockTable = NewLockTable() 16 | 17 | func loadLimits(params url.Values, config *Config) *ExtractLimits { 18 | limits := DefaultExtractLimits(config) 19 | 20 | { 21 | maxFileSize, err := getUint64Param(params, "maxFileSize") 22 | if err == nil { 23 | limits.MaxFileSize = maxFileSize 24 | } 25 | } 26 | 27 | { 28 | maxTotalSize, err := getUint64Param(params, "maxTotalSize") 29 | if err == nil { 30 | limits.MaxTotalSize = maxTotalSize 31 | } 32 | } 33 | 34 | { 35 | maxNumFiles, err := getIntParam(params, "maxNumFiles") 36 | if err == nil { 37 | limits.MaxNumFiles = maxNumFiles 38 | } 39 | } 40 | 41 | { 42 | maxFileNameLength, err := getIntParam(params, "maxFileNameLength") 43 | if err == nil { 44 | limits.MaxFileNameLength = maxFileNameLength 45 | } 46 | } 47 | 48 | return limits 49 | } 50 | 51 | func extractHandler(w http.ResponseWriter, r *http.Request) error { 52 | params := r.URL.Query() 53 | key, err := getParam(params, "key") 54 | if err != nil { 55 | return err 56 | } 57 | 58 | prefix, err := getParam(params, "prefix") 59 | if err != nil { 60 | return err 61 | } 62 | 63 | hasLock := extractLockTable.tryLockKey(key) 64 | if !hasLock { 65 | // already being extracted in another handler, ask consumer to wait 66 | return writeJSONMessage(w, struct{ Processing bool }{true}) 67 | } 68 | 69 | limits := loadLimits(params, globalConfig) 70 | 71 | process := func(ctx context.Context) ([]ExtractedFile, error) { 72 | archiver := NewArchiver(globalConfig) 73 | files, err := archiver.ExtractZip(ctx, key, prefix, limits) 74 | 75 | return files, err 76 | } 77 | 78 | // sync codepath 79 | asyncURL := params.Get("async") 80 | if asyncURL == "" { 81 | defer extractLockTable.releaseKey(key) 82 | 83 | ctx, cancel := context.WithTimeout(r.Context(), time.Duration(globalConfig.JobTimeout)) 84 | defer cancel() 85 | 86 | extracted, err := process(ctx) 87 | if err != nil { 88 | globalMetrics.TotalErrors.Add(1) 89 | return writeJSONError(w, "ExtractError", err) 90 | } 91 | 92 | return writeJSONMessage(w, struct { 93 | Success bool 94 | ExtractedFiles []ExtractedFile 95 | }{true, extracted}) 96 | } 97 | 98 | // async codepath 99 | go (func() { 100 | defer extractLockTable.releaseKey(key) 101 | 102 | // This job is expected to outlive the incoming request, so create a detached context. 103 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(globalConfig.JobTimeout)) 104 | defer cancel() 105 | 106 | extracted, err := process(ctx) 107 | resValues := url.Values{} 108 | 109 | if err != nil { 110 | errMessage := err.Error() 111 | 112 | if errors.Is(err, context.DeadlineExceeded) { 113 | errMessage = "Zip extraction timed out" 114 | } 115 | 116 | globalMetrics.TotalErrors.Add(1) 117 | resValues.Add("Type", "ExtractError") 118 | resValues.Add("Error", errMessage) 119 | log.Print("Extraction failed ", err) 120 | } else { 121 | resValues.Add("Success", "true") 122 | for idx, extractedFile := range extracted { 123 | resValues.Add(fmt.Sprintf("ExtractedFiles[%d][Key])", idx+1), 124 | extractedFile.Key) 125 | resValues.Add(fmt.Sprintf("ExtractedFiles[%d][Size])", idx+1), 126 | fmt.Sprintf("%v", extractedFile.Size)) 127 | } 128 | } 129 | 130 | log.Print("Notifying " + asyncURL) 131 | 132 | nofityCtx, nofifyCancel := context.WithTimeout(context.Background(), time.Duration(globalConfig.AsyncNotificationTimeout)) 133 | defer nofifyCancel() 134 | 135 | outBody := bytes.NewBufferString(resValues.Encode()) 136 | req, err := http.NewRequestWithContext(nofityCtx, http.MethodPost, asyncURL, outBody) 137 | if err != nil { 138 | log.Printf("Failed to create callback request: %v", err) 139 | return 140 | } 141 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded") 142 | 143 | asyncResponse, err := http.DefaultClient.Do(req) 144 | if err == nil { 145 | asyncResponse.Body.Close() 146 | } else { 147 | log.Print("Failed to deliver callback: " + err.Error()) 148 | } 149 | })() 150 | 151 | return writeJSONMessage(w, struct { 152 | Processing bool 153 | Async bool 154 | }{true, true}) 155 | } 156 | -------------------------------------------------------------------------------- /zipserver/extract_handler_test.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func Test_Limits(t *testing.T) { 12 | var values url.Values 13 | 14 | el := loadLimits(values, &defaultConfig) 15 | assert.EqualValues(t, el.MaxFileSize, defaultConfig.MaxFileSize) 16 | 17 | const customMaxFileSize = 9428 18 | values, err := url.ParseQuery(fmt.Sprintf("maxFileSize=%d", customMaxFileSize)) 19 | assert.NoError(t, err) 20 | 21 | el = loadLimits(values, &defaultConfig) 22 | assert.EqualValues(t, el.MaxFileSize, customMaxFileSize) 23 | } 24 | -------------------------------------------------------------------------------- /zipserver/gcs_storage.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net/http" 10 | "os" 11 | 12 | "golang.org/x/oauth2/google" 13 | "golang.org/x/oauth2/jwt" 14 | ) 15 | 16 | var ( 17 | baseURL = "https://storage.googleapis.com/" 18 | scope = "https://www.googleapis.com/auth/devstorage.full_control" 19 | ) 20 | 21 | // GcsStorage is a simple interface to Google Cloud Storage 22 | // 23 | // Example usage: 24 | // 25 | // storage := NewStorageClient(config) 26 | // readCloser, err = storage.GetFile("my_bucket", "my_file") 27 | type GcsStorage struct { 28 | jwtConfig *jwt.Config 29 | } 30 | 31 | // interface guard 32 | var _ Storage = (*GcsStorage)(nil) 33 | 34 | // NewGcsStorage returns a new GCS-backed storage 35 | func NewGcsStorage(config *Config) (*GcsStorage, error) { 36 | pemBytes, err := os.ReadFile(config.PrivateKeyPath) 37 | 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | jwtConfig := &jwt.Config{ 43 | Email: config.ClientEmail, 44 | PrivateKey: pemBytes, 45 | TokenURL: google.JWTTokenURL, 46 | Scopes: []string{scope}, 47 | } 48 | 49 | return &GcsStorage{ 50 | jwtConfig: jwtConfig, 51 | }, nil 52 | } 53 | 54 | func (c *GcsStorage) httpClient() (*http.Client, error) { 55 | return c.jwtConfig.Client(context.Background()), nil 56 | } 57 | 58 | func (c *GcsStorage) url(bucket, key, logName string) string { 59 | // return "http://127.0.0.1:5656" 60 | url := baseURL + bucket + "/" + key 61 | log.Print(logName + " " + url) 62 | return url 63 | } 64 | 65 | // GetFile returns a reader for the contents of resource at bucket/key 66 | func (c *GcsStorage) GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, http.Header, error) { 67 | httpClient, err := c.httpClient() 68 | if err != nil { 69 | return nil, nil, err 70 | } 71 | 72 | url := c.url(bucket, key, "GET") 73 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 74 | if err != nil { 75 | return nil, nil, err 76 | } 77 | 78 | res, err := httpClient.Do(req) 79 | if err != nil { 80 | return nil, nil, err 81 | } 82 | 83 | if res.StatusCode != 200 { 84 | res.Body.Close() 85 | return nil, res.Header, errors.New(res.Status + " " + url) 86 | } 87 | 88 | trackedBody := metricsReadCloser{res.Body, &globalMetrics.TotalBytesDownloaded} 89 | 90 | return trackedBody, res.Header, nil 91 | } 92 | 93 | // PutFile uploads a file to GCS simply 94 | func (c *GcsStorage) PutFile(ctx context.Context, bucket, key string, contents io.Reader, mimeType string) error { 95 | return c.PutFileWithSetup(ctx, bucket, key, contents, func(req *http.Request) error { 96 | req.Header.Add("Content-Type", mimeType) 97 | req.Header.Add("x-goog-acl", "public-read") 98 | return nil 99 | }) 100 | } 101 | 102 | // PutFileWithSetup uploads a file to GCS letting the user set up the request first 103 | func (c *GcsStorage) PutFileWithSetup(ctx context.Context, bucket, key string, contents io.Reader, setup StorageSetupFunc) error { 104 | httpClient, err := c.httpClient() 105 | if err != nil { 106 | return err 107 | } 108 | 109 | contents = metricsReader(contents, &globalMetrics.TotalBytesUploaded) 110 | 111 | req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.url(bucket, key, "PUT"), contents) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | err = setup(req) 117 | if err != nil { 118 | return err 119 | } 120 | 121 | res, err := httpClient.Do(req) 122 | if err != nil { 123 | return err 124 | } 125 | 126 | defer res.Body.Close() 127 | 128 | if res.StatusCode != http.StatusOK { 129 | body, err := io.ReadAll(res.Body) 130 | if err != nil { 131 | return err 132 | } 133 | return fmt.Errorf("%s: %s", res.Status, body) 134 | } 135 | 136 | return nil 137 | } 138 | 139 | // DeleteFile removes a file from a GCS bucket 140 | func (c *GcsStorage) DeleteFile(ctx context.Context, bucket, key string) error { 141 | httpClient, err := c.httpClient() 142 | if err != nil { 143 | return err 144 | } 145 | 146 | url := c.url(bucket, key, "DELETE") 147 | req, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil) 148 | if err != nil { 149 | return err 150 | } 151 | 152 | res, err := httpClient.Do(req) 153 | if err != nil { 154 | return err 155 | } 156 | 157 | if res.StatusCode != 200 && res.StatusCode != 204 { 158 | return errors.New(res.Status + " " + url) 159 | } 160 | 161 | return nil 162 | } 163 | -------------------------------------------------------------------------------- /zipserver/gcs_storage_test.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | "strings" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | const testPrivateKey = "/home/leafo/code/go/cf45ea3f8a5f730a4b9702d11236439d9b014b20-privatekey.pem" 13 | 14 | type ClientFunc func(storage Storage, config *Config) 15 | 16 | func withGoogleCloudStorage(t *testing.T, cb ClientFunc) { 17 | _, err := os.Lstat(testPrivateKey) 18 | if err != nil { 19 | t.Logf("Skipping (no private key)") 20 | return 21 | } 22 | 23 | config := &Config{ 24 | PrivateKeyPath: testPrivateKey, 25 | ClientEmail: "507810471102@developer.gserviceaccount.com", 26 | Bucket: "leafo", 27 | JobTimeout: Duration(10 * time.Second), 28 | FileGetTimeout: Duration(10 * time.Second), 29 | FilePutTimeout: Duration(10 * time.Second), 30 | } 31 | 32 | storage, err := NewGcsStorage(config) 33 | 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | 38 | cb(storage, config) 39 | } 40 | 41 | func TestGetFile(t *testing.T) { 42 | ctx := context.Background() 43 | 44 | withGoogleCloudStorage(t, func(storage Storage, config *Config) { 45 | reader, _, err := storage.GetFile(ctx, config.Bucket, "text.txt") 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | 50 | defer reader.Close() 51 | bytesContent, err := io.ReadAll(reader) 52 | if err != nil { 53 | t.Fatal(err) 54 | } 55 | 56 | str := string(bytesContent) 57 | 58 | if !strings.Contains(str, "Gravity") { 59 | t.Fatal("Expected to get string from text.txt") 60 | } 61 | }) 62 | } 63 | 64 | func TestPutAndDeleteFile(t *testing.T) { 65 | ctx := context.Background() 66 | 67 | withGoogleCloudStorage(t, func(storage Storage, config *Config) { 68 | err := storage.PutFile(ctx, config.Bucket, "zipserver_test.txt", strings.NewReader("hello zipserver!"), "text/plain") 69 | 70 | if err != nil { 71 | t.Fatal(err) 72 | } 73 | 74 | err = storage.DeleteFile(ctx, config.Bucket, "zipserver_test.txt") 75 | 76 | if err != nil { 77 | t.Fatal(err) 78 | } 79 | }) 80 | } 81 | -------------------------------------------------------------------------------- /zipserver/list_handler.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "context" 7 | "errors" 8 | "io" 9 | "net/http" 10 | "time" 11 | ) 12 | 13 | type fileTuple struct { 14 | Filename string 15 | Size uint64 16 | } 17 | 18 | func listZip(body []byte, w http.ResponseWriter, r *http.Request) error { 19 | zipFile, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) 20 | 21 | if err != nil { 22 | return err 23 | } 24 | 25 | var filesOut []fileTuple 26 | 27 | for _, file := range zipFile.File { 28 | filesOut = append(filesOut, fileTuple{ 29 | file.Name, file.UncompressedSize64, 30 | }) 31 | } 32 | 33 | return writeJSONMessage(w, filesOut) 34 | } 35 | 36 | func listFromBucket(ctx context.Context, key string, w http.ResponseWriter, r *http.Request) error { 37 | storage, err := NewGcsStorage(globalConfig) 38 | if storage == nil { 39 | return err 40 | } 41 | 42 | reader, _, err := storage.GetFile(ctx, globalConfig.Bucket, key) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | defer reader.Close() 48 | 49 | body, err := io.ReadAll(reader) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | return listZip(body, w, r) 55 | } 56 | 57 | func listFromUrl(ctx context.Context, url string, w http.ResponseWriter, r *http.Request) error { 58 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | response, err := http.DefaultClient.Do(req) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | defer response.Body.Close() 69 | body, err := io.ReadAll(response.Body) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | return listZip(body, w, r) 75 | } 76 | 77 | func listHandler(w http.ResponseWriter, r *http.Request) error { 78 | ctx, cancel := context.WithTimeout(r.Context(), time.Duration(globalConfig.FileGetTimeout)) 79 | defer cancel() 80 | 81 | params := r.URL.Query() 82 | 83 | key, err := getParam(params, "key") 84 | if err == nil { 85 | return listFromBucket(ctx, key, w, r) 86 | } 87 | 88 | url, err := getParam(params, "url") 89 | if err == nil { 90 | return listFromUrl(ctx, url, w, r) 91 | } 92 | 93 | return errors.New("missing key or url") 94 | } 95 | -------------------------------------------------------------------------------- /zipserver/lock_table.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | type LockTable struct { 9 | // maps aren't thread-safe in golang, this protects openKeys 10 | sync.Mutex 11 | // We're using the map to store the time at which a lock is obtained for a key 12 | openKeys map[string]time.Time 13 | } 14 | 15 | func NewLockTable() *LockTable { 16 | return &LockTable{ 17 | openKeys: make(map[string]time.Time), 18 | } 19 | } 20 | 21 | // tryLockKey tries acquiring the lock for a given key 22 | // it returns true if we successfully acquired the lock, 23 | // false if the key is locked by someone else 24 | func (lt *LockTable) tryLockKey(key string) bool { 25 | lt.Lock() 26 | defer lt.Unlock() 27 | 28 | // test for key existence 29 | if _, ok := lt.openKeys[key]; ok { 30 | // locked by someone else 31 | return false 32 | } 33 | lt.openKeys[key] = time.Now() 34 | return true 35 | } 36 | 37 | func (lt *LockTable) releaseKey(key string) { 38 | lt.Lock() 39 | defer lt.Unlock() 40 | 41 | // delete key from map so the map doesn't keep growing 42 | delete(lt.openKeys, key) 43 | } 44 | 45 | type KeyInfo struct { 46 | Key string 47 | LockedAt time.Time 48 | LockedSeconds float64 49 | } 50 | 51 | // returns summary of held locks for debugging purposes 52 | func (lt *LockTable) GetLocks() []KeyInfo { 53 | lt.Lock() 54 | defer lt.Unlock() 55 | 56 | keys := make([]KeyInfo, 0, len(lt.openKeys)) 57 | for key, lockedAt := range lt.openKeys { 58 | keys = append(keys, KeyInfo{ 59 | Key: key, 60 | LockedAt: lockedAt, 61 | LockedSeconds: time.Since(lockedAt).Seconds(), 62 | }) 63 | } 64 | return keys 65 | } 66 | -------------------------------------------------------------------------------- /zipserver/lock_table_test.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func Test_LockTable(t *testing.T) { 10 | // Create a new lock table for the test 11 | lt := NewLockTable() 12 | 13 | // not the best test, more like a basic sanity check 14 | hasLock := lt.tryLockKey("foo") 15 | 16 | assert.True(t, hasLock, "should acquire foo") 17 | 18 | hasLock = lt.tryLockKey("foo") 19 | assert.False(t, hasLock, "should not acquire foo again") 20 | 21 | hasLock = lt.tryLockKey("bar") 22 | assert.True(t, hasLock, "should acquire bar") 23 | 24 | lt.releaseKey("foo") 25 | hasLock = lt.tryLockKey("bar") 26 | assert.False(t, hasLock, "should not acquire bar again") 27 | 28 | hasLock = lt.tryLockKey("foo") 29 | assert.True(t, hasLock, "should acquire foo again") 30 | } 31 | -------------------------------------------------------------------------------- /zipserver/mem_storage.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | // MemStorage implements Storage interface in memory, storing objects in a map. 4 | // This is used for the serving of a zip file over http. Keep in mind extracted 5 | // zips are stored forever, this should only be used for testing or one-off use 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "fmt" 11 | "io" 12 | "net/http" 13 | "sync" 14 | "time" 15 | 16 | errors "github.com/go-errors/errors" 17 | ) 18 | 19 | type memObject struct { 20 | data []byte 21 | headers http.Header 22 | } 23 | 24 | // MemStorage implements Storage on a directory 25 | // it stores things in `baseDir/bucket/prefix...` 26 | type MemStorage struct { 27 | mutex sync.Mutex 28 | objects map[string]memObject 29 | failingPaths map[string]struct{} 30 | putDelay time.Duration 31 | } 32 | 33 | // interface guard 34 | var _ Storage = (*MemStorage)(nil) 35 | 36 | // NewMemStorage creates a new fs storage working in the given directory 37 | func NewMemStorage() (*MemStorage, error) { 38 | return &MemStorage{ 39 | objects: make(map[string]memObject), 40 | failingPaths: make(map[string]struct{}), 41 | }, nil 42 | } 43 | 44 | func (fs *MemStorage) objectPath(bucket, key string) string { 45 | return fmt.Sprintf("%s/%s", bucket, key) 46 | } 47 | 48 | func (fs *MemStorage) GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, http.Header, error) { 49 | fs.mutex.Lock() 50 | defer fs.mutex.Unlock() 51 | 52 | objectPath := fs.objectPath(bucket, key) 53 | 54 | if obj, ok := fs.objects[objectPath]; ok { 55 | return io.NopCloser(bytes.NewReader(obj.data)), obj.headers, nil 56 | } 57 | 58 | err := fmt.Errorf("%s: object not found", objectPath) 59 | return nil, nil, errors.Wrap(err, 0) 60 | } 61 | 62 | func (fs *MemStorage) getHeaders(bucket, key string) (http.Header, error) { 63 | fs.mutex.Lock() 64 | defer fs.mutex.Unlock() 65 | 66 | objectPath := fs.objectPath(bucket, key) 67 | 68 | if obj, ok := fs.objects[objectPath]; ok { 69 | return obj.headers, nil 70 | } 71 | 72 | err := fmt.Errorf("%s: object not found", objectPath) 73 | return nil, errors.Wrap(err, 0) 74 | } 75 | 76 | func (fs *MemStorage) PutFile(ctx context.Context, bucket, key string, contents io.Reader, mimeType string) error { 77 | return fs.PutFileWithSetup(ctx, bucket, key, contents, func(req *http.Request) error { 78 | req.Header.Set("Content-Type", mimeType) 79 | return nil 80 | }) 81 | } 82 | 83 | func (fs *MemStorage) PutFileWithSetup(ctx context.Context, bucket, key string, contents io.Reader, setup StorageSetupFunc) error { 84 | fs.mutex.Lock() 85 | defer fs.mutex.Unlock() 86 | 87 | objectPath := fs.objectPath(bucket, key) 88 | if _, ok := fs.failingPaths[objectPath]; ok { 89 | return errors.Wrap(errors.New("intentional failure"), 0) 90 | } 91 | 92 | time.Sleep(fs.putDelay) 93 | 94 | req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://127.0.0.1/dummy", nil) 95 | if err != nil { 96 | return errors.Wrap(err, 0) 97 | } 98 | 99 | err = setup(req) 100 | if err != nil { 101 | return errors.Wrap(err, 0) 102 | } 103 | 104 | data, err := io.ReadAll(contents) 105 | if err != nil { 106 | return errors.Wrap(err, 0) 107 | } 108 | 109 | fs.objects[objectPath] = memObject{ 110 | data, 111 | req.Header, 112 | } 113 | 114 | return nil 115 | } 116 | 117 | func (fs *MemStorage) DeleteFile(ctx context.Context, bucket, key string) error { 118 | fs.mutex.Lock() 119 | defer fs.mutex.Unlock() 120 | 121 | delete(fs.objects, fs.objectPath(bucket, key)) 122 | return nil 123 | } 124 | 125 | func (fs *MemStorage) planForFailure(bucket, key string) { 126 | fs.mutex.Lock() 127 | defer fs.mutex.Unlock() 128 | 129 | objectPath := fs.objectPath(bucket, key) 130 | 131 | fs.failingPaths[objectPath] = struct{}{} 132 | } 133 | -------------------------------------------------------------------------------- /zipserver/metrics.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "os" 8 | "reflect" 9 | "strings" 10 | "sync/atomic" 11 | ) 12 | 13 | var globalMetrics = &MetricsCounter{} 14 | 15 | type MetricsCounter struct { 16 | TotalRequests atomic.Int64 `metric:"zipserver_requests_total""` 17 | TotalErrors atomic.Int64 `metric:"zipserver_errors_total""` 18 | TotalExtractedFiles atomic.Int64 `metric:"zipserver_extracted_files_total"` 19 | TotalCopiedFiles atomic.Int64 `metric:"zipserver_copied_files_total"` 20 | TotalBytesDownloaded atomic.Int64 `metric:"zipserver_downloaded_bytes_total"` 21 | TotalBytesUploaded atomic.Int64 `metric:"zipserver_uploaded_bytes_total"` 22 | } 23 | 24 | // render the metrics in a prometheus compatible format 25 | func (m *MetricsCounter) RenderMetrics(config *Config) string { 26 | var metrics strings.Builder 27 | 28 | valueOfMetrics := reflect.ValueOf(m).Elem() 29 | 30 | hostname := config.MetricsHost 31 | if hostname == "" { 32 | hostname, _ = os.Hostname() 33 | } 34 | 35 | for i := 0; i < valueOfMetrics.NumField(); i++ { 36 | metricTag := valueOfMetrics.Type().Field(i).Tag.Get("metric") 37 | if metricTag == "" { 38 | continue 39 | } 40 | fieldValue := valueOfMetrics.Field(i).Addr().Interface().(*atomic.Int64).Load() 41 | 42 | metrics.WriteString(fmt.Sprintf("%s{host=\"%s\"} %v\n", metricTag, hostname, fieldValue)) 43 | 44 | } 45 | 46 | return metrics.String() 47 | } 48 | 49 | // wrap a reader to count bytes read into the counter 50 | func metricsReader(reader io.Reader, counter *atomic.Int64) readerClosure { 51 | return func(p []byte) (int, error) { 52 | bytesRead, err := reader.Read(p) 53 | counter.Add(int64(bytesRead)) 54 | return bytesRead, err 55 | } 56 | } 57 | 58 | type metricsReadCloser struct { 59 | io.ReadCloser 60 | counter *atomic.Int64 61 | } 62 | 63 | // Read reads data from the underlying io.ReadCloser, tracking the bytes read 64 | func (mrc metricsReadCloser) Read(p []byte) (int, error) { 65 | bytesRead, err := mrc.ReadCloser.Read(p) 66 | mrc.counter.Add(int64(bytesRead)) 67 | return bytesRead, err 68 | } 69 | 70 | // Close closes the underlying io.ReadCloser and returns the result 71 | func (mrc metricsReadCloser) Close() error { 72 | return mrc.ReadCloser.Close() 73 | } 74 | 75 | // http endpoint to render the global metrics 76 | func metricsHandler(w http.ResponseWriter, r *http.Request) error { 77 | w.Header().Set("Content-Type", "text/plain") 78 | w.Write([]byte(globalMetrics.RenderMetrics(globalConfig))) 79 | return nil 80 | } 81 | -------------------------------------------------------------------------------- /zipserver/metrics_test.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func Test_Metrics(t *testing.T) { 12 | metrics := &MetricsCounter{} 13 | 14 | // Test initial values 15 | assert.Equal(t, int64(0), metrics.TotalRequests.Load()) 16 | assert.Equal(t, int64(0), metrics.TotalExtractedFiles.Load()) 17 | 18 | metrics.TotalRequests.Add(1) 19 | assert.Equal(t, int64(1), metrics.TotalRequests.Load()) 20 | 21 | metrics.TotalExtractedFiles.Add(1) 22 | assert.Equal(t, int64(1), metrics.TotalExtractedFiles.Load()) 23 | 24 | // create a temp byte buffer wrapped in metricsReader to test updating BytesDownloaded 25 | buf := bytes.NewBufferString("testing") 26 | reader := metricsReader(buf, &metrics.TotalBytesDownloaded) 27 | 28 | // Read from the reader to trigger the metrics update 29 | _, _ = ioutil.ReadAll(reader) 30 | 31 | config := &Config{ 32 | MetricsHost: "localhost", 33 | } 34 | 35 | expectedMetrics := `zipserver_requests_total{host="localhost"} 1 36 | zipserver_errors_total{host="localhost"} 0 37 | zipserver_extracted_files_total{host="localhost"} 1 38 | zipserver_copied_files_total{host="localhost"} 0 39 | zipserver_downloaded_bytes_total{host="localhost"} 7 40 | zipserver_uploaded_bytes_total{host="localhost"} 0 41 | ` 42 | assert.Equal(t, expectedMetrics, metrics.RenderMetrics(config)) 43 | } 44 | -------------------------------------------------------------------------------- /zipserver/readers.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "time" 8 | ) 9 | 10 | type readerClosure func(p []byte) (int, error) 11 | 12 | func (fn readerClosure) Read(p []byte) (int, error) { 13 | return fn(p) 14 | } 15 | 16 | // debug reader 17 | func annotatedReader(reader io.Reader) readerClosure { 18 | return func(p []byte) (int, error) { 19 | bytesRead, err := reader.Read(p) 20 | log.Printf("Read %d bytes", bytesRead) 21 | return bytesRead, err 22 | } 23 | } 24 | 25 | // wraps a reader to fail if it reads more than max of maxBytes, also tracks 26 | // the total amount of bytes read 27 | func limitedReader(reader io.Reader, maxBytes uint64, totalBytes *uint64) readerClosure { 28 | return func(p []byte) (int, error) { 29 | bytesRead, err := reader.Read(p) 30 | *totalBytes += uint64(bytesRead) 31 | 32 | if *totalBytes > maxBytes { 33 | return bytesRead, fmt.Errorf("File too large (max %d bytes)", maxBytes) 34 | } 35 | 36 | return bytesRead, err 37 | } 38 | } 39 | 40 | type measuredReader struct { 41 | reader io.Reader // The underlying reader 42 | BytesRead int64 // Total bytes read 43 | StartTime time.Time // Time when reading started 44 | Duration time.Duration // Duration of the read operation 45 | } 46 | 47 | func newMeasuredReader(r io.Reader) *measuredReader { 48 | return &measuredReader{ 49 | reader: r, 50 | StartTime: time.Now(), 51 | } 52 | } 53 | 54 | // Read reads data from the underlying io.Reader, tracking the bytes read and duration 55 | func (mr *measuredReader) Read(p []byte) (int, error) { 56 | n, err := mr.reader.Read(p) 57 | mr.BytesRead += int64(n) 58 | mr.Duration = time.Since(mr.StartTime) 59 | 60 | return n, err 61 | } 62 | 63 | // TransferSpeed returns the average transfer speed in bytes per second 64 | func (mr *measuredReader) TransferSpeed() float64 { 65 | if mr.Duration.Seconds() == 0 { 66 | return 0 67 | } 68 | return float64(mr.BytesRead) / mr.Duration.Seconds() 69 | } 70 | -------------------------------------------------------------------------------- /zipserver/readers_test.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func Test_annotatedReader(t *testing.T) { 12 | s := "Hello, world" 13 | 14 | sr := bytes.NewReader([]byte(s)) 15 | ar := annotatedReader(sr) 16 | 17 | buf := make([]byte, 4) 18 | var totalBytes int 19 | for { 20 | n, err := ar.Read(buf) 21 | totalBytes += n 22 | if err == io.EOF { 23 | break 24 | } 25 | assert.NoError(t, err) 26 | } 27 | assert.EqualValues(t, len(s), totalBytes) 28 | } 29 | 30 | func Test_limitedReader(t *testing.T) { 31 | s := "Hello, world" 32 | 33 | sr := bytes.NewReader([]byte(s)) 34 | var totalBytes uint64 35 | lr := limitedReader(sr, 128, &totalBytes) 36 | 37 | result, err := io.ReadAll(lr) 38 | assert.NoError(t, err) 39 | assert.EqualValues(t, s, string(result)) 40 | assert.EqualValues(t, len(s), totalBytes) 41 | 42 | sr.Seek(0, io.SeekStart) 43 | lr = limitedReader(sr, 5, &totalBytes) 44 | _, err = io.ReadAll(lr) 45 | assert.Error(t, err) 46 | } 47 | -------------------------------------------------------------------------------- /zipserver/s3_storage.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "context" 5 | "crypto/md5" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "net/url" 10 | "strconv" 11 | 12 | "github.com/aws/aws-sdk-go/aws" 13 | "github.com/aws/aws-sdk-go/aws/credentials" 14 | "github.com/aws/aws-sdk-go/aws/session" 15 | "github.com/aws/aws-sdk-go/service/s3" 16 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 17 | ) 18 | 19 | type S3Storage struct { 20 | Session *session.Session 21 | config *StorageConfig 22 | } 23 | 24 | func NewS3Storage(config *StorageConfig) (*S3Storage, error) { 25 | var creds *credentials.Credentials 26 | 27 | if config.S3AccessKeyID == "" || config.S3SecretKey == "" { 28 | creds = credentials.NewEnvCredentials() 29 | } else { 30 | creds = credentials.NewStaticCredentials(config.S3AccessKeyID, config.S3SecretKey, "") 31 | } 32 | 33 | sess, err := session.NewSession(&aws.Config{ 34 | Credentials: creds, 35 | Endpoint: aws.String(config.S3Endpoint), 36 | Region: aws.String(config.S3Region), 37 | }) 38 | 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | return &S3Storage{ 44 | config: config, 45 | Session: sess, 46 | }, nil 47 | } 48 | 49 | // upload file and return md5 checksum of transferred bytes 50 | func (c *S3Storage) PutFile(ctx context.Context, bucket, key string, contents io.Reader, uploadHeaders http.Header) (string, error) { 51 | uploader := s3manager.NewUploaderWithClient(s3.New(c.Session), func(u *s3manager.Uploader) { 52 | u.PartSize = 1024 * 1024 * 50 // 50Mb per part to avoid excess API calls 53 | }) 54 | 55 | contents = metricsReader(contents, &globalMetrics.TotalBytesUploaded) 56 | 57 | hash := md5.New() 58 | 59 | // duplicate reads into the md5 hasher 60 | multi := io.TeeReader(contents, hash) 61 | 62 | uploadInput := &s3manager.UploadInput{ 63 | Bucket: aws.String(bucket), 64 | Key: aws.String(key), 65 | Body: multi, 66 | } 67 | 68 | if contentType := uploadHeaders.Get("Content-Type"); contentType != "" { 69 | uploadInput.ContentType = aws.String(contentType) 70 | } 71 | 72 | if contentDisposition := uploadHeaders.Get("Content-Disposition"); contentDisposition != "" { 73 | uploadInput.ContentDisposition = aws.String(contentDisposition) 74 | } 75 | 76 | _, err := uploader.UploadWithContext(ctx, uploadInput) 77 | 78 | if err != nil { 79 | return "", err 80 | } 81 | 82 | // Compute the checksum from the hash. 83 | checksum := hash.Sum(nil) 84 | 85 | // Convert the checksum to a hexadecimal string. 86 | checksumStr := fmt.Sprintf("%x", checksum) 87 | 88 | return checksumStr, nil 89 | } 90 | 91 | // get some specific metadata for file 92 | func (c *S3Storage) HeadFile(ctx context.Context, bucket, key string) (url.Values, error) { 93 | svc := s3.New(c.Session) 94 | input := &s3.HeadObjectInput{ 95 | Bucket: aws.String(bucket), 96 | Key: aws.String(key), 97 | } 98 | 99 | result, err := svc.HeadObjectWithContext(ctx, input) 100 | if err != nil { 101 | return nil, err 102 | } 103 | 104 | out := url.Values{} 105 | if result.ChecksumSHA256 != nil { 106 | out.Add("ChecksumSHA256", *result.ChecksumSHA256) 107 | } 108 | 109 | if result.ContentType != nil { 110 | out.Add("ContentType", *result.ContentType) 111 | } 112 | 113 | if result.ContentLength != nil { 114 | out.Add("ContentLength", strconv.FormatInt(*result.ContentLength, 10)) 115 | } 116 | 117 | return out, nil 118 | } 119 | 120 | func (c *S3Storage) DeleteFile(ctx context.Context, bucket, key string) error { 121 | svc := s3.New(c.Session) 122 | input := &s3.DeleteObjectInput{ 123 | Bucket: aws.String(bucket), 124 | Key: aws.String(key), 125 | } 126 | 127 | _, err := svc.DeleteObjectWithContext(ctx, input) 128 | if err != nil { 129 | return err 130 | } 131 | 132 | return nil 133 | } 134 | -------------------------------------------------------------------------------- /zipserver/serve_zip.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "log" 8 | "net/http" 9 | "os" 10 | "strings" 11 | "time" 12 | 13 | errors "github.com/go-errors/errors" 14 | ) 15 | 16 | type memoryHttpHandler struct { 17 | storage *MemStorage 18 | bucket string 19 | prefix string 20 | fileGetTimeout time.Duration 21 | } 22 | 23 | var _ http.Handler = (*memoryHttpHandler)(nil) 24 | 25 | func printError(err error) { 26 | if se, ok := err.(*errors.Error); ok { 27 | log.Printf("error: %s", se.ErrorStack()) 28 | } else { 29 | log.Printf("error: %s", se.Error()) 30 | } 31 | } 32 | 33 | func dumpError(w http.ResponseWriter, err error) { 34 | printError(err) 35 | w.WriteHeader(500) 36 | w.Write([]byte("Internal error")) 37 | } 38 | 39 | func (mhh *memoryHttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 40 | path := strings.TrimPrefix(r.URL.Path, "/") 41 | 42 | objectPath := fmt.Sprintf("%s/%s", mhh.prefix, path) 43 | log.Printf("Requesting %s", objectPath) 44 | 45 | ctx, cancel := context.WithTimeout(r.Context(), mhh.fileGetTimeout) 46 | defer cancel() 47 | 48 | reader, headers, err := mhh.storage.GetFile(ctx, mhh.bucket, objectPath) 49 | if err != nil { 50 | printError(err) 51 | w.WriteHeader(404) 52 | w.Write([]byte("Not found")) 53 | return 54 | } 55 | defer reader.Close() 56 | 57 | if headers != nil { 58 | log.Printf("Headers: %v", headers) 59 | 60 | for k, vv := range headers { 61 | for _, v := range vv { 62 | w.Header().Add(k, v) 63 | } 64 | } 65 | } 66 | 67 | w.WriteHeader(200) 68 | 69 | _, err = io.Copy(w, reader) 70 | if err != nil { 71 | dumpError(w, err) 72 | return 73 | } 74 | } 75 | 76 | // ServeZip takes the path to zip file in the local fs and serves 77 | // it as http 78 | func ServeZip(config *Config, serve string) error { 79 | config.Bucket = "local" 80 | 81 | storage, err := NewMemStorage() 82 | if err != nil { 83 | return errors.Wrap(err, 0) 84 | } 85 | 86 | reader, err := os.Open(serve) 87 | if err != nil { 88 | return errors.Wrap(err, 0) 89 | } 90 | 91 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(config.JobTimeout)) 92 | defer cancel() 93 | 94 | putCtx, putCtxCancel := context.WithTimeout(ctx, time.Duration(config.FilePutTimeout)) 95 | defer putCtxCancel() 96 | 97 | key := "serve.zip" 98 | err = storage.PutFile(putCtx, config.Bucket, key, reader, "application/zip") 99 | if err != nil { 100 | return errors.Wrap(err, 0) 101 | } 102 | 103 | archiver := &Archiver{storage, config} 104 | 105 | prefix := "extracted" 106 | _, err = archiver.ExtractZip(ctx, key, prefix, DefaultExtractLimits(config)) 107 | if err != nil { 108 | return errors.Wrap(err, 0) 109 | } 110 | 111 | handler := &memoryHttpHandler{ 112 | storage: storage, 113 | bucket: config.Bucket, 114 | prefix: prefix, 115 | fileGetTimeout: time.Duration(config.FileGetTimeout), 116 | } 117 | 118 | s := &http.Server{ 119 | Addr: "localhost:8091", 120 | Handler: handler, 121 | } 122 | log.Printf("Listening on %s...", s.Addr) 123 | return s.ListenAndServe() 124 | } 125 | -------------------------------------------------------------------------------- /zipserver/server.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "net/url" 8 | "strconv" 9 | 10 | "fmt" 11 | ) 12 | 13 | var globalConfig *Config 14 | 15 | type wrapErrors func(http.ResponseWriter, *http.Request) error 16 | 17 | func (fn wrapErrors) ServeHTTP(w http.ResponseWriter, r *http.Request) { 18 | globalMetrics.TotalRequests.Add(1) 19 | 20 | if err := fn(w, r); err != nil { 21 | globalMetrics.TotalErrors.Add(1) 22 | log.Println("Error", r.Method, r.URL.Path, err) 23 | http.Error(w, err.Error(), 500) 24 | } 25 | } 26 | 27 | // get the first value of param or error 28 | func getParam(params url.Values, name string) (string, error) { 29 | val := params.Get(name) 30 | 31 | if val == "" { 32 | return "", fmt.Errorf("Missing param %v", name) 33 | } 34 | 35 | return val, nil 36 | } 37 | 38 | func getUint64Param(params url.Values, name string) (uint64, error) { 39 | valStr, err := getParam(params, name) 40 | if err != nil { 41 | return 0, err 42 | } 43 | 44 | valUint64, err := strconv.ParseUint(valStr, 10, 64) 45 | if err != nil { 46 | return 0, err 47 | } 48 | 49 | return valUint64, nil 50 | } 51 | 52 | func getIntParam(params url.Values, name string) (int, error) { 53 | valStr, err := getParam(params, name) 54 | if err != nil { 55 | return 0, err 56 | } 57 | 58 | valInt, err := strconv.Atoi(valStr) 59 | if err != nil { 60 | return 0, err 61 | } 62 | 63 | return valInt, nil 64 | } 65 | 66 | func writeJSONMessage(w http.ResponseWriter, msg interface{}) error { 67 | blob, err := json.Marshal(msg) 68 | if err != nil { 69 | return err 70 | } 71 | w.Header()["Content-Type"] = []string{"application/json"} 72 | w.Write(blob) 73 | return nil 74 | } 75 | 76 | func writeJSONError(w http.ResponseWriter, kind string, err error) error { 77 | return writeJSONMessage(w, struct { 78 | Type string 79 | Error string 80 | }{kind, err.Error()}) 81 | } 82 | 83 | func statusHandler(w http.ResponseWriter, r *http.Request) error { 84 | copyKeys := copyLockTable.GetLocks() 85 | extractKeys := extractLockTable.GetLocks() 86 | 87 | return writeJSONMessage(w, struct { 88 | CopyLocks []KeyInfo `json:"copy_locks"` 89 | ExtractLocks []KeyInfo `json:"extract_locks"` 90 | }{ 91 | CopyLocks: copyKeys, 92 | ExtractLocks: extractKeys, 93 | }) 94 | } 95 | 96 | // StartZipServer starts listening for extract and slurp requests 97 | func StartZipServer(listenTo string, _config *Config) error { 98 | globalConfig = _config 99 | 100 | // Extract a .zip file (downloaded from GCS), stores each 101 | // individual file on GCS in a given bucket/prefix 102 | http.Handle("/extract", wrapErrors(extractHandler)) 103 | 104 | http.Handle("/copy", wrapErrors(copyHandler)) 105 | 106 | // show the files in the zip 107 | http.Handle("/list", wrapErrors(listHandler)) 108 | 109 | // Download a file from an http{,s} URL and store it on GCS 110 | http.Handle("/slurp", wrapErrors(slurpHandler)) 111 | 112 | http.Handle("/status", wrapErrors(statusHandler)) 113 | http.Handle("/metrics", wrapErrors(metricsHandler)) 114 | 115 | log.Print("Listening on: " + listenTo) 116 | return http.ListenAndServe(listenTo, nil) 117 | } 118 | -------------------------------------------------------------------------------- /zipserver/slurp_handler.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net/http" 10 | "net/url" 11 | "strconv" 12 | "time" 13 | ) 14 | 15 | var slurpLockTable = NewLockTable() 16 | 17 | func slurpHandler(w http.ResponseWriter, r *http.Request) error { 18 | ctx, cancel := context.WithTimeout(r.Context(), time.Duration(globalConfig.JobTimeout)) 19 | defer cancel() 20 | 21 | params := r.URL.Query() 22 | 23 | key, err := getParam(params, "key") 24 | if err != nil { 25 | return err 26 | } 27 | 28 | slurpURL, err := getParam(params, "url") 29 | if err != nil { 30 | return err 31 | } 32 | 33 | contentType := params.Get("content_type") 34 | maxBytesStr := params.Get("max_bytes") 35 | acl := params.Get("acl") 36 | contentDisposition := params.Get("content_disposition") 37 | 38 | var maxBytes uint64 39 | if maxBytesStr != "" { 40 | maxBytes, err = strconv.ParseUint(maxBytesStr, 10, 64) 41 | if err != nil { 42 | return err 43 | } 44 | } 45 | 46 | process := func(ctx context.Context) error { 47 | if !slurpLockTable.tryLockKey(key) { 48 | return fmt.Errorf("Key is currently being processed: %s", key) 49 | } 50 | defer slurpLockTable.releaseKey(key) 51 | 52 | getCtx, cancel := context.WithTimeout(ctx, time.Duration(globalConfig.FileGetTimeout)) 53 | defer cancel() 54 | 55 | log.Print("Fetching URL: ", slurpURL) 56 | 57 | req, err := http.NewRequestWithContext(getCtx, http.MethodGet, slurpURL, nil) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | res, err := http.DefaultClient.Do(req) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | defer res.Body.Close() 68 | 69 | if res.StatusCode != 200 { 70 | return fmt.Errorf("Failed to fetch file: %d", res.StatusCode) 71 | } 72 | 73 | if contentType == "" { 74 | contentType = res.Header.Get("Content-Type") 75 | } 76 | 77 | if contentType == "" { 78 | contentType = "application/octet-stream" 79 | } 80 | 81 | body := io.Reader(res.Body) 82 | 83 | if maxBytes > 0 { 84 | if uint64(res.ContentLength) > maxBytes { 85 | return fmt.Errorf("Content-Length is greater than max bytes (%d > %d)", 86 | res.ContentLength, maxBytes) 87 | } 88 | 89 | var bytesRead uint64 90 | body = limitedReader(body, maxBytes, &bytesRead) 91 | } 92 | 93 | log.Print("Uploading ", contentType, " (size: ", res.ContentLength, ") to ", key) 94 | log.Print("ACL: ", acl) 95 | log.Print("Content-Disposition: ", contentDisposition) 96 | 97 | storage, err := NewGcsStorage(globalConfig) 98 | 99 | if storage == nil { 100 | log.Fatal("Failed to create storage:", err) 101 | } 102 | 103 | putCtx, cancel := context.WithTimeout(ctx, time.Duration(globalConfig.FilePutTimeout)) 104 | defer cancel() 105 | 106 | return storage.PutFileWithSetup(putCtx, globalConfig.Bucket, key, body, func(req *http.Request) error { 107 | req.Header.Add("Content-Type", contentType) 108 | 109 | if contentDisposition != "" { 110 | req.Header.Add("Content-Disposition", contentDisposition) 111 | } 112 | 113 | req.Header.Add("x-goog-acl", acl) 114 | return nil 115 | }) 116 | } 117 | 118 | asyncURL := params.Get("async") 119 | if asyncURL == "" { 120 | err = process(ctx) 121 | if err != nil { 122 | return writeJSONError(w, "SlurpError", err) 123 | } 124 | 125 | return writeJSONMessage(w, struct { 126 | Success bool 127 | }{true}) 128 | } 129 | 130 | go (func() { 131 | // This job is expected to outlive the incoming request, so create a detached context. 132 | ctx := context.Background() 133 | 134 | err = process(ctx) 135 | log.Print("Notifying " + asyncURL) 136 | 137 | resValues := url.Values{} 138 | if err != nil { 139 | resValues.Add("Type", "SlurpError") 140 | resValues.Add("Error", err.Error()) 141 | } else { 142 | resValues.Add("Success", "true") 143 | } 144 | 145 | ctx, cancel := context.WithTimeout(ctx, time.Duration(globalConfig.AsyncNotificationTimeout)) 146 | defer cancel() 147 | 148 | outBody := bytes.NewBufferString(resValues.Encode()) 149 | req, err := http.NewRequestWithContext(ctx, http.MethodPost, asyncURL, outBody) 150 | if err != nil { 151 | log.Printf("Failed to create callback request: %v", err) 152 | } 153 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded") 154 | 155 | _, err = http.DefaultClient.Do(req) 156 | if err != nil { 157 | log.Print("Failed to deliver callback: " + err.Error()) 158 | } 159 | })() 160 | 161 | return writeJSONMessage(w, struct { 162 | Processing bool 163 | Async bool 164 | }{true, true}) 165 | } 166 | -------------------------------------------------------------------------------- /zipserver/specifications.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "path" 7 | "strings" 8 | ) 9 | 10 | // ResourceSpec contains all the info for an HTTP resource relevant for 11 | // setting http headers and keeping track of the extraction work 12 | type ResourceSpec struct { 13 | size uint64 14 | key string 15 | contentType string 16 | contentEncoding string 17 | } 18 | 19 | func (rs *ResourceSpec) String() string { 20 | formattedEncoding := "" 21 | if rs.contentEncoding != "" { 22 | formattedEncoding = fmt.Sprintf(", %s encoding", rs.contentEncoding) 23 | } 24 | 25 | return fmt.Sprintf("%s (%s%s)", rs.key, rs.contentType, formattedEncoding) 26 | } 27 | 28 | // setupRequest sets the proper HTTP headers on a request for storing this resource 29 | func (rs *ResourceSpec) setupRequest(req *http.Request) error { 30 | // All extracted files must be readable without authentication 31 | req.Header.Set("x-goog-acl", "public-read") 32 | 33 | req.Header.Set("content-type", rs.contentType) 34 | if rs.contentEncoding != "" { 35 | req.Header.Set("content-encoding", rs.contentEncoding) 36 | } 37 | return nil 38 | } 39 | 40 | // RewriteSpec contains rules for rewriting file extensions 41 | type RewriteSpec struct { 42 | oldExtension string 43 | newExtension string 44 | } 45 | 46 | var rewriteSpecs = []RewriteSpec{ 47 | // // For Unity WebGL up to 5.5, see 48 | // // https://docs.unity3d.com/550/Documentation/Manual/webgl-deploying.html 49 | {".jsgz", ".js"}, 50 | {".datagz", ".data"}, 51 | {".memgz", ".mem"}, 52 | {".unity3dgz", ".unity3d"}, 53 | } 54 | 55 | func (rs *ResourceSpec) applyRewriteRules() { 56 | // rewrite rules only apply when we've identified the gzip suffix 57 | if rs.contentEncoding != "gzip" { 58 | return 59 | } 60 | 61 | extension := path.Ext(rs.key) 62 | 63 | for _, spec := range rewriteSpecs { 64 | if extension == spec.oldExtension { 65 | rs.key = strings.TrimSuffix(rs.key, spec.oldExtension) + spec.newExtension 66 | // only apply one rule at most 67 | return 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /zipserver/storage.go: -------------------------------------------------------------------------------- 1 | package zipserver 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "net/http" 7 | ) 8 | 9 | // StorageSetupFunc gives the consumer a chance to set HTTP headers before storing something 10 | type StorageSetupFunc func(*http.Request) error 11 | 12 | // Storage is a place we can get files from, put files into, or delete files from 13 | type Storage interface { 14 | GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, http.Header, error) 15 | PutFile(ctx context.Context, bucket, key string, contents io.Reader, mimeType string) error 16 | PutFileWithSetup(ctx context.Context, bucket, key string, contents io.Reader, setup StorageSetupFunc) error 17 | DeleteFile(ctx context.Context, bucket, key string) error 18 | } 19 | --------------------------------------------------------------------------------