├── .gitignore ├── .travis.yml ├── .vscode └── spell.json ├── Godeps ├── Godeps.json └── Readme ├── Gopkg.lock ├── Gopkg.toml ├── LICENSE ├── README.md ├── SECURITY.md ├── _build └── build.cmd ├── _config.yml ├── args.go ├── args_test.go ├── blobporter.go ├── docs ├── Makefile ├── README.md ├── bptransfer.png ├── conf.py ├── examples.rst ├── gettingstarted.rst ├── index.rst ├── make.bat ├── perfmode.rst └── resumabletrans.rst ├── files.sh ├── go.mod ├── go.sum ├── img └── bptransfer.png ├── internal ├── azutil.go ├── const.go ├── eventsink.go ├── eventsink_test.go ├── handleman.go ├── tracker.go └── tracker_test.go ├── inttest._m.sh ├── inttest.sh ├── pipeline ├── pipeline.go └── pipeline_test.go ├── pipelinefactory.go ├── progstate.go ├── sources ├── azblobinfo.go ├── fileinfo.go ├── http.go ├── multifile.go ├── perfsource.go ├── s3info.go ├── sourcefactory.go └── types.go ├── targets ├── azureblock.go ├── azurepage.go ├── multifile.go ├── params.go └── perftarget.go ├── transfer ├── commit.go ├── transfer.go ├── transfer_test.go └── worker.go ├── util ├── util.go └── util_test.go └── vendor ├── github.com ├── Azure │ ├── azure-pipeline-go │ │ ├── LICENSE │ │ └── pipeline │ │ │ ├── core.go │ │ │ ├── defaultlog_syslog.go │ │ │ ├── defaultlog_windows.go │ │ │ ├── doc.go │ │ │ ├── error.go │ │ │ ├── progress.go │ │ │ ├── request.go │ │ │ ├── response.go │ │ │ └── version.go │ └── azure-storage-blob-go │ │ ├── LICENSE │ │ └── azblob │ │ ├── access_conditions.go │ │ ├── atomicmorph.go │ │ ├── highlevel.go │ │ ├── parsing_urls.go │ │ ├── sas_service.go │ │ ├── service_codes_blob.go │ │ ├── url_append_blob.go │ │ ├── url_blob.go │ │ ├── url_block_blob.go │ │ ├── url_container.go │ │ ├── url_page_blob.go │ │ ├── url_service.go │ │ ├── version.go │ │ ├── zc_credential_anonymous.go │ │ ├── zc_credential_shared_key.go │ │ ├── zc_credential_token.go │ │ ├── zc_mmf_unix.go │ │ ├── zc_mmf_windows.go │ │ ├── zc_pipeline.go │ │ ├── zc_policy_request_log.go │ │ ├── zc_policy_retry.go │ │ ├── zc_policy_telemetry.go │ │ ├── zc_policy_unique_request_id.go │ │ ├── zc_retry_reader.go │ │ ├── zc_sas_account.go │ │ ├── zc_sas_query_params.go │ │ ├── zc_service_codes_common.go │ │ ├── zc_storage_error.go │ │ ├── zc_util_validate.go │ │ ├── zc_uuid.go │ │ ├── zt_doc.go │ │ ├── zz_generated_append_blob.go │ │ ├── zz_generated_blob.go │ │ ├── zz_generated_block_blob.go │ │ ├── zz_generated_client.go │ │ ├── zz_generated_container.go │ │ ├── zz_generated_models.go │ │ ├── zz_generated_page_blob.go │ │ ├── zz_generated_responder_policy.go │ │ ├── zz_generated_response_error.go │ │ ├── zz_generated_service.go │ │ ├── zz_generated_validation.go │ │ ├── zz_generated_version.go │ │ └── zz_response_helpers.go ├── davecgh │ └── go-spew │ │ ├── LICENSE │ │ └── spew │ │ ├── bypass.go │ │ ├── bypasssafe.go │ │ ├── common.go │ │ ├── config.go │ │ ├── doc.go │ │ ├── dump.go │ │ ├── format.go │ │ └── spew.go ├── go-ini │ └── ini │ │ ├── .gitignore │ │ ├── .travis.yml │ │ ├── LICENSE │ │ ├── Makefile │ │ ├── README.md │ │ ├── error.go │ │ ├── file.go │ │ ├── ini.go │ │ ├── key.go │ │ ├── parser.go │ │ ├── section.go │ │ └── struct.go ├── minio │ └── minio-go │ │ ├── .gitignore │ │ ├── .travis.yml │ │ ├── CONTRIBUTING.md │ │ ├── LICENSE │ │ ├── MAINTAINERS.md │ │ ├── Makefile │ │ ├── NOTICE │ │ ├── README.md │ │ ├── README_zh_CN.md │ │ ├── api-compose-object.go │ │ ├── api-datatypes.go │ │ ├── api-error-response.go │ │ ├── api-get-object-context.go │ │ ├── api-get-object-file.go │ │ ├── api-get-object.go │ │ ├── api-get-options.go │ │ ├── api-get-policy.go │ │ ├── api-list.go │ │ ├── api-notification.go │ │ ├── api-presigned.go │ │ ├── api-put-bucket.go │ │ ├── api-put-object-common.go │ │ ├── api-put-object-context.go │ │ ├── api-put-object-copy.go │ │ ├── api-put-object-encrypted.go │ │ ├── api-put-object-file-context.go │ │ ├── api-put-object-file.go │ │ ├── api-put-object-multipart.go │ │ ├── api-put-object-streaming.go │ │ ├── api-put-object.go │ │ ├── api-remove.go │ │ ├── api-s3-datatypes.go │ │ ├── api-stat.go │ │ ├── api.go │ │ ├── appveyor.yml │ │ ├── bucket-cache.go │ │ ├── bucket-notification.go │ │ ├── constants.go │ │ ├── core.go │ │ ├── functional_tests.go │ │ ├── hook-reader.go │ │ ├── pkg │ │ ├── credentials │ │ │ ├── chain.go │ │ │ ├── config.json.sample │ │ │ ├── credentials.go │ │ │ ├── credentials.sample │ │ │ ├── doc.go │ │ │ ├── env_aws.go │ │ │ ├── env_minio.go │ │ │ ├── file_aws_credentials.go │ │ │ ├── file_minio_client.go │ │ │ ├── iam_aws.go │ │ │ ├── signature-type.go │ │ │ └── static.go │ │ ├── encrypt │ │ │ ├── cbc.go │ │ │ ├── interface.go │ │ │ └── keys.go │ │ ├── policy │ │ │ ├── bucket-policy-condition.go │ │ │ └── bucket-policy.go │ │ ├── s3signer │ │ │ ├── request-signature-streaming.go │ │ │ ├── request-signature-v2.go │ │ │ ├── request-signature-v4.go │ │ │ └── utils.go │ │ ├── s3utils │ │ │ └── utils.go │ │ └── set │ │ │ └── stringset.go │ │ ├── post-policy.go │ │ ├── retry-continous.go │ │ ├── retry.go │ │ ├── s3-endpoints.go │ │ ├── s3-error.go │ │ ├── transport.go │ │ └── utils.go ├── mitchellh │ └── go-homedir │ │ ├── LICENSE │ │ ├── README.md │ │ └── homedir.go ├── pmezard │ └── go-difflib │ │ ├── LICENSE │ │ └── difflib │ │ └── difflib.go └── stretchr │ └── testify │ ├── LICENSE │ └── assert │ ├── assertion_format.go │ ├── assertion_format.go.tmpl │ ├── assertion_forward.go │ ├── assertion_forward.go.tmpl │ ├── assertions.go │ ├── doc.go │ ├── errors.go │ ├── forward_assertions.go │ └── http_assertions.go ├── golang.org └── x │ ├── net │ ├── AUTHORS │ ├── CONTRIBUTORS │ ├── LICENSE │ ├── PATENTS │ ├── idna │ │ ├── idna.go │ │ ├── punycode.go │ │ ├── tables.go │ │ ├── trie.go │ │ └── trieval.go │ └── lex │ │ └── httplex │ │ └── httplex.go │ └── text │ ├── AUTHORS │ ├── CONTRIBUTORS │ ├── LICENSE │ ├── PATENTS │ ├── secure │ └── bidirule │ │ ├── bidirule.go │ │ ├── bidirule10.0.0.go │ │ └── bidirule9.0.0.go │ ├── transform │ └── transform.go │ └── unicode │ ├── bidi │ ├── bidi.go │ ├── bracket.go │ ├── core.go │ ├── gen.go │ ├── gen_ranges.go │ ├── gen_trieval.go │ ├── prop.go │ ├── tables10.0.0.go │ ├── tables9.0.0.go │ └── trieval.go │ └── norm │ ├── composition.go │ ├── forminfo.go │ ├── input.go │ ├── iter.go │ ├── maketables.go │ ├── normalize.go │ ├── readwriter.go │ ├── tables10.0.0.go │ ├── tables9.0.0.go │ ├── transform.go │ ├── trie.go │ └── triegen.go └── modules.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | __*.* 6 | 7 | # Folders 8 | _obj 9 | _test 10 | _build/ 11 | _build/linux_amd64 12 | _build/windows_amd64 13 | _wd 14 | _old 15 | _vendor* 16 | .vscode 17 | *_testdata 18 | 19 | # Architecture specific extensions/prefixes 20 | *.[568vq] 21 | [568vq].out 22 | debug 23 | launch.json 24 | *.cgo1.go 25 | *.cgo2.c 26 | _cgo_defun.c 27 | _cgo_gotypes.go 28 | _cgo_export.* 29 | 30 | _testmain.go 31 | 32 | *.exe 33 | *.test 34 | *.prof 35 | 36 | # Output of the go coverage tool, specifically when used with LiteIDE 37 | *.out 38 | 39 | # external packages folder 40 | #vendor/ 41 | .vscode/launch.json 42 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | language: go 3 | go: 4 | - 1.11 5 | before_install: 6 | - sudo apt-get install zip 7 | # Download the binary to bin folder in $GOPATH 8 | - curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep 9 | # Make the binary executable 10 | #- chmod +x $GOPATH/bin/dep 11 | #- go get -d github.com/stretchr/testify/assert 12 | install: 13 | - export GO111MODULE=on 14 | script: 15 | - go test -v ./... 16 | - mkdir linux_amd64 17 | - mkdir windows_amd64 18 | - go build -o ./linux_amd64/blobporter github.com/Azure/blobporter 19 | - export GOOS=windows 20 | - go build -o ./windows_amd64/BlobPorter.exe github.com/Azure/blobporter 21 | - cp LICENSE ./linux_amd64/LICENSE 22 | - cp LICENSE ./windows_amd64/LICENSE 23 | - md5sum ./linux_amd64/blobporter > ./linux_amd64/blobporter.md5 24 | - md5sum ./windows_amd64/BlobPorter.exe > ./windows_amd64/BlobPorter.exe.md5 25 | - tar cvzf bp_linux.tar.gz linux_amd64 26 | - zip bp_windows.zip ./windows_amd64/* 27 | - bash inttest.sh inttest 28 | deploy: 29 | skip_cleanup: true 30 | provider: releases 31 | api_key: 32 | secure: OluucIYf78l/1cMQq5MvJxGdwXS1tSxingFi9mS9onI7RPK3qXdUWY1AHxd3xlJo4vjAlPwSA4NREm8k0zcq3tTtJOVsbREoZpa7Dhbn8zkJlVuurDK15HAlU1m0F2zo1LvlNGjulQj+MF/Tk676kVtRZqd8KQ3ivTk7IkL5Q5/uMnp8/gYtHWfJZiVdVznqf+enel4ghH1zDc1a6W/SkuTTfP2lmkzJrit4H1YIzOcW+DDOAuAqmgOJPFZdfQs3kKwCEzcXVKjUMe+iiyO0yNfvtmT1QqyI48VC65y/3sPXKWYBA40O0VZ8vAQtwV2IDdYwSk1X5dqgU5U7vnrMYogyF/x5PFkBdbfALJFIZQZf07rPeVjuX6l5uskf4i3TN+bHyRElvnYshLlN9QTQkoVyHHLtoH2BCbnZhPL81S9nfIbQgp5ldIdizQgSTiClhRDdOOoLErMT4yY0NU+EO4edjKgpLGC8+OG0p6wXoNGXhPfuaTY/4cYEfnO5EW3kJk2T0iMaxhcOpPaz6xcgK6h99pZfzzdUYkyhmKld4Fb3bxULvRrHo13CBxnOlUDid4jn5Ckl1HtsmtdoY4qfUQnJumOj8C92/Zub1J0smVV05yuj8r1OiGk75Q4nTxrkSL19dsrZnd6ZitKrthOr1F7WBb8N913g8lXqgn7APAw= 33 | file: 34 | - bp_linux.tar.gz 35 | - bp_windows.zip 36 | on: 37 | repo: Azure/blobporter 38 | tags: true -------------------------------------------------------------------------------- /.vscode/spell.json: -------------------------------------------------------------------------------- 1 | { 2 | "language": "en", 3 | "ignoreWordsList": [ 4 | "i7Rfbrpu5VHgcmOShTg", 5 | "TODO", 6 | "mystorage", 7 | "oZcLo3zNU6aIo32NVzda4EJj0hjS2Jp7NVLAD3sFp7C67z", 8 | "myblob", 9 | "ZeroOnly", 10 | "SqGSr" 11 | ], 12 | "mistakeTypeToStatus": { 13 | "Passive voice": "Hint", 14 | "Spelling": "Error", 15 | "Complex Expression": "Disable", 16 | "Hidden Verbs": "Information", 17 | "Hyphen Required": "Disable", 18 | "Redundant Expression": "Disable", 19 | "Did you mean...": "Disable", 20 | "Repeated Word": "Warning", 21 | "Missing apostrophe": "Warning", 22 | "Cliches": "Disable", 23 | "Missing Word": "Disable", 24 | "Make I uppercase": "Warning" 25 | }, 26 | "languageIDs": [ 27 | "markdown", 28 | "plaintext" 29 | ], 30 | "ignoreRegExp": [ 31 | "/\\(.*\\.(jpg|jpeg|png|md|gif|JPG|JPEG|PNG|MD|GIF)\\)/g", 32 | "/((http|https|ftp|git)\\S*)/g" 33 | ] 34 | } -------------------------------------------------------------------------------- /Godeps/Godeps.json: -------------------------------------------------------------------------------- 1 | { 2 | "ImportPath": "github.com/Azure/blobporter", 3 | "GoVersion": "go1.9", 4 | "GodepVersion": "v79", 5 | "Deps": [ 6 | { 7 | "ImportPath": "github.com/Azure/azure-pipeline-go/pipeline", 8 | "Comment": "1.1.3", 9 | "Rev": "f4da77e3846319876aebb41f2e6dccc6b1d7f1e0" 10 | }, 11 | { 12 | "ImportPath": "github.com/Azure/azure-sdk-for-go/storage", 13 | "Comment": "v7.0.1-beta-14-gd05c22d", 14 | "Rev": "d05c22dc3f9fdd2ec6e3593839fdd44df17f2de5" 15 | }, 16 | { 17 | "ImportPath": "github.com/Azure/azure-storage-blob-go/2016-05-31/azblob", 18 | "Comment": "0.1.1-12-gfb32827", 19 | "Rev": "fb328272929aed4cd6e5108011abe489d6e445fa" 20 | }, 21 | { 22 | "ImportPath": "github.com/go-ini/ini", 23 | "Comment": "v1.32.0", 24 | "Rev": "32e4c1e6bc4e7d0d8451aa6b75200d19e37a536a" 25 | }, 26 | { 27 | "ImportPath": "github.com/minio/minio-go", 28 | "Comment": "4.0.6", 29 | "Rev": "14f1d472d115bac5ca4804094aa87484a72ced61" 30 | }, 31 | { 32 | "ImportPath": "github.com/minio/minio-go/pkg/credentials", 33 | "Comment": "4.0.6", 34 | "Rev": "14f1d472d115bac5ca4804094aa87484a72ced61" 35 | }, 36 | { 37 | "ImportPath": "github.com/minio/minio-go/pkg/encrypt", 38 | "Comment": "4.0.6", 39 | "Rev": "14f1d472d115bac5ca4804094aa87484a72ced61" 40 | }, 41 | { 42 | "ImportPath": "github.com/minio/minio-go/pkg/policy", 43 | "Comment": "4.0.6", 44 | "Rev": "14f1d472d115bac5ca4804094aa87484a72ced61" 45 | }, 46 | { 47 | "ImportPath": "github.com/minio/minio-go/pkg/s3signer", 48 | "Comment": "4.0.6", 49 | "Rev": "14f1d472d115bac5ca4804094aa87484a72ced61" 50 | }, 51 | { 52 | "ImportPath": "github.com/minio/minio-go/pkg/s3utils", 53 | "Comment": "4.0.6", 54 | "Rev": "14f1d472d115bac5ca4804094aa87484a72ced61" 55 | }, 56 | { 57 | "ImportPath": "github.com/minio/minio-go/pkg/set", 58 | "Comment": "4.0.6", 59 | "Rev": "14f1d472d115bac5ca4804094aa87484a72ced61" 60 | }, 61 | { 62 | "ImportPath": "github.com/mitchellh/go-homedir", 63 | "Rev": "b8bc1bf767474819792c23f32d8286a45736f1c6" 64 | }, 65 | { 66 | "ImportPath": "golang.org/x/net/idna", 67 | "Rev": "5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec" 68 | }, 69 | { 70 | "ImportPath": "golang.org/x/net/lex/httplex", 71 | "Rev": "5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec" 72 | }, 73 | { 74 | "ImportPath": "golang.org/x/text/secure/bidirule", 75 | "Rev": "e19ae1496984b1c655b8044a65c0300a3c878dd3" 76 | }, 77 | { 78 | "ImportPath": "golang.org/x/text/transform", 79 | "Rev": "e19ae1496984b1c655b8044a65c0300a3c878dd3" 80 | }, 81 | { 82 | "ImportPath": "golang.org/x/text/unicode/bidi", 83 | "Rev": "e19ae1496984b1c655b8044a65c0300a3c878dd3" 84 | }, 85 | { 86 | "ImportPath": "golang.org/x/text/unicode/norm", 87 | "Rev": "e19ae1496984b1c655b8044a65c0300a3c878dd3" 88 | } 89 | ] 90 | } 91 | -------------------------------------------------------------------------------- /Godeps/Readme: -------------------------------------------------------------------------------- 1 | This directory tree is generated automatically by godep. 2 | 3 | Please do not edit. 4 | 5 | See https://github.com/tools/godep for more information. 6 | -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | name = "github.com/Azure/azure-pipeline-go" 6 | packages = ["pipeline"] 7 | revision = "7571e8eb0876932ab505918ff7ed5107773e5ee2" 8 | version = "0.1.7" 9 | 10 | [[projects]] 11 | branch = "master" 12 | name = "github.com/Azure/azure-storage-blob-go" 13 | packages = ["2018-03-28/azblob"] 14 | revision = "f0a732ea9441c73511367d5861da6bda664ffabd" 15 | 16 | [[projects]] 17 | name = "github.com/davecgh/go-spew" 18 | packages = ["spew"] 19 | revision = "346938d642f2ec3594ed81d874461961cd0faa76" 20 | version = "v1.1.0" 21 | 22 | [[projects]] 23 | branch = "master" 24 | name = "github.com/dustin/go-humanize" 25 | packages = ["."] 26 | revision = "02af3965c54e8cacf948b97fef38925c4120652c" 27 | 28 | [[projects]] 29 | name = "github.com/go-ini/ini" 30 | packages = ["."] 31 | revision = "06f5f3d67269ccec1fe5fe4134ba6e982984f7f5" 32 | version = "v1.37.0" 33 | 34 | [[projects]] 35 | name = "github.com/minio/minio-go" 36 | packages = [ 37 | ".", 38 | "pkg/credentials", 39 | "pkg/encrypt", 40 | "pkg/policy", 41 | "pkg/s3signer", 42 | "pkg/s3utils", 43 | "pkg/set" 44 | ] 45 | revision = "9e124ec59547551cb3f1324f73623bbb30650cf8" 46 | version = "4.0.9" 47 | 48 | [[projects]] 49 | branch = "master" 50 | name = "github.com/mitchellh/go-homedir" 51 | packages = ["."] 52 | revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66" 53 | 54 | [[projects]] 55 | name = "github.com/pmezard/go-difflib" 56 | packages = ["difflib"] 57 | revision = "792786c7400a136282c1664665ae0a8db921c6c2" 58 | version = "v1.0.0" 59 | 60 | [[projects]] 61 | name = "github.com/sirupsen/logrus" 62 | packages = ["."] 63 | revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc" 64 | version = "v1.0.5" 65 | 66 | [[projects]] 67 | name = "github.com/stretchr/testify" 68 | packages = ["assert"] 69 | revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" 70 | version = "v1.2.2" 71 | 72 | [[projects]] 73 | branch = "master" 74 | name = "golang.org/x/crypto" 75 | packages = ["ssh/terminal"] 76 | revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602" 77 | 78 | [[projects]] 79 | branch = "release-branch.go1.10" 80 | name = "golang.org/x/net" 81 | packages = [ 82 | "idna", 83 | "lex/httplex" 84 | ] 85 | revision = "0ed95abb35c445290478a5348a7b38bb154135fd" 86 | 87 | [[projects]] 88 | branch = "master" 89 | name = "golang.org/x/sys" 90 | packages = [ 91 | "unix", 92 | "windows" 93 | ] 94 | revision = "1b2967e3c290b7c545b3db0deeda16e9be4f98a2" 95 | 96 | [[projects]] 97 | name = "golang.org/x/text" 98 | packages = [ 99 | "collate", 100 | "collate/build", 101 | "internal/colltab", 102 | "internal/gen", 103 | "internal/tag", 104 | "internal/triegen", 105 | "internal/ucd", 106 | "language", 107 | "secure/bidirule", 108 | "transform", 109 | "unicode/bidi", 110 | "unicode/cldr", 111 | "unicode/norm", 112 | "unicode/rangetable" 113 | ] 114 | revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" 115 | version = "v0.3.0" 116 | 117 | [solve-meta] 118 | analyzer-name = "dep" 119 | analyzer-version = 1 120 | inputs-digest = "38a337a1f2a24ec869a14255c1af2e0d97e4691c39249b4f8a18f63489df7d79" 121 | solver-name = "gps-cdcl" 122 | solver-version = 1 123 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | # Gopkg.toml example 2 | # 3 | # Refer to https://golang.github.io/dep/docs/Gopkg.toml.html 4 | # for detailed Gopkg.toml documentation. 5 | # 6 | # required = ["github.com/user/thing/cmd/thing"] 7 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 8 | # 9 | # [[constraint]] 10 | # name = "github.com/user/project" 11 | # version = "1.0.0" 12 | # 13 | # [[constraint]] 14 | # name = "github.com/user/project2" 15 | # branch = "dev" 16 | # source = "github.com/myfork/project2" 17 | # 18 | # [[override]] 19 | # name = "github.com/x/y" 20 | # version = "2.4.0" 21 | # 22 | # [prune] 23 | # non-go = false 24 | # go-tests = true 25 | # unused-packages = true 26 | 27 | 28 | [[constraint]] 29 | name = "github.com/Azure/azure-pipeline-go" 30 | version = "0.1.7" 31 | 32 | [[constraint]] 33 | branch = "master" 34 | name = "github.com/Azure/azure-storage-blob-go" 35 | 36 | [[constraint]] 37 | name = "github.com/minio/minio-go" 38 | version = "4.0.6" 39 | 40 | [[constraint]] 41 | name = "github.com/stretchr/testify" 42 | version = "1.2.2" 43 | 44 | [prune] 45 | go-tests = true 46 | unused-packages = true 47 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /_build/build.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | set GOOS=linux 3 | go build -o .\linux_amd64\blobporter github.com/Azure/blobporter 4 | set GOOS=windows 5 | go build -o .\windows_amd64\BlobPorter.exe github.com/Azure/blobporter 6 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = BlobPorter 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/blobporter/ac23782d56d2dc8aa8bc20303415255dea7ddeb0/docs/README.md -------------------------------------------------------------------------------- /docs/bptransfer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/blobporter/ac23782d56d2dc8aa8bc20303415255dea7ddeb0/docs/bptransfer.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. BlobPorter documentation master file, created by 2 | sphinx-quickstart on Mon Feb 12 13:52:41 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | BlobPorter 7 | ====================================== 8 | 9 | BlobPorter is a data transfer tool for Azure Blob Storage that maximizes throughput through concurrent reads and writes that can scale up and down independently. 10 | 11 | .. image:: bptransfer.png 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | :caption: Contents: 16 | 17 | gettingstarted 18 | examples 19 | resumabletrans 20 | perfmode 21 | 22 | 23 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=BlobPorter 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/perfmode.rst: -------------------------------------------------------------------------------- 1 | Performance Considerations 2 | ========================== 3 | 4 | Best Practices 5 | -------------- 6 | 7 | 8 | - By default, BlobPorter creates 5 readers and 8 workers for each core on the computer. You can overwrite these values by using the options -r (number of readers) and -g (number of workers). When overriding these options there are few considerations: 9 | 10 | - If during the transfer the buffer level is constant at 000%, workers could be waiting for data. Consider increasing the number of readers. If the level is 100% the opposite applies; increasing the number of workers could help. 11 | 12 | - Each reader or worker correlates to one goroutine. Goroutines are lightweight and a Go program can create a high number of goroutines, however, there's a point where the overhead of context switching impacts overall performance. Increase these values in small increments, e.g. 5. 13 | 14 | - For transfers from fast disks (SSD) or HTTP sources reducing the number readers or workers could provide better performance than the default values. Reduce these values if you want to minimize resource utilization. Lowering these numbers reduces contention and the likelihood of experiencing throttling conditions. 15 | 16 | - Transfers can be batched. Each batch transfer will concurrently read and transfer up to 500 files (default value) from the source. The batch size can be modified using the -x option. 17 | 18 | - Blobs smaller than the block size are transferred in a single operation. With relatively small files (<32MB) performance may be better if you set a block size equal to the size of the files. Setting the number of workers and readers to the number of files could yield performance gains. 19 | 20 | - The block size can have a significant memory impact if set to a large value (e.g. 100MB). For large files, use a block size that is close to the minimum required for the transfer and reduce the number of workers (g option). 21 | 22 | The following table list the maximum file size for typical block sizes. 23 | 24 | =============== =================== 25 | Block Size (MB) Max File Size (GB) 26 | =============== =================== 27 | 8 400 28 | 16 800 29 | 32 1600 30 | 64 3200 31 | =============== =================== 32 | 33 | Performance Measurement Mode 34 | ---------------------------- 35 | 36 | BlobPorter has a performance measurement mode that uploads random data generated in memory and measures the performance of the operation without the impact of disk i/o. 37 | The performance mode for uploads could help you identify the potential upper limit of the data throughput that your environment can provide. 38 | 39 | For example, the following command will upload 10 x 10GB files to a storage account. 40 | 41 | :: 42 | 43 | blobporter -f "1GB:10" -c perft -t perf-blockblob 44 | 45 | You can also use this mode to see if increasing (or decreasing) the number of workers/writers (-g option) will have a potential impact. 46 | 47 | :: 48 | 49 | blobporter -f "1GB:10" -c perft -t perf-blockblob -g 20 50 | 51 | Similarly, for downloads, you can simulate downloading data from a storage account without writing to disk. This mode could also help you fine-tune the number of readers (-r option) and get an idea of the maximum download throughput. 52 | 53 | The following command downloads the data previously uploaded. 54 | 55 | :: 56 | 57 | export SRC_ACCOUNT_KEY=$ACCOUNT_KEY 58 | 59 | .. 60 | 61 | :: 62 | 63 | blobporter -f "https://myaccount.blob.core.windows.net/perft" -t blob-perf`` 64 | 65 | 66 | Then you can download the file to disk. 67 | 68 | :: 69 | 70 | blobporter -c perft -t blob-file 71 | 72 | The performance difference will provide with a base-line of the impact of disk i/o. 73 | -------------------------------------------------------------------------------- /docs/resumabletrans.rst: -------------------------------------------------------------------------------- 1 | Resumable Transfers 2 | =================== 3 | 4 | BlobPorter supports resumable transfers. This feature is enabled when the -l option is set with the path where the transfer status file will be created. 5 | In case of failure, if the same status file is specified, BlobPorter will skip files that were already transferred. 6 | 7 | :: 8 | 9 | blobporter -f "manyfiles/*" -c many -l mylog 10 | 11 | For each file in the transfer two entries will be created in the status file. One when file is queued (Started) and another when the file is successfully transferred (Completed). 12 | 13 | The log entries are created with the following tab-delimited format: 14 | 15 | :: 16 | 17 | [Timestamp] [Filename] [Status (1:Started,2:Completed,3:Ignored)] [Size] [Transfer ID ] 18 | 19 | 20 | The following output from a transfer status file shows that three files were included in the transfer: **file10** , **file11** and **file15** . 21 | However, only **file10** and **file11** were successfully transferred. For **file15** the output indicates that it was queued but there's no second entry confirming that it was transferred successfully (status = 2). :: 22 | 23 | 2018-03-05T03:31:13.034245807Z file10 1 104857600 938520246_mylog 24 | 2018-03-05T03:31:13.034390509Z file11 1 104857600 938520246_mylog 25 | 2018-03-05T03:31:13.034437109Z file15 1 104857600 938520246_mylog 26 | 2018-03-05T03:31:25.232572306Z file10 2 104857600 938520246_mylog 27 | 2018-03-05T03:31:25.591239355Z file11 2 104857600 938520246_mylog 28 | 29 | Consider the previous scenario and assume that the transfer was executed again. 30 | In this case, the status file shows two new entries for **file15** in a new transfer (the transfer ID is different) which is an indication that this time the file was transferred successfully. :: 31 | 32 | 2018-03-05T03:31:13.034245807Z file10 1 104857600 938520246_mylog 33 | 2018-03-05T03:31:13.034390509Z file11 1 104857600 938520246_mylog 34 | 2018-03-05T03:31:13.034437109Z file15 1 104857600 938520246_mylog 35 | 2018-03-05T03:31:25.232572306Z file10 2 104857600 938520246_mylog 36 | 2018-03-05T03:31:25.591239355Z file11 2 104857600 938520246_mylog 37 | 2018-03-05T03:54:33.660161772Z file15 1 104857600 495675852_mylog 38 | 2018-03-05T03:54:34.579295059Z file15 2 104857600 495675852_mylog 39 | 40 | Finally, since the process completed successfully, a summary is appended to the transfer status file. :: 41 | 42 | ---------------------------------------------------------- 43 | Transfer Completed---------------------------------------- 44 | Start Summary--------------------------------------------- 45 | Last Transfer ID:495675852_mylog 46 | Date:Mon Mar 5 03:54:34 UTC 2018 47 | File:file15 Size:104857600 TID:495675852_mylog 48 | File:file10 Size:104857600 TID:938520246_mylog 49 | File:file11 Size:104857600 TID:938520246_mylog 50 | Transferred Files:3 Total Size:314572800 51 | End Summary----------------------------------------------- 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /files.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | for n in {51..200}; do 3 | dd if=/dev/urandom of=file$( printf %03d "$n" ).bin bs=1 count=$(( 50000 * 1024 )) & 4 | done 5 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Azure/blobporter 2 | 3 | require ( 4 | github.com/Azure/azure-pipeline-go v0.1.8 5 | github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804 6 | github.com/davecgh/go-spew v1.1.0 7 | github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e 8 | github.com/go-ini/ini v1.37.0 9 | github.com/minio/minio-go v0.0.0-20180307105924-9e124ec59547 10 | github.com/mitchellh/go-homedir v0.0.0-20180523094522-3864e76763d9 11 | github.com/pmezard/go-difflib v1.0.0 12 | github.com/sirupsen/logrus v1.0.5 13 | github.com/stretchr/testify v1.2.2 14 | golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8 15 | golang.org/x/net v0.0.0-20180124060956-0ed95abb35c4 16 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 17 | golang.org/x/sys v0.0.0-20180709060233-1b2967e3c290 18 | golang.org/x/text v0.3.0 19 | ) 20 | -------------------------------------------------------------------------------- /img/bptransfer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/blobporter/ac23782d56d2dc8aa8bc20303415255dea7ddeb0/img/bptransfer.png -------------------------------------------------------------------------------- /internal/const.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | ) 7 | 8 | //ProgramVersion blobporter version 9 | const ProgramVersion = "0.6.20" 10 | 11 | //HTTPClientTimeout HTTP client timeout when reading from HTTP sources and try timeout for blob storage operations. 12 | var HTTPClientTimeout = 90 13 | 14 | //UserAgentStr value included in the user-agent header when calling the blob API 15 | var UserAgentStr = fmt.Sprintf("%s/%s/%s", "BlobPorter", ProgramVersion, runtime.GOARCH) 16 | -------------------------------------------------------------------------------- /internal/eventsink_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | type mockDel struct { 10 | numOfEventItem int 11 | numOfCalls int 12 | aggregateResult int 13 | } 14 | 15 | func (m *mockDel) delegate(e EventItem, a EventItemAggregate) { 16 | m.numOfEventItem++ 17 | 18 | m.numOfCalls++ 19 | 20 | if e.Action == Sum { 21 | m.aggregateResult = int(a.Value) 22 | } 23 | } 24 | 25 | const testSource EventSource = 10 26 | const testEvent EventName = "test-event" 27 | const oneFloat64 float64 = 1 28 | 29 | func TestRealTimeEvents(t *testing.T) { 30 | es := newEventSink() 31 | md := mockDel{} 32 | 33 | es.AddSubscription(testSource, RealTime, md.delegate) 34 | es.AddEvent(testSource, testEvent, "", EventData{Value: 1}) 35 | es.AddEvent(testSource, testEvent, "", EventData{Value: 1}) 36 | es.AddEvent(testSource, testEvent, "", EventData{Value: 1}) 37 | 38 | es.FlushAndWait() 39 | 40 | assert.Equal(t, md.numOfEventItem, 3, "Values must match") 41 | assert.Equal(t, md.numOfCalls, 3, "Values must match") 42 | } 43 | 44 | func TestOnDoneEvents(t *testing.T) { 45 | es := newEventSink() 46 | md := mockDel{} 47 | 48 | es.AddSubscription(testSource, OnDone, md.delegate) 49 | es.AddSumEvent(testSource, testEvent, "", oneFloat64) 50 | es.AddSumEvent(testSource, testEvent, "", oneFloat64) 51 | es.AddSumEvent(testSource, testEvent, "", oneFloat64) 52 | 53 | assert.Equal(t, md.numOfCalls, 0, "Values must match") 54 | 55 | es.FlushAndWait() 56 | 57 | assert.Equal(t, md.aggregateResult, 3, "Values must match") 58 | assert.Equal(t, md.numOfCalls, 1, "Values must match") 59 | } 60 | 61 | func TestFlushCycle(t *testing.T) { 62 | es := newEventSink() 63 | md := mockDel{} 64 | 65 | es.AddSubscription(testSource, RealTime, md.delegate) 66 | es.AddEvent(testSource, testEvent, "", EventData{Value: 1}) 67 | 68 | es.Reset() 69 | 70 | assert.Equal(t, 1, md.numOfEventItem, "Values must match") 71 | assert.Equal(t, 1, md.numOfCalls, "Values must match") 72 | 73 | es.AddSubscription(testSource, RealTime, md.delegate) 74 | es.AddEvent(testSource, testEvent, "", EventData{Value: 1}) 75 | 76 | es.Reset() 77 | 78 | assert.Equal(t, 2, md.numOfEventItem, "Values must match") 79 | assert.Equal(t, 2, md.numOfCalls, "Values must match") 80 | 81 | //add a sum event 82 | es.AddSubscription(testSource, OnDone, md.delegate) 83 | es.AddSumEvent(testSource, testEvent, "", oneFloat64) 84 | 85 | es.Reset() 86 | 87 | assert.Equal(t, 1, md.aggregateResult, "Values must match") 88 | assert.Equal(t, 3, md.numOfCalls, "Values must match") 89 | } 90 | -------------------------------------------------------------------------------- /internal/tracker_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | //IMPORTANT: This test will create local files in internal_testdata/ 11 | //make sure *_testdata is in gitinore. 12 | const workdir = "internal_testdata" 13 | const logfile = workdir + "/my_log" 14 | 15 | func TestBasicTracking(t *testing.T) { 16 | os.Mkdir(workdir, 466) 17 | os.Remove(logfile) 18 | 19 | tracker, err := NewTransferTracker(logfile) 20 | 21 | assert.NoError(t, err, "unexpected error") 22 | 23 | istrans, err := tracker.IsTransferredAndTrackIfNot("file1", 10) 24 | assert.NoError(t, err, "unexpected error") 25 | assert.False(t, istrans, "must be fales, i.e. not in the log") 26 | 27 | err = tracker.TrackFileTransferComplete("file1") 28 | assert.NoError(t, err, "unexpected error") 29 | 30 | err = tracker.TrackTransferComplete() 31 | assert.NoError(t, err, "unexpected error") 32 | os.Remove(logfile) 33 | 34 | } 35 | 36 | func TestBasicTrackingAndResume(t *testing.T) { 37 | os.Mkdir(workdir, 0666) 38 | os.Remove(logfile) 39 | 40 | tracker, err := NewTransferTracker(logfile) 41 | 42 | assert.NoError(t, err, "unexpected error") 43 | 44 | istrans, err := tracker.IsTransferredAndTrackIfNot("file1", 10) 45 | assert.NoError(t, err, "unexpected error") 46 | assert.False(t, istrans, "must be false, i.e. not in the log") 47 | 48 | istrans, err = tracker.IsTransferredAndTrackIfNot("file2", 10) 49 | assert.NoError(t, err, "unexpected error") 50 | assert.False(t, istrans, "must be false, i.e. not in the log") 51 | 52 | //only one succeeds. 53 | err = tracker.TrackFileTransferComplete("file1") 54 | assert.NoError(t, err, "unexpected error") 55 | 56 | //closing the handle manually as the program will continue (handle to the file will exists) and need to simulate a crash 57 | err = tracker.loghandle.Close() 58 | assert.NoError(t, err, "unexpected error") 59 | 60 | //create another instance of a tracker at this point only one file should be transferred. 61 | tracker, err = NewTransferTracker(logfile) 62 | assert.NoError(t, err, "unexpected error") 63 | 64 | istrans, err = tracker.IsTransferredAndTrackIfNot("file1", 10) 65 | assert.NoError(t, err, "unexpected error") 66 | assert.True(t, istrans, "must be true, as this file was signaled as succesfully transferred") 67 | 68 | istrans, err = tracker.IsTransferredAndTrackIfNot("file2", 10) 69 | assert.NoError(t, err, "unexpected error") 70 | assert.False(t, istrans, "must be false, this was was not signaled as complete") 71 | 72 | err = tracker.TrackFileTransferComplete("file2") 73 | assert.NoError(t, err, "unexpected error") 74 | 75 | err = tracker.TrackTransferComplete() 76 | assert.NoError(t, err, "unexpected error") 77 | os.Remove(logfile) 78 | } 79 | -------------------------------------------------------------------------------- /inttest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | REF_FILE="tempfile" 3 | UP_REF_FILE="dfile" 4 | CONT=$1 5 | FILE_SIZE=5242880 #5MiB 6 | EXIT_CODE=0 7 | 8 | #create a temp file 9 | dd if=/dev/urandom of=$REF_FILE bs=64M count=1 iflag=fullblock 10 | 11 | 12 | #Set permissions 13 | sudo chmod +x ./linux_amd64/blobporter 14 | 15 | #Upload file 16 | ./linux_amd64/blobporter -f $REF_FILE -n $UP_REF_FILE -c $CONT 17 | 18 | #Download file 19 | ./linux_amd64/blobporter -c $CONT -n $UP_REF_FILE -t blob-file 20 | 21 | #Compare MD5's 22 | REF_MD5="$(md5sum $REF_FILE | awk '{print $1}')" 23 | VAL_MD5="$(md5sum $UP_REF_FILE | awk '{print $1}')" 24 | 25 | if [ "$REF_MD5" = "$VAL_MD5" ] ; then 26 | echo "Success!" 27 | else 28 | echo $REF_MD5 29 | echo $VAL_MD5 30 | echo "Failure!" 31 | $EXIT_CODE=1 32 | fi 33 | #clean up 34 | rm $REF_FILE 35 | rm $UP_REF_FILE 36 | 37 | exit $EXIT_CODE -------------------------------------------------------------------------------- /sources/azblobinfo.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "log" 5 | "path" 6 | "time" 7 | 8 | "github.com/Azure/azure-storage-blob-go/azblob" 9 | "github.com/Azure/blobporter/internal" 10 | "github.com/Azure/blobporter/pipeline" 11 | ) 12 | 13 | //AzureBlobParams parameters for the creation of Azure Blob source pipeline 14 | type AzureBlobParams struct { 15 | SourceParams 16 | Container string 17 | BlobNames []string 18 | AccountName string 19 | AccountKey string 20 | SasExp int 21 | BaseBlobURL string 22 | } 23 | 24 | const defaultSasExpHours = 2 25 | 26 | type azBlobInfoProvider struct { 27 | params *AzureBlobParams 28 | azUtil *internal.AzUtil 29 | } 30 | 31 | func newazBlobInfoProvider(params *AzureBlobParams) *azBlobInfoProvider { 32 | azutil, err := internal.NewAzUtil(params.AccountName, params.AccountKey, params.Container, params.BaseBlobURL) 33 | 34 | if err != nil { 35 | log.Fatal(err) 36 | } 37 | 38 | return &azBlobInfoProvider{params: params, azUtil: azutil} 39 | } 40 | 41 | func (b *azBlobInfoProvider) toSourceInfo(obj *azblob.BlobItem) (*pipeline.SourceInfo, error) { 42 | exp := b.params.SasExp 43 | if exp == 0 { 44 | exp = defaultSasExpHours 45 | } 46 | date := time.Now().Add(time.Duration(exp) * time.Minute).UTC() 47 | 48 | sourceURLWithSAS, err := b.azUtil.GetBlobURLWithReadOnlySASToken(obj.Name, date) 49 | 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | targetAlias := obj.Name 55 | if !b.params.KeepDirStructure { 56 | targetAlias = path.Base(obj.Name) 57 | } 58 | 59 | return &pipeline.SourceInfo{ 60 | SourceName: sourceURLWithSAS.String(), 61 | Size: uint64(*obj.Properties.ContentLength), 62 | TargetAlias: targetAlias}, nil 63 | } 64 | 65 | func (b *azBlobInfoProvider) listObjects(filter internal.SourceFilter) <-chan ObjectListingResult { 66 | sources := make(chan ObjectListingResult, 2) 67 | list := make([]pipeline.SourceInfo, 0) 68 | bsize := 0 69 | 70 | blobCallback := func(blob *azblob.BlobItem, prefix string) (bool, error) { 71 | include := true 72 | if b.params.UseExactNameMatch { 73 | include = blob.Name == prefix 74 | } 75 | 76 | transferred, err := filter.IsTransferredAndTrackIfNot(blob.Name, int64(*blob.Properties.ContentLength)) 77 | 78 | if err != nil { 79 | return true, err 80 | } 81 | 82 | if include && !transferred { 83 | 84 | si, err := b.toSourceInfo(blob) 85 | 86 | if err != nil { 87 | return true, err 88 | } 89 | 90 | list = append(list, *si) 91 | 92 | if bsize++; bsize == b.params.FilesPerPipeline { 93 | sources <- ObjectListingResult{Sources: list} 94 | list = make([]pipeline.SourceInfo, 0) 95 | bsize = 0 96 | } 97 | 98 | if b.params.UseExactNameMatch { 99 | //true, stops iteration 100 | return true, nil 101 | } 102 | } 103 | 104 | //false, continues the iteration 105 | return false, nil 106 | } 107 | 108 | go func() { 109 | for _, blobName := range b.params.BlobNames { 110 | if err := b.azUtil.IterateBlobList(blobName, blobCallback); err != nil { 111 | sources <- ObjectListingResult{Err: err} 112 | return 113 | } 114 | if bsize > 0 { 115 | sources <- ObjectListingResult{Sources: list} 116 | list = make([]pipeline.SourceInfo, 0) 117 | bsize = 0 118 | } 119 | } 120 | close(sources) 121 | 122 | }() 123 | 124 | return sources 125 | } 126 | -------------------------------------------------------------------------------- /sources/s3info.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "log" 5 | "net/url" 6 | "path" 7 | "strings" 8 | "time" 9 | 10 | "github.com/Azure/blobporter/internal" 11 | "github.com/Azure/blobporter/pipeline" 12 | "github.com/minio/minio-go" 13 | ) 14 | 15 | //S3Params parameters used to create a new instance of a S3 source pipeline 16 | type S3Params struct { 17 | SourceParams 18 | Bucket string 19 | Endpoint string 20 | Prefixes []string 21 | PreSignedExpMin int 22 | AccessKey string 23 | SecretKey string 24 | } 25 | 26 | const defaultExpHours = 4 27 | 28 | type s3InfoProvider struct { 29 | params *S3Params 30 | s3client *minio.Client 31 | } 32 | 33 | func newS3InfoProvider(params *S3Params) (*s3InfoProvider, error) { 34 | s3client, err := minio.New(params.Endpoint, params.AccessKey, params.SecretKey, true) 35 | 36 | 37 | if err != nil { 38 | log.Fatalln(err) 39 | } 40 | 41 | return &s3InfoProvider{params: params, s3client: s3client}, nil 42 | 43 | } 44 | 45 | func (s *s3InfoProvider) toSourceInfo(obj *minio.ObjectInfo) (*pipeline.SourceInfo, error) { 46 | exp := time.Duration(s.params.PreSignedExpMin) * time.Minute 47 | 48 | u, err := s.s3client.PresignedGetObject(s.params.Bucket, obj.Key, exp, url.Values{}) 49 | 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | targetAlias := obj.Key 55 | if !s.params.KeepDirStructure { 56 | targetAlias = path.Base(obj.Key) 57 | } 58 | 59 | return &pipeline.SourceInfo{ 60 | SourceName: u.String(), 61 | Size: uint64(obj.Size), 62 | TargetAlias: targetAlias}, nil 63 | 64 | } 65 | 66 | func (s *s3InfoProvider) listObjects(filter internal.SourceFilter) <-chan ObjectListingResult { 67 | sources := make(chan ObjectListingResult, 2) 68 | go func() { 69 | list := make([]pipeline.SourceInfo, 0) 70 | bsize := 0 71 | 72 | for _, prefix := range s.params.Prefixes { 73 | // Create a done channel to control 'ListObjects' go routine. 74 | done := make(chan struct{}) 75 | defer close(done) 76 | for object := range s.s3client.ListObjects(s.params.Bucket, prefix, true, done) { 77 | if object.Err != nil { 78 | sources <- ObjectListingResult{Err: object.Err} 79 | return 80 | } 81 | include := true 82 | 83 | if s.params.SourceParams.UseExactNameMatch { 84 | include = object.Key == prefix 85 | } 86 | 87 | isfolder := strings.HasSuffix(object.Key, "/") && object.Size == 0 88 | 89 | transferred, err := filter.IsTransferredAndTrackIfNot(object.Key, object.Size) 90 | 91 | if err != nil { 92 | sources <- ObjectListingResult{Err: err} 93 | return 94 | } 95 | 96 | if include && !isfolder && !transferred { 97 | si, err := s.toSourceInfo(&object) 98 | 99 | if err != nil { 100 | sources <- ObjectListingResult{Err: err} 101 | return 102 | } 103 | list = append(list, *si) 104 | 105 | if bsize++; bsize == s.params.FilesPerPipeline { 106 | sources <- ObjectListingResult{Sources: list} 107 | list = make([]pipeline.SourceInfo, 0) 108 | bsize = 0 109 | } 110 | } 111 | 112 | } 113 | 114 | if bsize > 0 { 115 | sources <- ObjectListingResult{Sources: list} 116 | list = make([]pipeline.SourceInfo, 0) 117 | bsize = 0 118 | } 119 | } 120 | close(sources) 121 | }() 122 | 123 | return sources 124 | } 125 | -------------------------------------------------------------------------------- /sources/types.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import "github.com/Azure/blobporter/internal" 4 | 5 | type objectListProvider interface { 6 | listObjects(filter internal.SourceFilter) <-chan ObjectListingResult 7 | } 8 | 9 | //SourceParams input base parameters for blob and S3 based pipelines 10 | type SourceParams struct { 11 | ReferenceMode bool 12 | CalculateMD5 bool 13 | UseExactNameMatch bool 14 | KeepDirStructure bool 15 | FilesPerPipeline int 16 | Tracker *internal.TransferTracker 17 | } 18 | 19 | //HTTPSourceParams input parameters for HTTP pipelines 20 | type HTTPSourceParams struct { 21 | SourceParams 22 | SourceURIs []string 23 | TargetAliases []string 24 | } 25 | -------------------------------------------------------------------------------- /targets/azurepage.go: -------------------------------------------------------------------------------- 1 | package targets 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "log" 7 | "time" 8 | 9 | "github.com/Azure/blobporter/internal" 10 | "github.com/Azure/blobporter/pipeline" 11 | "github.com/Azure/blobporter/util" 12 | ) 13 | 14 | //////////////////////////////////////////////////////////// 15 | ///// AzurePage Target 16 | //////////////////////////////////////////////////////////// 17 | 18 | //AzurePageTarget represents an Azure Block target 19 | type AzurePageTarget struct { 20 | azUtil *internal.AzUtil 21 | } 22 | 23 | //NewAzurePageTargetPipeline creates a new Azure Block target 24 | func NewAzurePageTargetPipeline(params AzureTargetParams) pipeline.TargetPipeline { 25 | az, err := internal.NewAzUtil(params.AccountName, params.AccountKey, params.Container, params.BaseBlobURL) 26 | 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | 31 | var notfound bool 32 | notfound, err = az.CreateContainerIfNotExists() 33 | 34 | if notfound { 35 | fmt.Printf("Info! Container was not found, creating it...\n") 36 | } 37 | 38 | if err != nil { 39 | log.Fatal(err) 40 | } 41 | return &AzurePageTarget{azUtil: az} 42 | } 43 | 44 | //Page blobs limits and units 45 | 46 | //PageSize page size for page blobs 47 | const PageSize uint64 = 512 48 | const maxPageSize uint64 = 4 * util.MB 49 | const maxPageBlobSize uint64 = 8 * util.TB 50 | 51 | //PreProcessSourceInfo implementation of PreProcessSourceInfo from the pipeline.TargetPipeline interface. 52 | //initializes the page blob. 53 | func (t *AzurePageTarget) PreProcessSourceInfo(source *pipeline.SourceInfo, blockSize uint64) (err error) { 54 | size := source.Size 55 | 56 | if size%PageSize != 0 { 57 | return fmt.Errorf(" invalid size for a page blob. The size of the file %v (%v) is not a multiple of %v ", source.SourceName, source.Size, PageSize) 58 | } 59 | 60 | if size > maxPageBlobSize { 61 | return fmt.Errorf(" the file %v is too big (%v). Tha maximum size of a page blob is %v ", source.SourceName, source.Size, maxPageBlobSize) 62 | } 63 | 64 | if blockSize > maxPageSize || blockSize < PageSize { 65 | return fmt.Errorf(" invalid block size for page blob: %v. The value must be greater than %v and less than %v", source.SourceName, PageSize, maxPageSize) 66 | } 67 | 68 | err = t.azUtil.CreatePageBlob(source.TargetAlias, size) 69 | return 70 | } 71 | 72 | //CommitList implements CommitList from the pipeline.TargetPipeline interface. 73 | //Passthrough no need to a commit for page blob. 74 | func (t *AzurePageTarget) CommitList(listInfo *pipeline.TargetCommittedListInfo, NumberOfBlocks int, targetName string) (msg string, err error) { 75 | msg = "Page blob committed" 76 | err = nil 77 | return 78 | } 79 | 80 | //ProcessWrittenPart implements ProcessWrittenPart from the pipeline.TargetPipeline interface. 81 | //Passthrough no need to process a written part when transferring to a page blob. 82 | func (t *AzurePageTarget) ProcessWrittenPart(result *pipeline.WorkerResult, listInfo *pipeline.TargetCommittedListInfo) (requeue bool, err error) { 83 | requeue = false 84 | err = nil 85 | return 86 | } 87 | 88 | //WritePart implements WritePart from the pipeline.TargetPipeline interface. 89 | //Performs a PUT page operation with the data contained in the part. 90 | //This assumes the part.BytesToRead is a multiple of the PageSize 91 | func (t *AzurePageTarget) WritePart(part *pipeline.Part) (duration time.Duration, startTime time.Time, numOfRetries int, err error) { 92 | 93 | start := int64(part.Offset) 94 | end := int64(part.Offset + uint64(part.BytesToRead) - 1) 95 | defer util.PrintfIfDebug("WritePart -> start:%v end:%v name:%v err:%v", start, end, part.TargetAlias, err) 96 | 97 | var md5 []byte 98 | if part.IsMD5Computed() { 99 | md5 = part.MD5Bytes() 100 | } 101 | err = t.azUtil.PutPages(part.TargetAlias, start, end, bytes.NewReader(part.Data), md5) 102 | 103 | return 104 | } 105 | -------------------------------------------------------------------------------- /targets/multifile.go: -------------------------------------------------------------------------------- 1 | package targets 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "time" 8 | 9 | "github.com/Azure/blobporter/internal" 10 | "github.com/Azure/blobporter/util" 11 | 12 | "github.com/Azure/blobporter/pipeline" 13 | ) 14 | 15 | //////////////////////////////////////////////////////////// 16 | ///// FileSystemTarget 17 | //////////////////////////////////////////////////////////// 18 | 19 | //FileSystemTarget represents an OS file(s) target 20 | type FileSystemTarget struct { 21 | fileHandlesMan *internal.FileHandlePool 22 | } 23 | 24 | //NewFileSystemTargetPipeline creates a new multi file target and 'n' number of handles for concurrent writes to a file. 25 | func NewFileSystemTargetPipeline(overwrite bool, numberOfHandles int) pipeline.TargetPipeline { 26 | 27 | fhm := internal.NewFileHandlePool(numberOfHandles, internal.Write, overwrite) 28 | 29 | return &FileSystemTarget{ 30 | fileHandlesMan: fhm} 31 | } 32 | 33 | //PreProcessSourceInfo implementation of PreProcessSourceInfo from the pipeline.TargetPipeline interface. 34 | //Passthrough no need to pre-process for a file target. 35 | func (t *FileSystemTarget) PreProcessSourceInfo(source *pipeline.SourceInfo, blockSize uint64) (err error) { 36 | return nil 37 | } 38 | 39 | //CommitList implements CommitList from the pipeline.TargetPipeline interface. 40 | //For a file download a final commit is not required and this implementation closes all the filehandles. 41 | func (t *FileSystemTarget) CommitList(listInfo *pipeline.TargetCommittedListInfo, numberOfBlocks int, targetName string) (msg string, err error) { 42 | msg = fmt.Sprintf("\rFile Saved:%v, Parts: %d", 43 | targetName, numberOfBlocks) 44 | err = t.fileHandlesMan.CloseHandles(targetName) 45 | return 46 | } 47 | 48 | //ProcessWrittenPart implements ProcessWrittenPart from the pipeline.TargetPipeline interface. 49 | //Passthrough implementation as no post-written-processing is required (e.g. maintain a list) when files are downloaded. 50 | func (t *FileSystemTarget) ProcessWrittenPart(result *pipeline.WorkerResult, listInfo *pipeline.TargetCommittedListInfo) (requeue bool, err error) { 51 | return false, nil 52 | } 53 | 54 | func (t *FileSystemTarget) loadHandle(part *pipeline.Part) (*os.File, error) { 55 | s := time.Now() 56 | defer util.PrintfIfDebug("loadHandle-> name:%v duration:%v", part.TargetAlias, time.Now().Sub(s)) 57 | return t.fileHandlesMan.GetHandle(part.TargetAlias) 58 | } 59 | 60 | func (t *FileSystemTarget) closeOrKeepHandle(part *pipeline.Part, fh *os.File) error { 61 | s := time.Now() 62 | defer util.PrintfIfDebug("closeOrKeepHandle-> name:%v duration:%v", part.TargetAlias, time.Now().Sub(s)) 63 | return t.fileHandlesMan.ReturnHandle(part.TargetAlias, fh) 64 | } 65 | 66 | //WritePart implements WritePart from the pipeline.TargetPipeline interface. 67 | //Writes to a local file using a filehandle received from a channel. 68 | func (t *FileSystemTarget) WritePart(part *pipeline.Part) (duration time.Duration, startTime time.Time, numOfRetries int, err error) { 69 | var fh *os.File 70 | 71 | if fh, err = t.loadHandle(part); err != nil { 72 | log.Fatal(fmt.Errorf("Failed to load the handle: %v", err)) 73 | } 74 | startTime = time.Now() 75 | 76 | if _, err = fh.WriteAt((*part).Data, int64((*part).Offset)); err != nil { 77 | log.Fatal(fmt.Errorf("Failed to write data: %v", err)) 78 | } 79 | 80 | duration = time.Now().Sub(startTime) 81 | 82 | if err = t.closeOrKeepHandle(part, fh); err != nil { 83 | log.Fatal(fmt.Errorf("Failed to close or keep the handle: %v", err)) 84 | } 85 | 86 | return 87 | } 88 | -------------------------------------------------------------------------------- /targets/params.go: -------------------------------------------------------------------------------- 1 | package targets 2 | 3 | //FileTargetParams TODO 4 | type FileTargetParams struct { 5 | Overwrite bool 6 | NumberOfHandlesPerFile int 7 | } 8 | 9 | //AzureTargetParams TODO 10 | type AzureTargetParams struct { 11 | AccountName string 12 | AccountKey string 13 | Container string 14 | BaseBlobURL string 15 | UseServerSide bool 16 | } 17 | -------------------------------------------------------------------------------- /targets/perftarget.go: -------------------------------------------------------------------------------- 1 | package targets 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/Azure/blobporter/pipeline" 7 | ) 8 | 9 | //////////////////////////////////////////////////////////// 10 | ///// Perf Target 11 | //////////////////////////////////////////////////////////// 12 | 13 | //PerfTarget TODO 14 | type PerfTarget struct { 15 | } 16 | 17 | //NewPerfTargetPipeline creates a new passthrough target used to measure the performance of the source 18 | func NewPerfTargetPipeline() pipeline.TargetPipeline { 19 | return &PerfTarget{} 20 | } 21 | 22 | //PreProcessSourceInfo passthrough, no processing. 23 | func (t *PerfTarget) PreProcessSourceInfo(source *pipeline.SourceInfo, blockSize uint64) (err error) { 24 | return nil 25 | } 26 | 27 | //CommitList passthrough, no processing. 28 | func (t *PerfTarget) CommitList(listInfo *pipeline.TargetCommittedListInfo, NumberOfBlocks int, targetName string) (msg string, err error) { 29 | msg = "Perf test committed" 30 | err = nil 31 | return 32 | } 33 | 34 | //ProcessWrittenPart passthrough, no processing. 35 | func (t *PerfTarget) ProcessWrittenPart(result *pipeline.WorkerResult, listInfo *pipeline.TargetCommittedListInfo) (requeue bool, err error) { 36 | requeue = false 37 | err = nil 38 | return 39 | } 40 | 41 | //WritePart passthrough, no processing. 42 | func (t *PerfTarget) WritePart(part *pipeline.Part) (duration time.Duration, startTime time.Time, numOfRetries int, err error) { 43 | 44 | s := time.Now() 45 | numOfRetries = 0 46 | err = nil 47 | startTime = s 48 | duration = time.Now().Sub(s) 49 | 50 | return 51 | } 52 | -------------------------------------------------------------------------------- /transfer/commit.go: -------------------------------------------------------------------------------- 1 | package transfer 2 | 3 | import ( 4 | "log" 5 | "sync" 6 | "time" 7 | 8 | "github.com/Azure/blobporter/internal" 9 | "github.com/Azure/blobporter/pipeline" 10 | ) 11 | 12 | type commitListHandler struct { 13 | commitInfos map[string]commitInfo 14 | tracker *internal.TransferTracker 15 | target pipeline.TargetPipeline 16 | resultQ chan pipeline.WorkerResult 17 | commitInfoReq chan commitInfo 18 | wg *sync.WaitGroup 19 | done chan bool 20 | } 21 | 22 | type commitInfo struct { 23 | list *pipeline.TargetCommittedListInfo 24 | numOfBlocks int32 25 | targetName string 26 | } 27 | 28 | const ( 29 | //WriteDurationEvent TODO 30 | WriteDurationEvent internal.EventName = "write-duration" 31 | ) 32 | 33 | func newCommitListHandler(tracker *internal.TransferTracker, target pipeline.TargetPipeline) *commitListHandler { 34 | 35 | commitInfos := make(map[string]commitInfo) 36 | resultQ := make(chan pipeline.WorkerResult, 10000) 37 | commitInfoReq := make(chan commitInfo, 10000) 38 | c := commitListHandler{ 39 | commitInfos: commitInfos, 40 | tracker: tracker, 41 | target: target, 42 | resultQ: resultQ, 43 | commitInfoReq: commitInfoReq, 44 | wg: &sync.WaitGroup{}, 45 | } 46 | c.wg.Add(1) 47 | go func() { 48 | defer close(commitInfoReq) 49 | defer c.wg.Done() 50 | var requeue bool 51 | var err error 52 | for { 53 | 54 | result, ok := <-resultQ 55 | 56 | if !ok { 57 | return 58 | } 59 | 60 | cinfo := commitInfos[result.SourceURI] 61 | if cinfo.list == nil { 62 | cinfo.list = &pipeline.TargetCommittedListInfo{} 63 | } 64 | if requeue, err = target.ProcessWrittenPart(&result, cinfo.list); err != nil { 65 | log.Fatal(err) 66 | } 67 | 68 | if requeue { 69 | resultQ <- result 70 | continue 71 | } 72 | 73 | cinfo.numOfBlocks++ 74 | cinfo.targetName = result.TargetName 75 | commitInfos[result.SourceURI] = cinfo 76 | 77 | internal.EventSink.AddSumEvent(internal.CommitListHandler, WriteDurationEvent, result.TargetName, float64(result.Stats.Duration)) 78 | 79 | if cinfo.numOfBlocks == int32(result.NumberOfBlocks) { 80 | commitInfoReq <- cinfo 81 | } 82 | } 83 | 84 | }() 85 | return &c 86 | } 87 | func (c *commitListHandler) setDone() { 88 | close(c.resultQ) 89 | } 90 | func (c *commitListHandler) Wait() { 91 | c.wg.Wait() 92 | } 93 | func (c *commitListHandler) commitRequests() chan commitInfo { 94 | return c.commitInfoReq 95 | } 96 | 97 | func (c *commitListHandler) trackCommitted(targetName string) error { 98 | if c.tracker != nil { 99 | return c.tracker.TrackFileTransferComplete(targetName) 100 | } 101 | return nil 102 | } 103 | 104 | func (c *commitListHandler) addWorkerResult(workerID int, tb *pipeline.Part, startTime time.Time, d time.Duration, status string, retries int) pipeline.WorkerResult { 105 | 106 | var wStats = pipeline.WorkerResultStats{ 107 | StartTime: startTime, 108 | Duration: d, 109 | Retries: retries} 110 | 111 | //tb.Data = nil 112 | //tb.BufferQ = nil 113 | 114 | wr := pipeline.WorkerResult{ 115 | Stats: &wStats, 116 | BlockSize: int(tb.BytesToRead), 117 | Result: status, 118 | WorkerID: workerID, 119 | ItemID: tb.BlockID, 120 | DuplicateOfBlockOrdinal: tb.DuplicateOfBlockOrdinal, 121 | Ordinal: tb.Ordinal, 122 | Offset: tb.Offset, 123 | SourceURI: tb.SourceURI, 124 | NumberOfBlocks: tb.NumberOfBlocks, 125 | TargetName: tb.TargetAlias} 126 | 127 | c.resultQ <- wr 128 | 129 | return wr 130 | } 131 | -------------------------------------------------------------------------------- /util/util_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestPrintSizes(t *testing.T) { 11 | var size uint64 = 12 12 | psize := "12B (12)" 13 | 14 | //bytes scenario 15 | assert.Equal(t, psize, PrintSize(size)) 16 | 17 | //zero scenario 18 | size = 0 19 | psize = "0B (0)" 20 | assert.Equal(t, psize, PrintSize(size)) 21 | 22 | //KB scenario 23 | size = KB + 1 24 | psize = fmt.Sprintf("1KiB (%v)", KB+1) 25 | assert.Equal(t, psize, PrintSize(size)) 26 | 27 | //MB scenario 28 | size = MB + 1 29 | psize = fmt.Sprintf("1MiB (%v)", MB+1) 30 | assert.Equal(t, psize, PrintSize(size)) 31 | 32 | //GB scenario 33 | size = GB + 1 34 | psize = fmt.Sprintf("1GiB (%v)", GB+1) 35 | assert.Equal(t, psize, PrintSize(size)) 36 | 37 | //KB scenario (exact) 38 | size = KB 39 | psize = fmt.Sprintf("1KiB (%v)", KB) 40 | assert.Equal(t, psize, PrintSize(size)) 41 | 42 | //MB scenario (exact) 43 | size = MB 44 | psize = fmt.Sprintf("1MiB (%v)", MB) 45 | assert.Equal(t, psize, PrintSize(size)) 46 | 47 | //GB scenario (exact) 48 | size = GB 49 | psize = fmt.Sprintf("1GiB (%v)", GB) 50 | assert.Equal(t, psize, PrintSize(size)) 51 | } 52 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-pipeline-go/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-pipeline-go/pipeline/defaultlog_syslog.go: -------------------------------------------------------------------------------- 1 | // +build !windows,!nacl,!plan9 2 | 3 | package pipeline 4 | 5 | import ( 6 | "log" 7 | "log/syslog" 8 | ) 9 | 10 | // ForceLog should rarely be used. It forceable logs an entry to the 11 | // Windows Event Log (on Windows) or to the SysLog (on Linux) 12 | func ForceLog(level LogLevel, msg string) { 13 | if defaultLogger == nil { 14 | return // Return fast if we failed to create the logger. 15 | } 16 | // We are logging it, ensure trailing newline 17 | if len(msg) == 0 || msg[len(msg)-1] != '\n' { 18 | msg += "\n" // Ensure trailing newline 19 | } 20 | switch level { 21 | case LogFatal: 22 | defaultLogger.Fatal(msg) 23 | case LogPanic: 24 | defaultLogger.Panic(msg) 25 | case LogError, LogWarning, LogInfo: 26 | defaultLogger.Print(msg) 27 | } 28 | } 29 | 30 | var defaultLogger = func() *log.Logger { 31 | l, _ := syslog.NewLogger(syslog.LOG_USER|syslog.LOG_WARNING, log.LstdFlags) 32 | return l 33 | }() 34 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-pipeline-go/pipeline/defaultlog_windows.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "os" 5 | "syscall" 6 | "unsafe" 7 | ) 8 | 9 | // ForceLog should rarely be used. It forceable logs an entry to the 10 | // Windows Event Log (on Windows) or to the SysLog (on Linux) 11 | func ForceLog(level LogLevel, msg string) { 12 | var el eventType 13 | switch level { 14 | case LogError, LogFatal, LogPanic: 15 | el = elError 16 | case LogWarning: 17 | el = elWarning 18 | case LogInfo: 19 | el = elInfo 20 | } 21 | // We are logging it, ensure trailing newline 22 | if len(msg) == 0 || msg[len(msg)-1] != '\n' { 23 | msg += "\n" // Ensure trailing newline 24 | } 25 | reportEvent(el, 0, msg) 26 | } 27 | 28 | type eventType int16 29 | 30 | const ( 31 | elSuccess eventType = 0 32 | elError eventType = 1 33 | elWarning eventType = 2 34 | elInfo eventType = 4 35 | ) 36 | 37 | var reportEvent = func() func(eventType eventType, eventID int32, msg string) { 38 | advAPI32 := syscall.MustLoadDLL("AdvAPI32.dll") 39 | registerEventSource := advAPI32.MustFindProc("RegisterEventSourceW") 40 | 41 | sourceName, _ := os.Executable() 42 | sourceNameUTF16, _ := syscall.UTF16PtrFromString(sourceName) 43 | handle, _, lastErr := registerEventSource.Call(uintptr(0), uintptr(unsafe.Pointer(sourceNameUTF16))) 44 | if lastErr == nil { // On error, logging is a no-op 45 | return func(eventType eventType, eventID int32, msg string) {} 46 | } 47 | reportEvent := advAPI32.MustFindProc("ReportEventW") 48 | return func(eventType eventType, eventID int32, msg string) { 49 | s, _ := syscall.UTF16PtrFromString(msg) 50 | _, _, _ = reportEvent.Call( 51 | uintptr(handle), // HANDLE hEventLog 52 | uintptr(eventType), // WORD wType 53 | uintptr(0), // WORD wCategory 54 | uintptr(eventID), // DWORD dwEventID 55 | uintptr(0), // PSID lpUserSid 56 | uintptr(1), // WORD wNumStrings 57 | uintptr(0), // DWORD dwDataSize 58 | uintptr(unsafe.Pointer(&s)), // LPCTSTR *lpStrings 59 | uintptr(0)) // LPVOID lpRawData 60 | } 61 | }() 62 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-pipeline-go/pipeline/error.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | ) 7 | 8 | type causer interface { 9 | Cause() error 10 | } 11 | 12 | // ErrorNode can be an embedded field in a private error object. This field 13 | // adds Program Counter support and a 'cause' (reference to a preceding error). 14 | // When initializing a error type with this embedded field, initialize the 15 | // ErrorNode field by calling ErrorNode{}.Initialize(cause). 16 | type ErrorNode struct { 17 | pc uintptr // Represents a Program Counter that you can get symbols for. 18 | cause error // Refers to the preceding error (or nil) 19 | } 20 | 21 | // Error returns a string with the PC's symbols or "" if the PC is invalid. 22 | // When defining a new error type, have its Error method call this one passing 23 | // it the string representation of the error. 24 | func (e *ErrorNode) Error(msg string) string { 25 | s := "" 26 | if fn := runtime.FuncForPC(e.pc); fn != nil { 27 | file, line := fn.FileLine(e.pc) 28 | s = fmt.Sprintf("-> %v, %v:%v\n", fn.Name(), file, line) 29 | } 30 | s += msg + "\n\n" 31 | if e.cause != nil { 32 | s += e.cause.Error() + "\n" 33 | } 34 | return s 35 | } 36 | 37 | // Cause returns the error that preceded this error. 38 | func (e *ErrorNode) Cause() error { return e.cause } 39 | 40 | // Temporary returns true if the error occurred due to a temporary condition. 41 | func (e ErrorNode) Temporary() bool { 42 | type temporary interface { 43 | Temporary() bool 44 | } 45 | 46 | for err := e.cause; err != nil; { 47 | if t, ok := err.(temporary); ok { 48 | return t.Temporary() 49 | } 50 | 51 | if cause, ok := err.(causer); ok { 52 | err = cause.Cause() 53 | } else { 54 | err = nil 55 | } 56 | } 57 | return false 58 | } 59 | 60 | // Timeout returns true if the error occurred due to time expiring. 61 | func (e ErrorNode) Timeout() bool { 62 | type timeout interface { 63 | Timeout() bool 64 | } 65 | 66 | for err := e.cause; err != nil; { 67 | if t, ok := err.(timeout); ok { 68 | return t.Timeout() 69 | } 70 | 71 | if cause, ok := err.(causer); ok { 72 | err = cause.Cause() 73 | } else { 74 | err = nil 75 | } 76 | } 77 | return false 78 | } 79 | 80 | // Initialize is used to initialize an embedded ErrorNode field. 81 | // It captures the caller's program counter and saves the cause (preceding error). 82 | // To initialize the field, use "ErrorNode{}.Initialize(cause, 3)". A callersToSkip 83 | // value of 3 is very common; but, depending on your code nesting, you may need 84 | // a different value. 85 | func (ErrorNode) Initialize(cause error, callersToSkip int) ErrorNode { 86 | // Get the PC of Initialize method's caller. 87 | pc := [1]uintptr{} 88 | _ = runtime.Callers(callersToSkip, pc[:]) 89 | return ErrorNode{pc: pc[0], cause: cause} 90 | } 91 | 92 | // Cause walks all the preceding errors and return the originating error. 93 | func Cause(err error) error { 94 | for err != nil { 95 | cause, ok := err.(causer) 96 | if !ok { 97 | break 98 | } 99 | err = cause.Cause() 100 | } 101 | return err 102 | } 103 | 104 | // NewError creates a simple string error (like Error.New). But, this 105 | // error also captures the caller's Program Counter and the preceding error. 106 | func NewError(cause error, msg string) error { 107 | return &pcError{ 108 | ErrorNode: ErrorNode{}.Initialize(cause, 3), 109 | msg: msg, 110 | } 111 | } 112 | 113 | // pcError is a simple string error (like error.New) with an ErrorNode (PC & cause). 114 | type pcError struct { 115 | ErrorNode 116 | msg string 117 | } 118 | 119 | // Error satisfies the error interface. It shows the error with Program Counter 120 | // symbols and calls Error on the preceding error so you can see the full error chain. 121 | func (e *pcError) Error() string { return e.ErrorNode.Error(e.msg) } 122 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-pipeline-go/pipeline/progress.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import "io" 4 | 5 | // ********** The following is common between the request body AND the response body. 6 | 7 | // ProgressReceiver defines the signature of a callback function invoked as progress is reported. 8 | type ProgressReceiver func(bytesTransferred int64) 9 | 10 | // ********** The following are specific to the request body (a ReadSeekCloser) 11 | 12 | // This struct is used when sending a body to the network 13 | type requestBodyProgress struct { 14 | requestBody io.ReadSeeker // Seeking is required to support retries 15 | pr ProgressReceiver 16 | } 17 | 18 | // NewRequestBodyProgress adds progress reporting to an HTTP request's body stream. 19 | func NewRequestBodyProgress(requestBody io.ReadSeeker, pr ProgressReceiver) io.ReadSeeker { 20 | if pr == nil { 21 | panic("pr must not be nil") 22 | } 23 | return &requestBodyProgress{requestBody: requestBody, pr: pr} 24 | } 25 | 26 | // Read reads a block of data from an inner stream and reports progress 27 | func (rbp *requestBodyProgress) Read(p []byte) (n int, err error) { 28 | n, err = rbp.requestBody.Read(p) 29 | if err != nil { 30 | return 31 | } 32 | // Invokes the user's callback method to report progress 33 | position, err := rbp.requestBody.Seek(0, io.SeekCurrent) 34 | if err != nil { 35 | panic(err) 36 | } 37 | rbp.pr(position) 38 | return 39 | } 40 | 41 | func (rbp *requestBodyProgress) Seek(offset int64, whence int) (offsetFromStart int64, err error) { 42 | return rbp.requestBody.Seek(offset, whence) 43 | } 44 | 45 | // requestBodyProgress supports Close but the underlying stream may not; if it does, Close will close it. 46 | func (rbp *requestBodyProgress) Close() error { 47 | if c, ok := rbp.requestBody.(io.Closer); ok { 48 | return c.Close() 49 | } 50 | return nil 51 | } 52 | 53 | // ********** The following are specific to the response body (a ReadCloser) 54 | 55 | // This struct is used when sending a body to the network 56 | type responseBodyProgress struct { 57 | responseBody io.ReadCloser 58 | pr ProgressReceiver 59 | offset int64 60 | } 61 | 62 | // NewResponseBodyProgress adds progress reporting to an HTTP response's body stream. 63 | func NewResponseBodyProgress(responseBody io.ReadCloser, pr ProgressReceiver) io.ReadCloser { 64 | if pr == nil { 65 | panic("pr must not be nil") 66 | } 67 | return &responseBodyProgress{responseBody: responseBody, pr: pr, offset: 0} 68 | } 69 | 70 | // Read reads a block of data from an inner stream and reports progress 71 | func (rbp *responseBodyProgress) Read(p []byte) (n int, err error) { 72 | n, err = rbp.responseBody.Read(p) 73 | rbp.offset += int64(n) 74 | 75 | // Invokes the user's callback method to report progress 76 | rbp.pr(rbp.offset) 77 | return 78 | } 79 | 80 | func (rbp *responseBodyProgress) Close() error { 81 | return rbp.responseBody.Close() 82 | } 83 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-pipeline-go/pipeline/response.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/http" 7 | "sort" 8 | "strings" 9 | ) 10 | 11 | // The Response interface exposes an http.Response object as it returns through the pipeline of Policy objects. 12 | // This ensures that Policy objects have access to the HTTP response. However, the object this interface encapsulates 13 | // might be a struct with additional fields that is created by a Policy object (typically a method-specific Factory). 14 | // The method that injected the method-specific Factory gets this returned Response and performs a type assertion 15 | // to the expected struct and returns the struct to its caller. 16 | type Response interface { 17 | Response() *http.Response 18 | } 19 | 20 | // This is the default struct that has the http.Response. 21 | // A method can replace this struct with its own struct containing an http.Response 22 | // field and any other additional fields. 23 | type httpResponse struct { 24 | response *http.Response 25 | } 26 | 27 | // NewHTTPResponse is typically called by a Policy object to return a Response object. 28 | func NewHTTPResponse(response *http.Response) Response { 29 | return &httpResponse{response: response} 30 | } 31 | 32 | // This method satisfies the public Response interface's Response method 33 | func (r httpResponse) Response() *http.Response { 34 | return r.response 35 | } 36 | 37 | // WriteRequestWithResponse appends a formatted HTTP request into a Buffer. If request and/or err are 38 | // not nil, then these are also written into the Buffer. 39 | func WriteRequestWithResponse(b *bytes.Buffer, request *http.Request, response *http.Response, err error) { 40 | // Write the request into the buffer. 41 | fmt.Fprint(b, " "+request.Method+" "+request.URL.String()+"\n") 42 | writeHeader(b, request.Header) 43 | if response != nil { 44 | fmt.Fprintln(b, " --------------------------------------------------------------------------------") 45 | fmt.Fprint(b, " RESPONSE Status: "+response.Status+"\n") 46 | writeHeader(b, response.Header) 47 | } 48 | if err != nil { 49 | fmt.Fprintln(b, " --------------------------------------------------------------------------------") 50 | fmt.Fprint(b, " ERROR:\n"+err.Error()+"\n") 51 | } 52 | } 53 | 54 | // formatHeaders appends an HTTP request's or response's header into a Buffer. 55 | func writeHeader(b *bytes.Buffer, header map[string][]string) { 56 | if len(header) == 0 { 57 | b.WriteString(" (no headers)\n") 58 | return 59 | } 60 | keys := make([]string, 0, len(header)) 61 | // Alphabetize the headers 62 | for k := range header { 63 | keys = append(keys, k) 64 | } 65 | sort.Strings(keys) 66 | for _, k := range keys { 67 | // Redact the value of any Authorization header to prevent security information from persisting in logs 68 | value := interface{}("REDACTED") 69 | if !strings.EqualFold(k, "Authorization") { 70 | value = header[k] 71 | } 72 | fmt.Fprintf(b, " %s: %+v\n", k, value) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-pipeline-go/pipeline/version.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | const ( 4 | // UserAgent is the string to be used in the user agent string when making requests. 5 | UserAgent = "azure-pipeline-go/" + Version 6 | 7 | // Version is the semantic version (see http://semver.org) of the pipeline package. 8 | Version = "0.1.0" 9 | ) 10 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/access_conditions.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // ModifiedAccessConditions identifies standard HTTP access conditions which you optionally set. 8 | type ModifiedAccessConditions struct { 9 | IfModifiedSince time.Time 10 | IfUnmodifiedSince time.Time 11 | IfMatch ETag 12 | IfNoneMatch ETag 13 | } 14 | 15 | // pointers is for internal infrastructure. It returns the fields as pointers. 16 | func (ac ModifiedAccessConditions) pointers() (ims *time.Time, ius *time.Time, ime *ETag, inme *ETag) { 17 | if !ac.IfModifiedSince.IsZero() { 18 | ims = &ac.IfModifiedSince 19 | } 20 | if !ac.IfUnmodifiedSince.IsZero() { 21 | ius = &ac.IfUnmodifiedSince 22 | } 23 | if ac.IfMatch != ETagNone { 24 | ime = &ac.IfMatch 25 | } 26 | if ac.IfNoneMatch != ETagNone { 27 | inme = &ac.IfNoneMatch 28 | } 29 | return 30 | } 31 | 32 | // ContainerAccessConditions identifies container-specific access conditions which you optionally set. 33 | type ContainerAccessConditions struct { 34 | ModifiedAccessConditions 35 | LeaseAccessConditions 36 | } 37 | 38 | // BlobAccessConditions identifies blob-specific access conditions which you optionally set. 39 | type BlobAccessConditions struct { 40 | ModifiedAccessConditions 41 | LeaseAccessConditions 42 | } 43 | 44 | // LeaseAccessConditions identifies lease access conditions for a container or blob which you optionally set. 45 | type LeaseAccessConditions struct { 46 | LeaseID string 47 | } 48 | 49 | // pointers is for internal infrastructure. It returns the fields as pointers. 50 | func (ac LeaseAccessConditions) pointers() (leaseID *string) { 51 | if ac.LeaseID != "" { 52 | leaseID = &ac.LeaseID 53 | } 54 | return 55 | } 56 | 57 | /* 58 | // getInt32 is for internal infrastructure. It is used with access condition values where 59 | // 0 (the default setting) is meaningful. The library interprets 0 as do not send the header 60 | // and the privately-storage field in the access condition object is stored as +1 higher than desired. 61 | // THis method returns true, if the value is > 0 (explicitly set) and the stored value - 1 (the set desired value). 62 | func getInt32(value int32) (bool, int32) { 63 | return value > 0, value - 1 64 | } 65 | */ 66 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/atomicmorph.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | import "sync/atomic" 4 | 5 | // AtomicMorpherInt32 identifies a method passed to and invoked by the AtomicMorphInt32 function. 6 | // The AtomicMorpher callback is passed a startValue and based on this value it returns 7 | // what the new value should be and the result that AtomicMorph should return to its caller. 8 | type atomicMorpherInt32 func(startVal int32) (val int32, morphResult interface{}) 9 | 10 | const targetAndMorpherMustNotBeNil = "target and morpher must not be nil" 11 | 12 | // AtomicMorph atomically morphs target in to new value (and result) as indicated bythe AtomicMorpher callback function. 13 | func atomicMorphInt32(target *int32, morpher atomicMorpherInt32) interface{} { 14 | for { 15 | currentVal := atomic.LoadInt32(target) 16 | desiredVal, morphResult := morpher(currentVal) 17 | if atomic.CompareAndSwapInt32(target, currentVal, desiredVal) { 18 | return morphResult 19 | } 20 | } 21 | } 22 | 23 | // AtomicMorpherUint32 identifies a method passed to and invoked by the AtomicMorph function. 24 | // The AtomicMorpher callback is passed a startValue and based on this value it returns 25 | // what the new value should be and the result that AtomicMorph should return to its caller. 26 | type atomicMorpherUint32 func(startVal uint32) (val uint32, morphResult interface{}) 27 | 28 | // AtomicMorph atomically morphs target in to new value (and result) as indicated bythe AtomicMorpher callback function. 29 | func atomicMorphUint32(target *uint32, morpher atomicMorpherUint32) interface{} { 30 | for { 31 | currentVal := atomic.LoadUint32(target) 32 | desiredVal, morphResult := morpher(currentVal) 33 | if atomic.CompareAndSwapUint32(target, currentVal, desiredVal) { 34 | return morphResult 35 | } 36 | } 37 | } 38 | 39 | // AtomicMorpherUint64 identifies a method passed to and invoked by the AtomicMorphUint64 function. 40 | // The AtomicMorpher callback is passed a startValue and based on this value it returns 41 | // what the new value should be and the result that AtomicMorph should return to its caller. 42 | type atomicMorpherInt64 func(startVal int64) (val int64, morphResult interface{}) 43 | 44 | // AtomicMorph atomically morphs target in to new value (and result) as indicated bythe AtomicMorpher callback function. 45 | func atomicMorphInt64(target *int64, morpher atomicMorpherInt64) interface{} { 46 | for { 47 | currentVal := atomic.LoadInt64(target) 48 | desiredVal, morphResult := morpher(currentVal) 49 | if atomic.CompareAndSwapInt64(target, currentVal, desiredVal) { 50 | return morphResult 51 | } 52 | } 53 | } 54 | 55 | // AtomicMorpherUint64 identifies a method passed to and invoked by the AtomicMorphUint64 function. 56 | // The AtomicMorpher callback is passed a startValue and based on this value it returns 57 | // what the new value should be and the result that AtomicMorph should return to its caller. 58 | type atomicMorpherUint64 func(startVal uint64) (val uint64, morphResult interface{}) 59 | 60 | // AtomicMorph atomically morphs target in to new value (and result) as indicated bythe AtomicMorpher callback function. 61 | func atomicMorphUint64(target *uint64, morpher atomicMorpherUint64) interface{} { 62 | for { 63 | currentVal := atomic.LoadUint64(target) 64 | desiredVal, morphResult := morpher(currentVal) 65 | if atomic.CompareAndSwapUint64(target, currentVal, desiredVal) { 66 | return morphResult 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/version.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | const serviceLibVersion = "0.5" 4 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/zc_credential_anonymous.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/Azure/azure-pipeline-go/pipeline" 7 | ) 8 | 9 | // Credential represent any credential type; it is used to create a credential policy Factory. 10 | type Credential interface { 11 | pipeline.Factory 12 | credentialMarker() 13 | } 14 | 15 | type credentialFunc pipeline.FactoryFunc 16 | 17 | func (f credentialFunc) New(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.Policy { 18 | return f(next, po) 19 | } 20 | 21 | // credentialMarker is a package-internal method that exists just to satisfy the Credential interface. 22 | func (credentialFunc) credentialMarker() {} 23 | 24 | ////////////////////////////// 25 | 26 | // NewAnonymousCredential creates an anonymous credential for use with HTTP(S) requests that read public resource 27 | // or for use with Shared Access Signatures (SAS). 28 | func NewAnonymousCredential() Credential { 29 | return anonymousCredentialFactory 30 | } 31 | 32 | var anonymousCredentialFactory Credential = &anonymousCredentialPolicyFactory{} // Singleton 33 | 34 | // anonymousCredentialPolicyFactory is the credential's policy factory. 35 | type anonymousCredentialPolicyFactory struct { 36 | } 37 | 38 | // New creates a credential policy object. 39 | func (f *anonymousCredentialPolicyFactory) New(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.Policy { 40 | return &anonymousCredentialPolicy{next: next} 41 | } 42 | 43 | // credentialMarker is a package-internal method that exists just to satisfy the Credential interface. 44 | func (*anonymousCredentialPolicyFactory) credentialMarker() {} 45 | 46 | // anonymousCredentialPolicy is the credential's policy object. 47 | type anonymousCredentialPolicy struct { 48 | next pipeline.Policy 49 | } 50 | 51 | // Do implements the credential's policy interface. 52 | func (p anonymousCredentialPolicy) Do(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { 53 | // For anonymous credentials, this is effectively a no-op 54 | return p.next.Do(ctx, request) 55 | } 56 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/zc_mmf_unix.go: -------------------------------------------------------------------------------- 1 | // +build linux darwin freebsd openbsd netbsd 2 | 3 | package azblob 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | ) 9 | 10 | type mmf []byte 11 | 12 | func newMMF(file *os.File, writable bool, offset int64, length int) (mmf, error) { 13 | prot, flags := syscall.PROT_READ, syscall.MAP_SHARED // Assume read-only 14 | if writable { 15 | prot, flags = syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED 16 | } 17 | addr, err := syscall.Mmap(int(file.Fd()), offset, length, prot, flags) 18 | return mmf(addr), err 19 | } 20 | 21 | func (m *mmf) unmap() { 22 | err := syscall.Munmap(*m) 23 | *m = nil 24 | if err != nil { 25 | panic("if we are unable to unmap the memory-mapped file, there is serious concern for memory corruption") 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/zc_mmf_windows.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | "syscall" 7 | "unsafe" 8 | ) 9 | 10 | type mmf []byte 11 | 12 | func newMMF(file *os.File, writable bool, offset int64, length int) (mmf, error) { 13 | prot, access := uint32(syscall.PAGE_READONLY), uint32(syscall.FILE_MAP_READ) // Assume read-only 14 | if writable { 15 | prot, access = uint32(syscall.PAGE_READWRITE), uint32(syscall.FILE_MAP_WRITE) 16 | } 17 | hMMF, errno := syscall.CreateFileMapping(syscall.Handle(file.Fd()), nil, prot, uint32(int64(length)>>32), uint32(int64(length)&0xffffffff), nil) 18 | if hMMF == 0 { 19 | return nil, os.NewSyscallError("CreateFileMapping", errno) 20 | } 21 | defer syscall.CloseHandle(hMMF) 22 | addr, errno := syscall.MapViewOfFile(hMMF, access, uint32(offset>>32), uint32(offset&0xffffffff), uintptr(length)) 23 | m := mmf{} 24 | h := (*reflect.SliceHeader)(unsafe.Pointer(&m)) 25 | h.Data = addr 26 | h.Len = length 27 | h.Cap = h.Len 28 | return m, nil 29 | } 30 | 31 | func (m *mmf) unmap() { 32 | addr := uintptr(unsafe.Pointer(&(([]byte)(*m)[0]))) 33 | *m = mmf{} 34 | err := syscall.UnmapViewOfFile(addr) 35 | if err != nil { 36 | panic("if we are unable to unmap the memory-mapped file, there is serious concern for memory corruption") 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/zc_pipeline.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | import ( 4 | "github.com/Azure/azure-pipeline-go/pipeline" 5 | ) 6 | 7 | // PipelineOptions is used to configure a request policy pipeline's retry policy and logging. 8 | type PipelineOptions struct { 9 | // Log configures the pipeline's logging infrastructure indicating what information is logged and where. 10 | Log pipeline.LogOptions 11 | 12 | // Retry configures the built-in retry policy behavior. 13 | Retry RetryOptions 14 | 15 | // RequestLog configures the built-in request logging policy. 16 | RequestLog RequestLogOptions 17 | 18 | // Telemetry configures the built-in telemetry policy behavior. 19 | Telemetry TelemetryOptions 20 | } 21 | 22 | // NewPipeline creates a Pipeline using the specified credentials and options. 23 | func NewPipeline(c Credential, o PipelineOptions) pipeline.Pipeline { 24 | // Closest to API goes first; closest to the wire goes last 25 | f := []pipeline.Factory{ 26 | NewTelemetryPolicyFactory(o.Telemetry), 27 | NewUniqueRequestIDPolicyFactory(), 28 | NewRetryPolicyFactory(o.Retry), 29 | } 30 | 31 | if _, ok := c.(*anonymousCredentialPolicyFactory); !ok { 32 | // For AnonymousCredential, we optimize out the policy factory since it doesn't do anything 33 | // NOTE: The credential's policy factory must appear close to the wire so it can sign any 34 | // changes made by other factories (like UniqueRequestIDPolicyFactory) 35 | f = append(f, c) 36 | } 37 | f = append(f, 38 | pipeline.MethodFactoryMarker(), // indicates at what stage in the pipeline the method factory is invoked 39 | NewRequestLogPolicyFactory(o.RequestLog)) 40 | 41 | return pipeline.NewPipeline(f, pipeline.Options{HTTPSender: nil, Log: o.Log}) 42 | } 43 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/zc_policy_telemetry.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "os" 8 | "runtime" 9 | 10 | "github.com/Azure/azure-pipeline-go/pipeline" 11 | ) 12 | 13 | // TelemetryOptions configures the telemetry policy's behavior. 14 | type TelemetryOptions struct { 15 | // Value is a string prepended to each request's User-Agent and sent to the service. 16 | // The service records the user-agent in logs for diagnostics and tracking of client requests. 17 | Value string 18 | } 19 | 20 | // NewTelemetryPolicyFactory creates a factory that can create telemetry policy objects 21 | // which add telemetry information to outgoing HTTP requests. 22 | func NewTelemetryPolicyFactory(o TelemetryOptions) pipeline.Factory { 23 | b := &bytes.Buffer{} 24 | b.WriteString(o.Value) 25 | if b.Len() > 0 { 26 | b.WriteRune(' ') 27 | } 28 | fmt.Fprintf(b, "Azure-Storage/%s %s", serviceLibVersion, platformInfo) 29 | telemetryValue := b.String() 30 | 31 | return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc { 32 | return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { 33 | request.Header.Set("User-Agent", telemetryValue) 34 | return next.Do(ctx, request) 35 | } 36 | }) 37 | } 38 | 39 | // NOTE: the ONLY function that should write to this variable is this func 40 | var platformInfo = func() string { 41 | // Azure-Storage/version (runtime; os type and version)” 42 | // Azure-Storage/1.4.0 (NODE-VERSION v4.5.0; Windows_NT 10.0.14393)' 43 | operatingSystem := runtime.GOOS // Default OS string 44 | switch operatingSystem { 45 | case "windows": 46 | operatingSystem = os.Getenv("OS") // Get more specific OS information 47 | case "linux": // accept default OS info 48 | case "freebsd": // accept default OS info 49 | } 50 | return fmt.Sprintf("(%s; %s)", runtime.Version(), operatingSystem) 51 | }() 52 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/zc_policy_unique_request_id.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/Azure/azure-pipeline-go/pipeline" 7 | ) 8 | 9 | // NewUniqueRequestIDPolicyFactory creates a UniqueRequestIDPolicyFactory object 10 | // that sets the request's x-ms-client-request-id header if it doesn't already exist. 11 | func NewUniqueRequestIDPolicyFactory() pipeline.Factory { 12 | return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc { 13 | // This is Policy's Do method: 14 | return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { 15 | id := request.Header.Get(xMsClientRequestID) 16 | if id == "" { // Add a unique request ID if the caller didn't specify one already 17 | request.Header.Set(xMsClientRequestID, newUUID().String()) 18 | } 19 | return next.Do(ctx, request) 20 | } 21 | }) 22 | } 23 | 24 | const xMsClientRequestID = "x-ms-client-request-id" 25 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/zc_storage_error.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | import ( 4 | "bytes" 5 | "encoding/xml" 6 | "fmt" 7 | "net/http" 8 | "sort" 9 | 10 | "github.com/Azure/azure-pipeline-go/pipeline" 11 | ) 12 | 13 | func init() { 14 | // wire up our custom error handling constructor 15 | responseErrorFactory = newStorageError 16 | } 17 | 18 | // ServiceCodeType is a string identifying a storage service error. 19 | // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/status-and-error-codes2 20 | type ServiceCodeType string 21 | 22 | // StorageError identifies a responder-generated network or response parsing error. 23 | type StorageError interface { 24 | // ResponseError implements error's Error(), net.Error's Temporary() and Timeout() methods & Response(). 25 | ResponseError 26 | 27 | // ServiceCode returns a service error code. Your code can use this to make error recovery decisions. 28 | ServiceCode() ServiceCodeType 29 | } 30 | 31 | // storageError is the internal struct that implements the public StorageError interface. 32 | type storageError struct { 33 | responseError 34 | serviceCode ServiceCodeType 35 | details map[string]string 36 | } 37 | 38 | // newStorageError creates an error object that implements the error interface. 39 | func newStorageError(cause error, response *http.Response, description string) error { 40 | return &storageError{ 41 | responseError: responseError{ 42 | ErrorNode: pipeline.ErrorNode{}.Initialize(cause, 3), 43 | response: response, 44 | description: description, 45 | }, 46 | serviceCode: ServiceCodeType(response.Header.Get("x-ms-error-code")), 47 | } 48 | } 49 | 50 | // ServiceCode returns service-error information. The caller may examine these values but should not modify any of them. 51 | func (e *storageError) ServiceCode() ServiceCodeType { 52 | return e.serviceCode 53 | } 54 | 55 | // Error implements the error interface's Error method to return a string representation of the error. 56 | func (e *storageError) Error() string { 57 | b := &bytes.Buffer{} 58 | fmt.Fprintf(b, "===== RESPONSE ERROR (ServiceCode=%s) =====\n", e.serviceCode) 59 | fmt.Fprintf(b, "Description=%s, Details: ", e.description) 60 | if len(e.details) == 0 { 61 | b.WriteString("(none)\n") 62 | } else { 63 | b.WriteRune('\n') 64 | keys := make([]string, 0, len(e.details)) 65 | // Alphabetize the details 66 | for k := range e.details { 67 | keys = append(keys, k) 68 | } 69 | sort.Strings(keys) 70 | for _, k := range keys { 71 | fmt.Fprintf(b, " %s: %+v\n", k, e.details[k]) 72 | } 73 | } 74 | req := pipeline.Request{Request: e.response.Request}.Copy() // Make a copy of the response's request 75 | pipeline.WriteRequestWithResponse(b, prepareRequestForLogging(req), e.response, nil) 76 | return e.ErrorNode.Error(b.String()) 77 | } 78 | 79 | // Temporary returns true if the error occurred due to a temporary condition (including an HTTP status of 500 or 503). 80 | func (e *storageError) Temporary() bool { 81 | if e.response != nil { 82 | if (e.response.StatusCode == http.StatusInternalServerError) || (e.response.StatusCode == http.StatusServiceUnavailable) { 83 | return true 84 | } 85 | } 86 | return e.ErrorNode.Temporary() 87 | } 88 | 89 | // UnmarshalXML performs custom unmarshalling of XML-formatted Azure storage request errors. 90 | func (e *storageError) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) { 91 | tokName := "" 92 | var t xml.Token 93 | for t, err = d.Token(); err == nil; t, err = d.Token() { 94 | switch tt := t.(type) { 95 | case xml.StartElement: 96 | tokName = tt.Name.Local 97 | break 98 | case xml.CharData: 99 | switch tokName { 100 | case "Message": 101 | e.description = string(tt) 102 | default: 103 | if e.details == nil { 104 | e.details = map[string]string{} 105 | } 106 | e.details[tokName] = string(tt) 107 | } 108 | } 109 | } 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/zc_util_validate.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "strconv" 8 | ) 9 | 10 | // httpRange defines a range of bytes within an HTTP resource, starting at offset and 11 | // ending at offset+count. A zero-value httpRange indicates the entire resource. An httpRange 12 | // which has an offset but na zero value count indicates from the offset to the resource's end. 13 | type httpRange struct { 14 | offset int64 15 | count int64 16 | } 17 | 18 | func (r httpRange) pointers() *string { 19 | if r.offset == 0 && r.count == CountToEnd { // Do common case first for performance 20 | return nil // No specified range 21 | } 22 | endOffset := "" // if count == CountToEnd (0) 23 | if r.count > 0 { 24 | endOffset = strconv.FormatInt((r.offset+r.count)-1, 10) 25 | } 26 | dataRange := fmt.Sprintf("bytes=%v-%s", r.offset, endOffset) 27 | return &dataRange 28 | } 29 | 30 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// 31 | 32 | func validateSeekableStreamAt0AndGetCount(body io.ReadSeeker) (int64, error) { 33 | if body == nil { // nil body's are "logically" seekable to 0 and are 0 bytes long 34 | return 0, nil 35 | } 36 | 37 | err := validateSeekableStreamAt0(body) 38 | if err != nil { 39 | return 0, err 40 | } 41 | 42 | count, err := body.Seek(0, io.SeekEnd) 43 | if err != nil { 44 | return 0, errors.New("body stream must be seekable") 45 | } 46 | 47 | body.Seek(0, io.SeekStart) 48 | return count, nil 49 | } 50 | 51 | // return an error if body is not a valid seekable stream at 0 52 | func validateSeekableStreamAt0(body io.ReadSeeker) error { 53 | if body == nil { // nil body's are "logically" seekable to 0 54 | return nil 55 | } 56 | if pos, err := body.Seek(0, io.SeekCurrent); pos != 0 || err != nil { 57 | // Help detect programmer error 58 | if err != nil { 59 | return errors.New("body stream must be seekable") 60 | } 61 | return errors.New("body stream must be set to position 0") 62 | } 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/zc_uuid.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | import ( 4 | "crypto/rand" 5 | "fmt" 6 | "strconv" 7 | ) 8 | 9 | // The UUID reserved variants. 10 | const ( 11 | reservedNCS byte = 0x80 12 | reservedRFC4122 byte = 0x40 13 | reservedMicrosoft byte = 0x20 14 | reservedFuture byte = 0x00 15 | ) 16 | 17 | // A UUID representation compliant with specification in RFC 4122 document. 18 | type uuid [16]byte 19 | 20 | // NewUUID returns a new uuid using RFC 4122 algorithm. 21 | func newUUID() (u uuid) { 22 | u = uuid{} 23 | // Set all bits to randomly (or pseudo-randomly) chosen values. 24 | rand.Read(u[:]) 25 | u[8] = (u[8] | reservedRFC4122) & 0x7F // u.setVariant(ReservedRFC4122) 26 | 27 | var version byte = 4 28 | u[6] = (u[6] & 0xF) | (version << 4) // u.setVersion(4) 29 | return 30 | } 31 | 32 | // String returns an unparsed version of the generated UUID sequence. 33 | func (u uuid) String() string { 34 | return fmt.Sprintf("%x-%x-%x-%x-%x", u[0:4], u[4:6], u[6:8], u[8:10], u[10:]) 35 | } 36 | 37 | // ParseUUID parses a string formatted as "003020100-0504-0706-0809-0a0b0c0d0e0f" 38 | // or "{03020100-0504-0706-0809-0a0b0c0d0e0f}" into a UUID. 39 | func parseUUID(uuidStr string) uuid { 40 | char := func(hexString string) byte { 41 | i, _ := strconv.ParseUint(hexString, 16, 8) 42 | return byte(i) 43 | } 44 | if uuidStr[0] == '{' { 45 | uuidStr = uuidStr[1:] // Skip over the '{' 46 | } 47 | // 03020100 - 05 04 - 07 06 - 08 09 - 0a 0b 0c 0d 0e 0f 48 | // 1 11 1 11 11 1 12 22 2 22 22 22 33 33 33 49 | // 01234567 8 90 12 3 45 67 8 90 12 3 45 67 89 01 23 45 50 | uuidVal := uuid{ 51 | char(uuidStr[0:2]), 52 | char(uuidStr[2:4]), 53 | char(uuidStr[4:6]), 54 | char(uuidStr[6:8]), 55 | 56 | char(uuidStr[9:11]), 57 | char(uuidStr[11:13]), 58 | 59 | char(uuidStr[14:16]), 60 | char(uuidStr[16:18]), 61 | 62 | char(uuidStr[19:21]), 63 | char(uuidStr[21:23]), 64 | 65 | char(uuidStr[24:26]), 66 | char(uuidStr[26:28]), 67 | char(uuidStr[28:30]), 68 | char(uuidStr[30:32]), 69 | char(uuidStr[32:34]), 70 | char(uuidStr[34:36]), 71 | } 72 | return uuidVal 73 | } 74 | 75 | func (u uuid) bytes() []byte { 76 | return u[:] 77 | } 78 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/zz_generated_client.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | // Code generated by Microsoft (R) AutoRest Code Generator. 4 | // Changes may cause incorrect behavior and will be lost if the code is regenerated. 5 | 6 | import ( 7 | "net/url" 8 | 9 | "github.com/Azure/azure-pipeline-go/pipeline" 10 | ) 11 | 12 | const ( 13 | // ServiceVersion specifies the version of the operations used in this package. 14 | ServiceVersion = "2018-03-28" 15 | ) 16 | 17 | // managementClient is the base client for Azblob. 18 | type managementClient struct { 19 | url url.URL 20 | p pipeline.Pipeline 21 | } 22 | 23 | // newManagementClient creates an instance of the managementClient client. 24 | func newManagementClient(url url.URL, p pipeline.Pipeline) managementClient { 25 | return managementClient{ 26 | url: url, 27 | p: p, 28 | } 29 | } 30 | 31 | // URL returns a copy of the URL for this client. 32 | func (mc managementClient) URL() url.URL { 33 | return mc.url 34 | } 35 | 36 | // Pipeline returns the pipeline for this client. 37 | func (mc managementClient) Pipeline() pipeline.Pipeline { 38 | return mc.p 39 | } 40 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/zz_generated_responder_policy.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | // Code generated by Microsoft (R) AutoRest Code Generator. 4 | // Changes may cause incorrect behavior and will be lost if the code is regenerated. 5 | 6 | import ( 7 | "bytes" 8 | "context" 9 | "encoding/xml" 10 | "io/ioutil" 11 | 12 | "github.com/Azure/azure-pipeline-go/pipeline" 13 | ) 14 | 15 | type responder func(resp pipeline.Response) (result pipeline.Response, err error) 16 | 17 | // ResponderPolicyFactory is a Factory capable of creating a responder pipeline. 18 | type responderPolicyFactory struct { 19 | responder responder 20 | } 21 | 22 | // New creates a responder policy factory. 23 | func (arpf responderPolicyFactory) New(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.Policy { 24 | return responderPolicy{next: next, responder: arpf.responder} 25 | } 26 | 27 | type responderPolicy struct { 28 | next pipeline.Policy 29 | responder responder 30 | } 31 | 32 | // Do sends the request to the service and validates/deserializes the HTTP response. 33 | func (arp responderPolicy) Do(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { 34 | resp, err := arp.next.Do(ctx, request) 35 | if err != nil { 36 | return resp, err 37 | } 38 | return arp.responder(resp) 39 | } 40 | 41 | // validateResponse checks an HTTP response's status code against a legal set of codes. 42 | // If the response code is not legal, then validateResponse reads all of the response's body 43 | // (containing error information) and returns a response error. 44 | func validateResponse(resp pipeline.Response, successStatusCodes ...int) error { 45 | if resp == nil { 46 | return NewResponseError(nil, nil, "nil response") 47 | } 48 | responseCode := resp.Response().StatusCode 49 | for _, i := range successStatusCodes { 50 | if i == responseCode { 51 | return nil 52 | } 53 | } 54 | // only close the body in the failure case. in the 55 | // success case responders will close the body as required. 56 | defer resp.Response().Body.Close() 57 | b, err := ioutil.ReadAll(resp.Response().Body) 58 | if err != nil { 59 | return err 60 | } 61 | // the service code, description and details will be populated during unmarshalling 62 | responseError := NewResponseError(nil, resp.Response(), resp.Response().Status) 63 | if len(b) > 0 { 64 | if err = xml.Unmarshal(b, &responseError); err != nil { 65 | return NewResponseError(err, resp.Response(), "failed to unmarshal response body") 66 | } 67 | } 68 | return responseError 69 | } 70 | 71 | // removes any BOM from the byte slice 72 | func removeBOM(b []byte) []byte { 73 | // UTF8 74 | return bytes.TrimPrefix(b, []byte("\xef\xbb\xbf")) 75 | } 76 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/zz_generated_response_error.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | // Code generated by Microsoft (R) AutoRest Code Generator. 4 | // Changes may cause incorrect behavior and will be lost if the code is regenerated. 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "net" 10 | "net/http" 11 | 12 | "github.com/Azure/azure-pipeline-go/pipeline" 13 | ) 14 | 15 | // if you want to provide custom error handling set this variable to your constructor function 16 | var responseErrorFactory func(cause error, response *http.Response, description string) error 17 | 18 | // ResponseError identifies a responder-generated network or response parsing error. 19 | type ResponseError interface { 20 | // Error exposes the Error(), Temporary() and Timeout() methods. 21 | net.Error // Includes the Go error interface 22 | // Response returns the HTTP response. You may examine this but you should not modify it. 23 | Response() *http.Response 24 | } 25 | 26 | // NewResponseError creates an error object that implements the error interface. 27 | func NewResponseError(cause error, response *http.Response, description string) error { 28 | if responseErrorFactory != nil { 29 | return responseErrorFactory(cause, response, description) 30 | } 31 | return &responseError{ 32 | ErrorNode: pipeline.ErrorNode{}.Initialize(cause, 3), 33 | response: response, 34 | description: description, 35 | } 36 | } 37 | 38 | // responseError is the internal struct that implements the public ResponseError interface. 39 | type responseError struct { 40 | pipeline.ErrorNode // This is embedded so that responseError "inherits" Error, Temporary, Timeout, and Cause 41 | response *http.Response 42 | description string 43 | } 44 | 45 | // Error implements the error interface's Error method to return a string representation of the error. 46 | func (e *responseError) Error() string { 47 | b := &bytes.Buffer{} 48 | fmt.Fprintf(b, "===== RESPONSE ERROR (Code=%v) =====\n", e.response.StatusCode) 49 | fmt.Fprintf(b, "Status=%s, Description: %s\n", e.response.Status, e.description) 50 | s := b.String() 51 | return e.ErrorNode.Error(s) 52 | } 53 | 54 | // Response implements the ResponseError interface's method to return the HTTP response. 55 | func (e *responseError) Response() *http.Response { 56 | return e.response 57 | } 58 | 59 | // RFC7807 PROBLEM ------------------------------------------------------------------------------------ 60 | // RFC7807Problem ... This type can be publicly embedded in another type that wants to add additional members. 61 | /*type RFC7807Problem struct { 62 | // Mandatory: A (relative) URI reference identifying the problem type (it MAY refer to human-readable documentation). 63 | typeURI string // Should default to "about:blank" 64 | // Optional: Short, human-readable summary (maybe localized). 65 | title string 66 | // Optional: HTTP status code generated by the origin server 67 | status int 68 | // Optional: Human-readable explanation for this problem occurance. 69 | // Should help client correct the problem. Clients should NOT parse this string. 70 | detail string 71 | // Optional: A (relative) URI identifying this specific problem occurence (it may or may not be dereferenced). 72 | instance string 73 | } 74 | // NewRFC7807Problem ... 75 | func NewRFC7807Problem(typeURI string, status int, titleFormat string, a ...interface{}) error { 76 | return &RFC7807Problem{ 77 | typeURI: typeURI, 78 | status: status, 79 | title: fmt.Sprintf(titleFormat, a...), 80 | } 81 | } 82 | // Error returns the error information as a string. 83 | func (e *RFC7807Problem) Error() string { 84 | return e.title 85 | } 86 | // TypeURI ... 87 | func (e *RFC7807Problem) TypeURI() string { 88 | if e.typeURI == "" { 89 | e.typeURI = "about:blank" 90 | } 91 | return e.typeURI 92 | } 93 | // Members ... 94 | func (e *RFC7807Problem) Members() (status int, title, detail, instance string) { 95 | return e.status, e.title, e.detail, e.instance 96 | }*/ 97 | -------------------------------------------------------------------------------- /vendor/github.com/Azure/azure-storage-blob-go/azblob/zz_generated_version.go: -------------------------------------------------------------------------------- 1 | package azblob 2 | 3 | // Code generated by Microsoft (R) AutoRest Code Generator. 4 | // Changes may cause incorrect behavior and will be lost if the code is regenerated. 5 | 6 | // UserAgent returns the UserAgent string to use when sending http.Requests. 7 | func UserAgent() string { 8 | return "Azure-SDK-For-Go/0.0.0 azblob/2018-03-28" 9 | } 10 | 11 | // Version returns the semantic version (see http://semver.org) of the client. 12 | func Version() string { 13 | return "0.0.0" 14 | } 15 | -------------------------------------------------------------------------------- /vendor/github.com/davecgh/go-spew/LICENSE: -------------------------------------------------------------------------------- 1 | ISC License 2 | 3 | Copyright (c) 2012-2016 Dave Collins 4 | 5 | Permission to use, copy, modify, and distribute this software for any 6 | purpose with or without fee is hereby granted, provided that the above 7 | copyright notice and this permission notice appear in all copies. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | -------------------------------------------------------------------------------- /vendor/github.com/davecgh/go-spew/spew/bypasssafe.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2016 Dave Collins 2 | // 3 | // Permission to use, copy, modify, and distribute this software for any 4 | // purpose with or without fee is hereby granted, provided that the above 5 | // copyright notice and this permission notice appear in all copies. 6 | // 7 | // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | // ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | 15 | // NOTE: Due to the following build constraints, this file will only be compiled 16 | // when the code is running on Google App Engine, compiled by GopherJS, or 17 | // "-tags safe" is added to the go build command line. The "disableunsafe" 18 | // tag is deprecated and thus should not be used. 19 | // +build js appengine safe disableunsafe 20 | 21 | package spew 22 | 23 | import "reflect" 24 | 25 | const ( 26 | // UnsafeDisabled is a build-time constant which specifies whether or 27 | // not access to the unsafe package is available. 28 | UnsafeDisabled = true 29 | ) 30 | 31 | // unsafeReflectValue typically converts the passed reflect.Value into a one 32 | // that bypasses the typical safety restrictions preventing access to 33 | // unaddressable and unexported data. However, doing this relies on access to 34 | // the unsafe package. This is a stub version which simply returns the passed 35 | // reflect.Value when the unsafe package is not available. 36 | func unsafeReflectValue(v reflect.Value) reflect.Value { 37 | return v 38 | } 39 | -------------------------------------------------------------------------------- /vendor/github.com/go-ini/ini/.gitignore: -------------------------------------------------------------------------------- 1 | testdata/conf_out.ini 2 | ini.sublime-project 3 | ini.sublime-workspace 4 | testdata/conf_reflect.ini 5 | .idea 6 | /.vscode 7 | -------------------------------------------------------------------------------- /vendor/github.com/go-ini/ini/.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: go 3 | go: 4 | - 1.5.x 5 | - 1.6.x 6 | - 1.7.x 7 | - 1.8.x 8 | - 1.9.x 9 | - 1.10.x 10 | 11 | script: 12 | - go get golang.org/x/tools/cmd/cover 13 | - go get github.com/smartystreets/goconvey 14 | - mkdir -p $HOME/gopath/src/gopkg.in 15 | - ln -s $HOME/gopath/src/github.com/go-ini/ini $HOME/gopath/src/gopkg.in/ini.v1 16 | - cd $HOME/gopath/src/gopkg.in/ini.v1 17 | - go test -v -cover -race 18 | -------------------------------------------------------------------------------- /vendor/github.com/go-ini/ini/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build test bench vet coverage 2 | 3 | build: vet bench 4 | 5 | test: 6 | go test -v -cover -race 7 | 8 | bench: 9 | go test -v -cover -race -test.bench=. -test.benchmem 10 | 11 | vet: 12 | go vet 13 | 14 | coverage: 15 | go test -coverprofile=c.out && go tool cover -html=c.out && rm c.out 16 | -------------------------------------------------------------------------------- /vendor/github.com/go-ini/ini/README.md: -------------------------------------------------------------------------------- 1 | INI [![Build Status](https://travis-ci.org/go-ini/ini.svg?branch=master)](https://travis-ci.org/go-ini/ini) [![Sourcegraph](https://sourcegraph.com/github.com/go-ini/ini/-/badge.svg)](https://sourcegraph.com/github.com/go-ini/ini?badge) 2 | === 3 | 4 | ![](https://avatars0.githubusercontent.com/u/10216035?v=3&s=200) 5 | 6 | Package ini provides INI file read and write functionality in Go. 7 | 8 | ## Features 9 | 10 | - Load from multiple data sources(`[]byte`, file and `io.ReadCloser`) with overwrites. 11 | - Read with recursion values. 12 | - Read with parent-child sections. 13 | - Read with auto-increment key names. 14 | - Read with multiple-line values. 15 | - Read with tons of helper methods. 16 | - Read and convert values to Go types. 17 | - Read and **WRITE** comments of sections and keys. 18 | - Manipulate sections, keys and comments with ease. 19 | - Keep sections and keys in order as you parse and save. 20 | 21 | ## Installation 22 | 23 | To use a tagged revision: 24 | 25 | ```sh 26 | $ go get gopkg.in/ini.v1 27 | ``` 28 | 29 | To use with latest changes: 30 | 31 | ```sh 32 | $ go get github.com/go-ini/ini 33 | ``` 34 | 35 | Please add `-u` flag to update in the future. 36 | 37 | ## Getting Help 38 | 39 | - [Getting Started](https://ini.unknwon.io/docs/intro/getting_started) 40 | - [API Documentation](https://gowalker.org/gopkg.in/ini.v1) 41 | 42 | ## License 43 | 44 | This project is under Apache v2 License. See the [LICENSE](LICENSE) file for the full license text. 45 | -------------------------------------------------------------------------------- /vendor/github.com/go-ini/ini/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Unknwon 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package ini 16 | 17 | import ( 18 | "fmt" 19 | ) 20 | 21 | type ErrDelimiterNotFound struct { 22 | Line string 23 | } 24 | 25 | func IsErrDelimiterNotFound(err error) bool { 26 | _, ok := err.(ErrDelimiterNotFound) 27 | return ok 28 | } 29 | 30 | func (err ErrDelimiterNotFound) Error() string { 31 | return fmt.Sprintf("key-value delimiter not found: %s", err.Line) 32 | } 33 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.test 3 | validator 4 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: go 3 | 4 | os: 5 | - linux 6 | 7 | env: 8 | - ARCH=x86_64 9 | - ARCH=i686 10 | 11 | go: 12 | - 1.7.4 13 | - 1.8.x 14 | - 1.9.x 15 | - tip 16 | 17 | matrix: 18 | fast_finish: true 19 | allow_failures: 20 | - go: tip 21 | 22 | addons: 23 | apt: 24 | packages: 25 | - devscripts 26 | 27 | script: 28 | - diff -au <(gofmt -d .) <(printf "") 29 | - diff -au <(licensecheck --check '.go$' --recursive --lines 0 * | grep -v -w 'Apache (v2.0)') <(printf "") 30 | - make 31 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | ### Developer Guidelines 3 | 4 | ``minio-go`` welcomes your contribution. To make the process as seamless as possible, we ask for the following: 5 | 6 | * Go ahead and fork the project and make your changes. We encourage pull requests to discuss code changes. 7 | - Fork it 8 | - Create your feature branch (git checkout -b my-new-feature) 9 | - Commit your changes (git commit -am 'Add some feature') 10 | - Push to the branch (git push origin my-new-feature) 11 | - Create new Pull Request 12 | 13 | * When you're ready to create a pull request, be sure to: 14 | - Have test cases for the new code. If you have questions about how to do it, please ask in your pull request. 15 | - Run `go fmt` 16 | - Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request. 17 | - Make sure `go test -race ./...` and `go build` completes. 18 | NOTE: go test runs functional tests and requires you to have a AWS S3 account. Set them as environment variables 19 | ``ACCESS_KEY`` and ``SECRET_KEY``. To run shorter version of the tests please use ``go test -short -race ./...`` 20 | 21 | * Read [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project 22 | - `minio-go` project is strictly conformant with Golang style 23 | - if you happen to observe offending code, please feel free to send a pull request 24 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # For maintainers only 2 | 3 | ## Responsibilities 4 | 5 | Please go through this link [Maintainer Responsibility](https://gist.github.com/abperiasamy/f4d9b31d3186bbd26522) 6 | 7 | ### Making new releases 8 | Tag and sign your release commit, additionally this step requires you to have access to Minio's trusted private key. 9 | ```sh 10 | $ export GNUPGHOME=/media/${USER}/minio/trusted 11 | $ git tag -s 4.0.0 12 | $ git push 13 | $ git push --tags 14 | ``` 15 | 16 | ### Update version 17 | Once release has been made update `libraryVersion` constant in `api.go` to next to be released version. 18 | 19 | ```sh 20 | $ grep libraryVersion api.go 21 | libraryVersion = "4.0.1" 22 | ``` 23 | 24 | Commit your changes 25 | ``` 26 | $ git commit -a -m "Update version for next release" --author "Minio Trusted " 27 | ``` 28 | 29 | ### Announce 30 | Announce new release by adding release notes at https://github.com/minio/minio-go/releases from `trusted@minio.io` account. Release notes requires two sections `highlights` and `changelog`. Highlights is a bulleted list of salient features in this release and Changelog contains list of all commits since the last release. 31 | 32 | To generate `changelog` 33 | ```sh 34 | $ git log --no-color --pretty=format:'-%d %s (%cr) <%an>' .. 35 | ``` 36 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/Makefile: -------------------------------------------------------------------------------- 1 | all: checks 2 | 3 | checks: 4 | @go get -t ./... 5 | @go vet ./... 6 | @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... 7 | @go get github.com/dustin/go-humanize/... 8 | @go get github.com/sirupsen/logrus/... 9 | @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go 10 | @mkdir -p /tmp/examples && for i in $(echo examples/s3/*); do go build -o /tmp/examples/$(basename ${i:0:-3}) ${i}; done 11 | @go get -u github.com/a8m/mark/... 12 | @go get -u github.com/minio/cli/... 13 | @go get -u golang.org/x/tools/cmd/goimports 14 | @go get -u github.com/gernest/wow/... 15 | @go build docs/validator.go && ./validator -m docs/API.md -t docs/checker.go.tpl 16 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/NOTICE: -------------------------------------------------------------------------------- 1 | minio-go 2 | Copyright 2015-2017 Minio, Inc. -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/api-datatypes.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2015-2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | import ( 21 | "net/http" 22 | "time" 23 | ) 24 | 25 | // BucketInfo container for bucket metadata. 26 | type BucketInfo struct { 27 | // The name of the bucket. 28 | Name string `json:"name"` 29 | // Date the bucket was created. 30 | CreationDate time.Time `json:"creationDate"` 31 | } 32 | 33 | // ObjectInfo container for object metadata. 34 | type ObjectInfo struct { 35 | // An ETag is optionally set to md5sum of an object. In case of multipart objects, 36 | // ETag is of the form MD5SUM-N where MD5SUM is md5sum of all individual md5sums of 37 | // each parts concatenated into one string. 38 | ETag string `json:"etag"` 39 | 40 | Key string `json:"name"` // Name of the object 41 | LastModified time.Time `json:"lastModified"` // Date and time the object was last modified. 42 | Size int64 `json:"size"` // Size in bytes of the object. 43 | ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data. 44 | 45 | // Collection of additional metadata on the object. 46 | // eg: x-amz-meta-*, content-encoding etc. 47 | Metadata http.Header `json:"metadata" xml:"-"` 48 | 49 | // Owner name. 50 | Owner struct { 51 | DisplayName string `json:"name"` 52 | ID string `json:"id"` 53 | } `json:"owner"` 54 | 55 | // The class of storage used to store the object. 56 | StorageClass string `json:"storageClass"` 57 | 58 | // Error 59 | Err error `json:"-"` 60 | } 61 | 62 | // ObjectMultipartInfo container for multipart object metadata. 63 | type ObjectMultipartInfo struct { 64 | // Date and time at which the multipart upload was initiated. 65 | Initiated time.Time `type:"timestamp" timestampFormat:"iso8601"` 66 | 67 | Initiator initiator 68 | Owner owner 69 | 70 | // The type of storage to use for the object. Defaults to 'STANDARD'. 71 | StorageClass string 72 | 73 | // Key of the object for which the multipart upload was initiated. 74 | Key string 75 | 76 | // Size in bytes of the object. 77 | Size int64 78 | 79 | // Upload ID that identifies the multipart upload. 80 | UploadID string `xml:"UploadId"` 81 | 82 | // Error 83 | Err error 84 | } 85 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/api-get-object-context.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | import "context" 21 | 22 | // GetObjectWithContext - returns an seekable, readable object. 23 | // The options can be used to specify the GET request further. 24 | func (c Client) GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) { 25 | return c.getObjectWithContext(ctx, bucketName, objectName, opts) 26 | } 27 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/api-get-options.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2015-2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | import ( 21 | "fmt" 22 | "net/http" 23 | "time" 24 | 25 | "github.com/minio/minio-go/pkg/encrypt" 26 | ) 27 | 28 | // GetObjectOptions are used to specify additional headers or options 29 | // during GET requests. 30 | type GetObjectOptions struct { 31 | headers map[string]string 32 | 33 | Materials encrypt.Materials 34 | } 35 | 36 | // StatObjectOptions are used to specify additional headers or options 37 | // during GET info/stat requests. 38 | type StatObjectOptions struct { 39 | GetObjectOptions 40 | } 41 | 42 | // Header returns the http.Header representation of the GET options. 43 | func (o GetObjectOptions) Header() http.Header { 44 | headers := make(http.Header, len(o.headers)) 45 | for k, v := range o.headers { 46 | headers.Set(k, v) 47 | } 48 | return headers 49 | } 50 | 51 | // Set adds a key value pair to the options. The 52 | // key-value pair will be part of the HTTP GET request 53 | // headers. 54 | func (o *GetObjectOptions) Set(key, value string) { 55 | if o.headers == nil { 56 | o.headers = make(map[string]string) 57 | } 58 | o.headers[http.CanonicalHeaderKey(key)] = value 59 | } 60 | 61 | // SetMatchETag - set match etag. 62 | func (o *GetObjectOptions) SetMatchETag(etag string) error { 63 | if etag == "" { 64 | return ErrInvalidArgument("ETag cannot be empty.") 65 | } 66 | o.Set("If-Match", "\""+etag+"\"") 67 | return nil 68 | } 69 | 70 | // SetMatchETagExcept - set match etag except. 71 | func (o *GetObjectOptions) SetMatchETagExcept(etag string) error { 72 | if etag == "" { 73 | return ErrInvalidArgument("ETag cannot be empty.") 74 | } 75 | o.Set("If-None-Match", "\""+etag+"\"") 76 | return nil 77 | } 78 | 79 | // SetUnmodified - set unmodified time since. 80 | func (o *GetObjectOptions) SetUnmodified(modTime time.Time) error { 81 | if modTime.IsZero() { 82 | return ErrInvalidArgument("Modified since cannot be empty.") 83 | } 84 | o.Set("If-Unmodified-Since", modTime.Format(http.TimeFormat)) 85 | return nil 86 | } 87 | 88 | // SetModified - set modified time since. 89 | func (o *GetObjectOptions) SetModified(modTime time.Time) error { 90 | if modTime.IsZero() { 91 | return ErrInvalidArgument("Modified since cannot be empty.") 92 | } 93 | o.Set("If-Modified-Since", modTime.Format(http.TimeFormat)) 94 | return nil 95 | } 96 | 97 | // SetRange - set the start and end offset of the object to be read. 98 | // See https://tools.ietf.org/html/rfc7233#section-3.1 for reference. 99 | func (o *GetObjectOptions) SetRange(start, end int64) error { 100 | switch { 101 | case start == 0 && end < 0: 102 | // Read last '-end' bytes. `bytes=-N`. 103 | o.Set("Range", fmt.Sprintf("bytes=%d", end)) 104 | case 0 < start && end == 0: 105 | // Read everything starting from offset 106 | // 'start'. `bytes=N-`. 107 | o.Set("Range", fmt.Sprintf("bytes=%d-", start)) 108 | case 0 <= start && start <= end: 109 | // Read everything starting at 'start' till the 110 | // 'end'. `bytes=N-M` 111 | o.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) 112 | default: 113 | // All other cases such as 114 | // bytes=-3- 115 | // bytes=5-3 116 | // bytes=-2-4 117 | // bytes=-3-0 118 | // bytes=-3--2 119 | // are invalid. 120 | return ErrInvalidArgument( 121 | fmt.Sprintf( 122 | "Invalid range specified: start=%d end=%d", 123 | start, end)) 124 | } 125 | return nil 126 | } 127 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/api-get-policy.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2015-2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | import ( 21 | "context" 22 | "encoding/json" 23 | "io/ioutil" 24 | "net/http" 25 | "net/url" 26 | 27 | "github.com/minio/minio-go/pkg/policy" 28 | "github.com/minio/minio-go/pkg/s3utils" 29 | ) 30 | 31 | // GetBucketPolicy - get bucket policy at a given path. 32 | func (c Client) GetBucketPolicy(bucketName, objectPrefix string) (bucketPolicy policy.BucketPolicy, err error) { 33 | // Input validation. 34 | if err := s3utils.CheckValidBucketName(bucketName); err != nil { 35 | return policy.BucketPolicyNone, err 36 | } 37 | if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { 38 | return policy.BucketPolicyNone, err 39 | } 40 | policyInfo, err := c.getBucketPolicy(bucketName) 41 | if err != nil { 42 | errResponse := ToErrorResponse(err) 43 | if errResponse.Code == "NoSuchBucketPolicy" { 44 | return policy.BucketPolicyNone, nil 45 | } 46 | return policy.BucketPolicyNone, err 47 | } 48 | return policy.GetPolicy(policyInfo.Statements, bucketName, objectPrefix), nil 49 | } 50 | 51 | // ListBucketPolicies - list all policies for a given prefix and all its children. 52 | func (c Client) ListBucketPolicies(bucketName, objectPrefix string) (bucketPolicies map[string]policy.BucketPolicy, err error) { 53 | // Input validation. 54 | if err := s3utils.CheckValidBucketName(bucketName); err != nil { 55 | return map[string]policy.BucketPolicy{}, err 56 | } 57 | if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { 58 | return map[string]policy.BucketPolicy{}, err 59 | } 60 | policyInfo, err := c.getBucketPolicy(bucketName) 61 | if err != nil { 62 | errResponse := ToErrorResponse(err) 63 | if errResponse.Code == "NoSuchBucketPolicy" { 64 | return map[string]policy.BucketPolicy{}, nil 65 | } 66 | return map[string]policy.BucketPolicy{}, err 67 | } 68 | return policy.GetPolicies(policyInfo.Statements, bucketName, objectPrefix), nil 69 | } 70 | 71 | // Default empty bucket access policy. 72 | var emptyBucketAccessPolicy = policy.BucketAccessPolicy{ 73 | Version: "2012-10-17", 74 | } 75 | 76 | // Request server for current bucket policy. 77 | func (c Client) getBucketPolicy(bucketName string) (policy.BucketAccessPolicy, error) { 78 | // Get resources properly escaped and lined up before 79 | // using them in http request. 80 | urlValues := make(url.Values) 81 | urlValues.Set("policy", "") 82 | 83 | // Execute GET on bucket to list objects. 84 | resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ 85 | bucketName: bucketName, 86 | queryValues: urlValues, 87 | contentSHA256Hex: emptySHA256Hex, 88 | }) 89 | 90 | defer closeResponse(resp) 91 | if err != nil { 92 | return emptyBucketAccessPolicy, err 93 | } 94 | 95 | if resp != nil { 96 | if resp.StatusCode != http.StatusOK { 97 | return emptyBucketAccessPolicy, httpRespToErrorResponse(resp, bucketName, "") 98 | } 99 | } 100 | 101 | bucketPolicyBuf, err := ioutil.ReadAll(resp.Body) 102 | if err != nil { 103 | return emptyBucketAccessPolicy, err 104 | } 105 | 106 | policy := policy.BucketAccessPolicy{} 107 | err = json.Unmarshal(bucketPolicyBuf, &policy) 108 | return policy, err 109 | } 110 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/api-put-object-common.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2015-2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | import ( 21 | "context" 22 | "io" 23 | "math" 24 | "os" 25 | 26 | "github.com/minio/minio-go/pkg/s3utils" 27 | ) 28 | 29 | // Verify if reader is *minio.Object 30 | func isObject(reader io.Reader) (ok bool) { 31 | _, ok = reader.(*Object) 32 | return 33 | } 34 | 35 | // Verify if reader is a generic ReaderAt 36 | func isReadAt(reader io.Reader) (ok bool) { 37 | _, ok = reader.(io.ReaderAt) 38 | if ok { 39 | var v *os.File 40 | v, ok = reader.(*os.File) 41 | if ok { 42 | // Stdin, Stdout and Stderr all have *os.File type 43 | // which happen to also be io.ReaderAt compatible 44 | // we need to add special conditions for them to 45 | // be ignored by this function. 46 | for _, f := range []string{ 47 | "/dev/stdin", 48 | "/dev/stdout", 49 | "/dev/stderr", 50 | } { 51 | if f == v.Name() { 52 | ok = false 53 | break 54 | } 55 | } 56 | } 57 | } 58 | return 59 | } 60 | 61 | // optimalPartInfo - calculate the optimal part info for a given 62 | // object size. 63 | // 64 | // NOTE: Assumption here is that for any object to be uploaded to any S3 compatible 65 | // object storage it will have the following parameters as constants. 66 | // 67 | // maxPartsCount - 10000 68 | // minPartSize - 64MiB 69 | // maxMultipartPutObjectSize - 5TiB 70 | // 71 | func optimalPartInfo(objectSize int64) (totalPartsCount int, partSize int64, lastPartSize int64, err error) { 72 | // object size is '-1' set it to 5TiB. 73 | if objectSize == -1 { 74 | objectSize = maxMultipartPutObjectSize 75 | } 76 | // object size is larger than supported maximum. 77 | if objectSize > maxMultipartPutObjectSize { 78 | err = ErrEntityTooLarge(objectSize, maxMultipartPutObjectSize, "", "") 79 | return 80 | } 81 | // Use floats for part size for all calculations to avoid 82 | // overflows during float64 to int64 conversions. 83 | partSizeFlt := math.Ceil(float64(objectSize / maxPartsCount)) 84 | partSizeFlt = math.Ceil(partSizeFlt/minPartSize) * minPartSize 85 | // Total parts count. 86 | totalPartsCount = int(math.Ceil(float64(objectSize) / partSizeFlt)) 87 | // Part size. 88 | partSize = int64(partSizeFlt) 89 | // Last part size. 90 | lastPartSize = objectSize - int64(totalPartsCount-1)*partSize 91 | return totalPartsCount, partSize, lastPartSize, nil 92 | } 93 | 94 | // getUploadID - fetch upload id if already present for an object name 95 | // or initiate a new request to fetch a new upload id. 96 | func (c Client) newUploadID(ctx context.Context, bucketName, objectName string, opts PutObjectOptions) (uploadID string, err error) { 97 | // Input validation. 98 | if err := s3utils.CheckValidBucketName(bucketName); err != nil { 99 | return "", err 100 | } 101 | if err := s3utils.CheckValidObjectName(objectName); err != nil { 102 | return "", err 103 | } 104 | 105 | // Initiate multipart upload for an object. 106 | initMultipartUploadResult, err := c.initiateMultipartUpload(ctx, bucketName, objectName, opts) 107 | if err != nil { 108 | return "", err 109 | } 110 | return initMultipartUploadResult.UploadID, nil 111 | } 112 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/api-put-object-context.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | import ( 21 | "context" 22 | "io" 23 | ) 24 | 25 | // PutObjectWithContext - Identical to PutObject call, but accepts context to facilitate request cancellation. 26 | func (c Client) PutObjectWithContext(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, 27 | opts PutObjectOptions) (n int64, err error) { 28 | err = opts.validate() 29 | if err != nil { 30 | return 0, err 31 | } 32 | if opts.EncryptMaterials != nil { 33 | if err = opts.EncryptMaterials.SetupEncryptMode(reader); err != nil { 34 | return 0, err 35 | } 36 | return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, opts.EncryptMaterials, opts) 37 | } 38 | return c.putObjectCommon(ctx, bucketName, objectName, reader, objectSize, opts) 39 | } 40 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/api-put-object-copy.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | // CopyObject - copy a source object into a new object 21 | func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error { 22 | return c.ComposeObject(dst, []SourceInfo{src}) 23 | } 24 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/api-put-object-encrypted.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | import ( 21 | "context" 22 | "io" 23 | 24 | "github.com/minio/minio-go/pkg/encrypt" 25 | ) 26 | 27 | // PutEncryptedObject - Encrypt and store object. 28 | func (c Client) PutEncryptedObject(bucketName, objectName string, reader io.Reader, encryptMaterials encrypt.Materials) (n int64, err error) { 29 | 30 | if encryptMaterials == nil { 31 | return 0, ErrInvalidArgument("Unable to recognize empty encryption properties") 32 | } 33 | 34 | if err := encryptMaterials.SetupEncryptMode(reader); err != nil { 35 | return 0, err 36 | } 37 | 38 | return c.PutObjectWithContext(context.Background(), bucketName, objectName, reader, -1, PutObjectOptions{EncryptMaterials: encryptMaterials}) 39 | } 40 | 41 | // FPutEncryptedObject - Encrypt and store an object with contents from file at filePath. 42 | func (c Client) FPutEncryptedObject(bucketName, objectName, filePath string, encryptMaterials encrypt.Materials) (n int64, err error) { 43 | return c.FPutObjectWithContext(context.Background(), bucketName, objectName, filePath, PutObjectOptions{EncryptMaterials: encryptMaterials}) 44 | } 45 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/api-put-object-file-context.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | import ( 21 | "context" 22 | "mime" 23 | "os" 24 | "path/filepath" 25 | 26 | "github.com/minio/minio-go/pkg/s3utils" 27 | ) 28 | 29 | // FPutObjectWithContext - Create an object in a bucket, with contents from file at filePath. Allows request cancellation. 30 | func (c Client) FPutObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts PutObjectOptions) (n int64, err error) { 31 | // Input validation. 32 | if err := s3utils.CheckValidBucketName(bucketName); err != nil { 33 | return 0, err 34 | } 35 | if err := s3utils.CheckValidObjectName(objectName); err != nil { 36 | return 0, err 37 | } 38 | 39 | // Open the referenced file. 40 | fileReader, err := os.Open(filePath) 41 | // If any error fail quickly here. 42 | if err != nil { 43 | return 0, err 44 | } 45 | defer fileReader.Close() 46 | 47 | // Save the file stat. 48 | fileStat, err := fileReader.Stat() 49 | if err != nil { 50 | return 0, err 51 | } 52 | 53 | // Save the file size. 54 | fileSize := fileStat.Size() 55 | 56 | // Set contentType based on filepath extension if not given or default 57 | // value of "application/octet-stream" if the extension has no associated type. 58 | if opts.ContentType == "" { 59 | if opts.ContentType = mime.TypeByExtension(filepath.Ext(filePath)); opts.ContentType == "" { 60 | opts.ContentType = "application/octet-stream" 61 | } 62 | } 63 | return c.PutObjectWithContext(ctx, bucketName, objectName, fileReader, fileSize, opts) 64 | } 65 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/api-put-object-file.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2015-2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | import ( 21 | "context" 22 | ) 23 | 24 | // FPutObject - Create an object in a bucket, with contents from file at filePath 25 | func (c Client) FPutObject(bucketName, objectName, filePath string, opts PutObjectOptions) (n int64, err error) { 26 | return c.FPutObjectWithContext(context.Background(), bucketName, objectName, filePath, opts) 27 | } 28 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/appveyor.yml: -------------------------------------------------------------------------------- 1 | # version format 2 | version: "{build}" 3 | 4 | # Operating system (build VM template) 5 | os: Windows Server 2012 R2 6 | 7 | clone_folder: c:\gopath\src\github.com\minio\minio-go 8 | 9 | # environment variables 10 | environment: 11 | GOPATH: c:\gopath 12 | GO15VENDOREXPERIMENT: 1 13 | 14 | # scripts that run after cloning repository 15 | install: 16 | - set PATH=%GOPATH%\bin;c:\go\bin;%PATH% 17 | - go version 18 | - go env 19 | - go get -u github.com/golang/lint/golint 20 | - go get -u github.com/remyoudompheng/go-misc/deadcode 21 | - go get -u github.com/gordonklaus/ineffassign 22 | - go get -t ./... 23 | 24 | # to run your custom scripts instead of automatic MSBuild 25 | build_script: 26 | - go vet ./... 27 | - gofmt -s -l . 28 | - golint -set_exit_status github.com/minio/minio-go... 29 | - deadcode 30 | - ineffassign . 31 | - go test -short -v 32 | - go test -short -race -v 33 | 34 | # to disable automatic tests 35 | test: off 36 | 37 | # to disable deployment 38 | deploy: off 39 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/constants.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2015-2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | /// Multipart upload defaults. 21 | 22 | // absMinPartSize - absolute minimum part size (5 MiB) below which 23 | // a part in a multipart upload may not be uploaded. 24 | const absMinPartSize = 1024 * 1024 * 5 25 | 26 | // minPartSize - minimum part size 64MiB per object after which 27 | // putObject behaves internally as multipart. 28 | const minPartSize = 1024 * 1024 * 64 29 | 30 | // copyPartSize - default (and maximum) part size to copy in a 31 | // copy-object request (5GiB) 32 | const copyPartSize = 1024 * 1024 * 1024 * 5 33 | 34 | // maxPartsCount - maximum number of parts for a single multipart session. 35 | const maxPartsCount = 10000 36 | 37 | // maxPartSize - maximum part size 5GiB for a single multipart upload 38 | // operation. 39 | const maxPartSize = 1024 * 1024 * 1024 * 5 40 | 41 | // maxSinglePutObjectSize - maximum size 5GiB of object per PUT 42 | // operation. 43 | const maxSinglePutObjectSize = 1024 * 1024 * 1024 * 5 44 | 45 | // maxMultipartPutObjectSize - maximum size 5TiB of object for 46 | // Multipart operation. 47 | const maxMultipartPutObjectSize = 1024 * 1024 * 1024 * 1024 * 5 48 | 49 | // unsignedPayload - value to be set to X-Amz-Content-Sha256 header when 50 | // we don't want to sign the request payload 51 | const unsignedPayload = "UNSIGNED-PAYLOAD" 52 | 53 | // Total number of parallel workers used for multipart operation. 54 | const totalWorkers = 4 55 | 56 | // Signature related constants. 57 | const ( 58 | signV4Algorithm = "AWS4-HMAC-SHA256" 59 | iso8601DateFormat = "20060102T150405Z" 60 | ) 61 | 62 | // Encryption headers stored along with the object. 63 | const ( 64 | amzHeaderIV = "X-Amz-Meta-X-Amz-Iv" 65 | amzHeaderKey = "X-Amz-Meta-X-Amz-Key" 66 | amzHeaderMatDesc = "X-Amz-Meta-X-Amz-Matdesc" 67 | ) 68 | 69 | // Storage class header constant. 70 | const amzStorageClass = "X-Amz-Storage-Class" 71 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/hook-reader.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2015-2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | import "io" 21 | 22 | // hookReader hooks additional reader in the source stream. It is 23 | // useful for making progress bars. Second reader is appropriately 24 | // notified about the exact number of bytes read from the primary 25 | // source on each Read operation. 26 | type hookReader struct { 27 | source io.Reader 28 | hook io.Reader 29 | } 30 | 31 | // Seek implements io.Seeker. Seeks source first, and if necessary 32 | // seeks hook if Seek method is appropriately found. 33 | func (hr *hookReader) Seek(offset int64, whence int) (n int64, err error) { 34 | // Verify for source has embedded Seeker, use it. 35 | sourceSeeker, ok := hr.source.(io.Seeker) 36 | if ok { 37 | return sourceSeeker.Seek(offset, whence) 38 | } 39 | // Verify if hook has embedded Seeker, use it. 40 | hookSeeker, ok := hr.hook.(io.Seeker) 41 | if ok { 42 | return hookSeeker.Seek(offset, whence) 43 | } 44 | return n, nil 45 | } 46 | 47 | // Read implements io.Reader. Always reads from the source, the return 48 | // value 'n' number of bytes are reported through the hook. Returns 49 | // error for all non io.EOF conditions. 50 | func (hr *hookReader) Read(b []byte) (n int, err error) { 51 | n, err = hr.source.Read(b) 52 | if err != nil && err != io.EOF { 53 | return n, err 54 | } 55 | // Progress the hook with the total read bytes from the source. 56 | if _, herr := hr.hook.Read(b[:n]); herr != nil { 57 | if herr != io.EOF { 58 | return n, herr 59 | } 60 | } 61 | return n, err 62 | } 63 | 64 | // newHook returns a io.ReadSeeker which implements hookReader that 65 | // reports the data read from the source to the hook. 66 | func newHook(source, hook io.Reader) io.Reader { 67 | if hook == nil { 68 | return source 69 | } 70 | return &hookReader{source, hook} 71 | } 72 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/pkg/credentials/chain.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package credentials 19 | 20 | // A Chain will search for a provider which returns credentials 21 | // and cache that provider until Retrieve is called again. 22 | // 23 | // The Chain provides a way of chaining multiple providers together 24 | // which will pick the first available using priority order of the 25 | // Providers in the list. 26 | // 27 | // If none of the Providers retrieve valid credentials Value, ChainProvider's 28 | // Retrieve() will return the no credentials value. 29 | // 30 | // If a Provider is found which returns valid credentials Value ChainProvider 31 | // will cache that Provider for all calls to IsExpired(), until Retrieve is 32 | // called again after IsExpired() is true. 33 | // 34 | // creds := credentials.NewChainCredentials( 35 | // []credentials.Provider{ 36 | // &credentials.EnvAWSS3{}, 37 | // &credentials.EnvMinio{}, 38 | // }) 39 | // 40 | // // Usage of ChainCredentials. 41 | // mc, err := minio.NewWithCredentials(endpoint, creds, secure, "us-east-1") 42 | // if err != nil { 43 | // log.Fatalln(err) 44 | // } 45 | // 46 | type Chain struct { 47 | Providers []Provider 48 | curr Provider 49 | } 50 | 51 | // NewChainCredentials returns a pointer to a new Credentials object 52 | // wrapping a chain of providers. 53 | func NewChainCredentials(providers []Provider) *Credentials { 54 | return New(&Chain{ 55 | Providers: append([]Provider{}, providers...), 56 | }) 57 | } 58 | 59 | // Retrieve returns the credentials value, returns no credentials(anonymous) 60 | // if no credentials provider returned any value. 61 | // 62 | // If a provider is found with credentials, it will be cached and any calls 63 | // to IsExpired() will return the expired state of the cached provider. 64 | func (c *Chain) Retrieve() (Value, error) { 65 | for _, p := range c.Providers { 66 | creds, _ := p.Retrieve() 67 | // Always prioritize non-anonymous providers, if any. 68 | if creds.AccessKeyID == "" && creds.SecretAccessKey == "" { 69 | continue 70 | } 71 | c.curr = p 72 | return creds, nil 73 | } 74 | // At this point we have exhausted all the providers and 75 | // are left without any credentials return anonymous. 76 | return Value{ 77 | SignerType: SignatureAnonymous, 78 | }, nil 79 | } 80 | 81 | // IsExpired will returned the expired state of the currently cached provider 82 | // if there is one. If there is no current provider, true will be returned. 83 | func (c *Chain) IsExpired() bool { 84 | if c.curr != nil { 85 | return c.curr.IsExpired() 86 | } 87 | 88 | return true 89 | } 90 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/pkg/credentials/config.json.sample: -------------------------------------------------------------------------------- 1 | { 2 | "version": "8", 3 | "hosts": { 4 | "play": { 5 | "url": "https://play.minio.io:9000", 6 | "accessKey": "Q3AM3UQ867SPQQA43P2F", 7 | "secretKey": "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", 8 | "api": "S3v2" 9 | }, 10 | "s3": { 11 | "url": "https://s3.amazonaws.com", 12 | "accessKey": "accessKey", 13 | "secretKey": "secret", 14 | "api": "S3v4" 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/pkg/credentials/credentials.sample: -------------------------------------------------------------------------------- 1 | [default] 2 | aws_access_key_id = accessKey 3 | aws_secret_access_key = secret 4 | aws_session_token = token 5 | 6 | [no_token] 7 | aws_access_key_id = accessKey 8 | aws_secret_access_key = secret 9 | 10 | [with_colon] 11 | aws_access_key_id: accessKey 12 | aws_secret_access_key: secret 13 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/pkg/credentials/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | // Package credentials provides credential retrieval and management 19 | // for S3 compatible object storage. 20 | // 21 | // By default the Credentials.Get() will cache the successful result of a 22 | // Provider's Retrieve() until Provider.IsExpired() returns true. At which 23 | // point Credentials will call Provider's Retrieve() to get new credential Value. 24 | // 25 | // The Provider is responsible for determining when credentials have expired. 26 | // It is also important to note that Credentials will always call Retrieve the 27 | // first time Credentials.Get() is called. 28 | // 29 | // Example of using the environment variable credentials. 30 | // 31 | // creds := NewFromEnv() 32 | // // Retrieve the credentials value 33 | // credValue, err := creds.Get() 34 | // if err != nil { 35 | // // handle error 36 | // } 37 | // 38 | // Example of forcing credentials to expire and be refreshed on the next Get(). 39 | // This may be helpful to proactively expire credentials and refresh them sooner 40 | // than they would naturally expire on their own. 41 | // 42 | // creds := NewFromIAM("") 43 | // creds.Expire() 44 | // credsValue, err := creds.Get() 45 | // // New credentials will be retrieved instead of from cache. 46 | // 47 | // 48 | // Custom Provider 49 | // 50 | // Each Provider built into this package also provides a helper method to generate 51 | // a Credentials pointer setup with the provider. To use a custom Provider just 52 | // create a type which satisfies the Provider interface and pass it to the 53 | // NewCredentials method. 54 | // 55 | // type MyProvider struct{} 56 | // func (m *MyProvider) Retrieve() (Value, error) {...} 57 | // func (m *MyProvider) IsExpired() bool {...} 58 | // 59 | // creds := NewCredentials(&MyProvider{}) 60 | // credValue, err := creds.Get() 61 | // 62 | package credentials 63 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/pkg/credentials/env_aws.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package credentials 19 | 20 | import "os" 21 | 22 | // A EnvAWS retrieves credentials from the environment variables of the 23 | // running process. EnvAWSironment credentials never expire. 24 | // 25 | // EnvAWSironment variables used: 26 | // 27 | // * Access Key ID: AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY. 28 | // * Secret Access Key: AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY. 29 | // * Secret Token: AWS_SESSION_TOKEN. 30 | type EnvAWS struct { 31 | retrieved bool 32 | } 33 | 34 | // NewEnvAWS returns a pointer to a new Credentials object 35 | // wrapping the environment variable provider. 36 | func NewEnvAWS() *Credentials { 37 | return New(&EnvAWS{}) 38 | } 39 | 40 | // Retrieve retrieves the keys from the environment. 41 | func (e *EnvAWS) Retrieve() (Value, error) { 42 | e.retrieved = false 43 | 44 | id := os.Getenv("AWS_ACCESS_KEY_ID") 45 | if id == "" { 46 | id = os.Getenv("AWS_ACCESS_KEY") 47 | } 48 | 49 | secret := os.Getenv("AWS_SECRET_ACCESS_KEY") 50 | if secret == "" { 51 | secret = os.Getenv("AWS_SECRET_KEY") 52 | } 53 | 54 | signerType := SignatureV4 55 | if id == "" || secret == "" { 56 | signerType = SignatureAnonymous 57 | } 58 | 59 | e.retrieved = true 60 | return Value{ 61 | AccessKeyID: id, 62 | SecretAccessKey: secret, 63 | SessionToken: os.Getenv("AWS_SESSION_TOKEN"), 64 | SignerType: signerType, 65 | }, nil 66 | } 67 | 68 | // IsExpired returns if the credentials have been retrieved. 69 | func (e *EnvAWS) IsExpired() bool { 70 | return !e.retrieved 71 | } 72 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/pkg/credentials/env_minio.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package credentials 19 | 20 | import "os" 21 | 22 | // A EnvMinio retrieves credentials from the environment variables of the 23 | // running process. EnvMinioironment credentials never expire. 24 | // 25 | // EnvMinioironment variables used: 26 | // 27 | // * Access Key ID: MINIO_ACCESS_KEY. 28 | // * Secret Access Key: MINIO_SECRET_KEY. 29 | type EnvMinio struct { 30 | retrieved bool 31 | } 32 | 33 | // NewEnvMinio returns a pointer to a new Credentials object 34 | // wrapping the environment variable provider. 35 | func NewEnvMinio() *Credentials { 36 | return New(&EnvMinio{}) 37 | } 38 | 39 | // Retrieve retrieves the keys from the environment. 40 | func (e *EnvMinio) Retrieve() (Value, error) { 41 | e.retrieved = false 42 | 43 | id := os.Getenv("MINIO_ACCESS_KEY") 44 | secret := os.Getenv("MINIO_SECRET_KEY") 45 | 46 | signerType := SignatureV4 47 | if id == "" || secret == "" { 48 | signerType = SignatureAnonymous 49 | } 50 | 51 | e.retrieved = true 52 | return Value{ 53 | AccessKeyID: id, 54 | SecretAccessKey: secret, 55 | SignerType: signerType, 56 | }, nil 57 | } 58 | 59 | // IsExpired returns if the credentials have been retrieved. 60 | func (e *EnvMinio) IsExpired() bool { 61 | return !e.retrieved 62 | } 63 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/pkg/credentials/signature-type.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package credentials 19 | 20 | import "strings" 21 | 22 | // SignatureType is type of Authorization requested for a given HTTP request. 23 | type SignatureType int 24 | 25 | // Different types of supported signatures - default is SignatureV4 or SignatureDefault. 26 | const ( 27 | // SignatureDefault is always set to v4. 28 | SignatureDefault SignatureType = iota 29 | SignatureV4 30 | SignatureV2 31 | SignatureV4Streaming 32 | SignatureAnonymous // Anonymous signature signifies, no signature. 33 | ) 34 | 35 | // IsV2 - is signature SignatureV2? 36 | func (s SignatureType) IsV2() bool { 37 | return s == SignatureV2 38 | } 39 | 40 | // IsV4 - is signature SignatureV4? 41 | func (s SignatureType) IsV4() bool { 42 | return s == SignatureV4 || s == SignatureDefault 43 | } 44 | 45 | // IsStreamingV4 - is signature SignatureV4Streaming? 46 | func (s SignatureType) IsStreamingV4() bool { 47 | return s == SignatureV4Streaming 48 | } 49 | 50 | // IsAnonymous - is signature empty? 51 | func (s SignatureType) IsAnonymous() bool { 52 | return s == SignatureAnonymous 53 | } 54 | 55 | // Stringer humanized version of signature type, 56 | // strings returned here are case insensitive. 57 | func (s SignatureType) String() string { 58 | if s.IsV2() { 59 | return "S3v2" 60 | } else if s.IsV4() { 61 | return "S3v4" 62 | } else if s.IsStreamingV4() { 63 | return "S3v4Streaming" 64 | } 65 | return "Anonymous" 66 | } 67 | 68 | func parseSignatureType(str string) SignatureType { 69 | if strings.EqualFold(str, "S3v4") { 70 | return SignatureV4 71 | } else if strings.EqualFold(str, "S3v2") { 72 | return SignatureV2 73 | } else if strings.EqualFold(str, "S3v4Streaming") { 74 | return SignatureV4Streaming 75 | } 76 | return SignatureAnonymous 77 | } 78 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/pkg/credentials/static.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package credentials 19 | 20 | // A Static is a set of credentials which are set programmatically, 21 | // and will never expire. 22 | type Static struct { 23 | Value 24 | } 25 | 26 | // NewStaticV2 returns a pointer to a new Credentials object 27 | // wrapping a static credentials value provider, signature is 28 | // set to v2. If access and secret are not specified then 29 | // regardless of signature type set it Value will return 30 | // as anonymous. 31 | func NewStaticV2(id, secret, token string) *Credentials { 32 | return NewStatic(id, secret, token, SignatureV2) 33 | } 34 | 35 | // NewStaticV4 is similar to NewStaticV2 with similar considerations. 36 | func NewStaticV4(id, secret, token string) *Credentials { 37 | return NewStatic(id, secret, token, SignatureV4) 38 | } 39 | 40 | // NewStatic returns a pointer to a new Credentials object 41 | // wrapping a static credentials value provider. 42 | func NewStatic(id, secret, token string, signerType SignatureType) *Credentials { 43 | return New(&Static{ 44 | Value: Value{ 45 | AccessKeyID: id, 46 | SecretAccessKey: secret, 47 | SessionToken: token, 48 | SignerType: signerType, 49 | }, 50 | }) 51 | } 52 | 53 | // Retrieve returns the static credentials. 54 | func (s *Static) Retrieve() (Value, error) { 55 | if s.AccessKeyID == "" || s.SecretAccessKey == "" { 56 | // Anonymous is not an error 57 | return Value{SignerType: SignatureAnonymous}, nil 58 | } 59 | return s.Value, nil 60 | } 61 | 62 | // IsExpired returns if the credentials are expired. 63 | // 64 | // For Static, the credentials never expired. 65 | func (s *Static) IsExpired() bool { 66 | return false 67 | } 68 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/pkg/encrypt/interface.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | // Package encrypt implements a generic interface to encrypt any stream of data. 19 | // currently this package implements two types of encryption 20 | // - Symmetric encryption using AES. 21 | // - Asymmetric encrytion using RSA. 22 | package encrypt 23 | 24 | import "io" 25 | 26 | // Materials - provides generic interface to encrypt any stream of data. 27 | type Materials interface { 28 | 29 | // Closes the wrapped stream properly, initiated by the caller. 30 | Close() error 31 | 32 | // Returns encrypted/decrypted data, io.Reader compatible. 33 | Read(b []byte) (int, error) 34 | 35 | // Get randomly generated IV, base64 encoded. 36 | GetIV() (iv string) 37 | 38 | // Get content encrypting key (cek) in encrypted form, base64 encoded. 39 | GetKey() (key string) 40 | 41 | // Get user provided encryption material description in 42 | // JSON (UTF8) format. This is not used, kept for future. 43 | GetDesc() (desc string) 44 | 45 | // Setup encrypt mode, further calls of Read() function 46 | // will return the encrypted form of data streamed 47 | // by the passed reader 48 | SetupEncryptMode(stream io.Reader) error 49 | 50 | // Setup decrypted mode, further calls of Read() function 51 | // will return the decrypted form of data streamed 52 | // by the passed reader 53 | SetupDecryptMode(stream io.Reader, iv string, key string) error 54 | } 55 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/pkg/policy/bucket-policy-condition.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2015-2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package policy 19 | 20 | import "github.com/minio/minio-go/pkg/set" 21 | 22 | // ConditionKeyMap - map of policy condition key and value. 23 | type ConditionKeyMap map[string]set.StringSet 24 | 25 | // Add - adds key and value. The value is appended If key already exists. 26 | func (ckm ConditionKeyMap) Add(key string, value set.StringSet) { 27 | if v, ok := ckm[key]; ok { 28 | ckm[key] = v.Union(value) 29 | } else { 30 | ckm[key] = set.CopyStringSet(value) 31 | } 32 | } 33 | 34 | // Remove - removes value of given key. If key has empty after removal, the key is also removed. 35 | func (ckm ConditionKeyMap) Remove(key string, value set.StringSet) { 36 | if v, ok := ckm[key]; ok { 37 | if value != nil { 38 | ckm[key] = v.Difference(value) 39 | } 40 | 41 | if ckm[key].IsEmpty() { 42 | delete(ckm, key) 43 | } 44 | } 45 | } 46 | 47 | // RemoveKey - removes key and its value. 48 | func (ckm ConditionKeyMap) RemoveKey(key string) { 49 | if _, ok := ckm[key]; ok { 50 | delete(ckm, key) 51 | } 52 | } 53 | 54 | // CopyConditionKeyMap - returns new copy of given ConditionKeyMap. 55 | func CopyConditionKeyMap(condKeyMap ConditionKeyMap) ConditionKeyMap { 56 | out := make(ConditionKeyMap) 57 | 58 | for k, v := range condKeyMap { 59 | out[k] = set.CopyStringSet(v) 60 | } 61 | 62 | return out 63 | } 64 | 65 | // mergeConditionKeyMap - returns a new ConditionKeyMap which contains merged key/value of given two ConditionKeyMap. 66 | func mergeConditionKeyMap(condKeyMap1 ConditionKeyMap, condKeyMap2 ConditionKeyMap) ConditionKeyMap { 67 | out := CopyConditionKeyMap(condKeyMap1) 68 | 69 | for k, v := range condKeyMap2 { 70 | if ev, ok := out[k]; ok { 71 | out[k] = ev.Union(v) 72 | } else { 73 | out[k] = set.CopyStringSet(v) 74 | } 75 | } 76 | 77 | return out 78 | } 79 | 80 | // ConditionMap - map of condition and conditional values. 81 | type ConditionMap map[string]ConditionKeyMap 82 | 83 | // Add - adds condition key and condition value. The value is appended if key already exists. 84 | func (cond ConditionMap) Add(condKey string, condKeyMap ConditionKeyMap) { 85 | if v, ok := cond[condKey]; ok { 86 | cond[condKey] = mergeConditionKeyMap(v, condKeyMap) 87 | } else { 88 | cond[condKey] = CopyConditionKeyMap(condKeyMap) 89 | } 90 | } 91 | 92 | // Remove - removes condition key and its value. 93 | func (cond ConditionMap) Remove(condKey string) { 94 | if _, ok := cond[condKey]; ok { 95 | delete(cond, condKey) 96 | } 97 | } 98 | 99 | // mergeConditionMap - returns new ConditionMap which contains merged key/value of two ConditionMap. 100 | func mergeConditionMap(condMap1 ConditionMap, condMap2 ConditionMap) ConditionMap { 101 | out := make(ConditionMap) 102 | 103 | for k, v := range condMap1 { 104 | out[k] = CopyConditionKeyMap(v) 105 | } 106 | 107 | for k, v := range condMap2 { 108 | if ev, ok := out[k]; ok { 109 | out[k] = mergeConditionKeyMap(ev, v) 110 | } else { 111 | out[k] = CopyConditionKeyMap(v) 112 | } 113 | } 114 | 115 | return out 116 | } 117 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/pkg/s3signer/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2015-2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package s3signer 19 | 20 | import ( 21 | "crypto/hmac" 22 | "crypto/sha256" 23 | "net/http" 24 | ) 25 | 26 | // unsignedPayload - value to be set to X-Amz-Content-Sha256 header when 27 | const unsignedPayload = "UNSIGNED-PAYLOAD" 28 | 29 | // sum256 calculate sha256 sum for an input byte array. 30 | func sum256(data []byte) []byte { 31 | hash := sha256.New() 32 | hash.Write(data) 33 | return hash.Sum(nil) 34 | } 35 | 36 | // sumHMAC calculate hmac between two input byte array. 37 | func sumHMAC(key []byte, data []byte) []byte { 38 | hash := hmac.New(sha256.New, key) 39 | hash.Write(data) 40 | return hash.Sum(nil) 41 | } 42 | 43 | // getHostAddr returns host header if available, otherwise returns host from URL 44 | func getHostAddr(req *http.Request) string { 45 | if req.Host != "" { 46 | return req.Host 47 | } 48 | return req.URL.Host 49 | } 50 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/retry-continous.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2015-2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | import "time" 21 | 22 | // newRetryTimerContinous creates a timer with exponentially increasing delays forever. 23 | func (c Client) newRetryTimerContinous(unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int { 24 | attemptCh := make(chan int) 25 | 26 | // normalize jitter to the range [0, 1.0] 27 | if jitter < NoJitter { 28 | jitter = NoJitter 29 | } 30 | if jitter > MaxJitter { 31 | jitter = MaxJitter 32 | } 33 | 34 | // computes the exponential backoff duration according to 35 | // https://www.awsarchitectureblog.com/2015/03/backoff.html 36 | exponentialBackoffWait := func(attempt int) time.Duration { 37 | // 1< maxAttempt { 40 | attempt = maxAttempt 41 | } 42 | //sleep = random_between(0, min(cap, base * 2 ** attempt)) 43 | sleep := unit * time.Duration(1< cap { 45 | sleep = cap 46 | } 47 | if jitter != NoJitter { 48 | sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter) 49 | } 50 | return sleep 51 | } 52 | 53 | go func() { 54 | defer close(attemptCh) 55 | var nextBackoff int 56 | for { 57 | select { 58 | // Attempts starts. 59 | case attemptCh <- nextBackoff: 60 | nextBackoff++ 61 | case <-doneCh: 62 | // Stop the routine. 63 | return 64 | } 65 | time.Sleep(exponentialBackoffWait(nextBackoff)) 66 | } 67 | }() 68 | return attemptCh 69 | } 70 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/s3-endpoints.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 3 | * Copyright 2015-2017 Minio, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package minio 19 | 20 | // awsS3EndpointMap Amazon S3 endpoint map. 21 | var awsS3EndpointMap = map[string]string{ 22 | "us-east-1": "s3.amazonaws.com", 23 | "us-east-2": "s3-us-east-2.amazonaws.com", 24 | "us-west-2": "s3-us-west-2.amazonaws.com", 25 | "us-west-1": "s3-us-west-1.amazonaws.com", 26 | "ca-central-1": "s3-ca-central-1.amazonaws.com", 27 | "eu-west-1": "s3-eu-west-1.amazonaws.com", 28 | "eu-west-2": "s3-eu-west-2.amazonaws.com", 29 | "eu-west-3": "s3-eu-west-3.amazonaws.com", 30 | "eu-central-1": "s3-eu-central-1.amazonaws.com", 31 | "ap-south-1": "s3-ap-south-1.amazonaws.com", 32 | "ap-southeast-1": "s3-ap-southeast-1.amazonaws.com", 33 | "ap-southeast-2": "s3-ap-southeast-2.amazonaws.com", 34 | "ap-northeast-1": "s3-ap-northeast-1.amazonaws.com", 35 | "ap-northeast-2": "s3-ap-northeast-2.amazonaws.com", 36 | "sa-east-1": "s3-sa-east-1.amazonaws.com", 37 | "us-gov-west-1": "s3-us-gov-west-1.amazonaws.com", 38 | "cn-north-1": "s3.cn-north-1.amazonaws.com.cn", 39 | "cn-northwest-1": "s3.cn-northwest-1.amazonaws.com.cn", 40 | } 41 | 42 | // getS3Endpoint get Amazon S3 endpoint based on the bucket location. 43 | func getS3Endpoint(bucketLocation string) (s3Endpoint string) { 44 | s3Endpoint, ok := awsS3EndpointMap[bucketLocation] 45 | if !ok { 46 | // Default to 's3.amazonaws.com' endpoint. 47 | s3Endpoint = "s3.amazonaws.com" 48 | } 49 | return s3Endpoint 50 | } 51 | -------------------------------------------------------------------------------- /vendor/github.com/minio/minio-go/transport.go: -------------------------------------------------------------------------------- 1 | // +build go1.7 go1.8 2 | 3 | /* 4 | * Minio Go Library for Amazon S3 Compatible Cloud Storage 5 | * Copyright 2017-2018 Minio, Inc. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); 8 | * you may not use this file except in compliance with the License. 9 | * You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * See the License for the specific language governing permissions and 17 | * limitations under the License. 18 | */ 19 | 20 | package minio 21 | 22 | import ( 23 | "net" 24 | "net/http" 25 | "time" 26 | ) 27 | 28 | // DefaultTransport - this default transport is similar to 29 | // http.DefaultTransport but with additional param DisableCompression 30 | // is set to true to avoid decompressing content with 'gzip' encoding. 31 | var DefaultTransport http.RoundTripper = &http.Transport{ 32 | Proxy: http.ProxyFromEnvironment, 33 | DialContext: (&net.Dialer{ 34 | Timeout: 30 * time.Second, 35 | KeepAlive: 30 * time.Second, 36 | DualStack: true, 37 | }).DialContext, 38 | MaxIdleConns: 100, 39 | MaxIdleConnsPerHost: 100, 40 | IdleConnTimeout: 90 * time.Second, 41 | TLSHandshakeTimeout: 10 * time.Second, 42 | ExpectContinueTimeout: 1 * time.Second, 43 | // Set this value so that the underlying transport round-tripper 44 | // doesn't try to auto decode the body of objects with 45 | // content-encoding set to `gzip`. 46 | // 47 | // Refer: 48 | // https://golang.org/src/net/http/transport.go?h=roundTrip#L1843 49 | DisableCompression: true, 50 | } 51 | -------------------------------------------------------------------------------- /vendor/github.com/mitchellh/go-homedir/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Mitchell Hashimoto 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /vendor/github.com/mitchellh/go-homedir/README.md: -------------------------------------------------------------------------------- 1 | # go-homedir 2 | 3 | This is a Go library for detecting the user's home directory without 4 | the use of cgo, so the library can be used in cross-compilation environments. 5 | 6 | Usage is incredibly simple, just call `homedir.Dir()` to get the home directory 7 | for a user, and `homedir.Expand()` to expand the `~` in a path to the home 8 | directory. 9 | 10 | **Why not just use `os/user`?** The built-in `os/user` package requires 11 | cgo on Darwin systems. This means that any Go code that uses that package 12 | cannot cross compile. But 99% of the time the use for `os/user` is just to 13 | retrieve the home directory, which we can do for the current user without 14 | cgo. This library does that, enabling cross-compilation. 15 | -------------------------------------------------------------------------------- /vendor/github.com/pmezard/go-difflib/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, Patrick Mezard 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | The names of its contributors may not be used to endorse or promote 14 | products derived from this software without specific prior written 15 | permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 18 | IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 | TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 20 | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 23 | TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 25 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 26 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 - 2013 Mat Ryer and Tyler Bunnell 2 | 3 | Please consider promoting this project if you find it useful. 4 | 5 | Permission is hereby granted, free of charge, to any person 6 | obtaining a copy of this software and associated documentation 7 | files (the "Software"), to deal in the Software without restriction, 8 | including without limitation the rights to use, copy, modify, merge, 9 | publish, distribute, sublicense, and/or sell copies of the Software, 10 | and to permit persons to whom the Software is furnished to do so, 11 | subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included 14 | in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 20 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT 21 | OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE 22 | OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl: -------------------------------------------------------------------------------- 1 | {{.CommentFormat}} 2 | func {{.DocInfo.Name}}f(t TestingT, {{.ParamsFormat}}) bool { 3 | if h, ok := t.(tHelper); ok { h.Helper() } 4 | return {{.DocInfo.Name}}(t, {{.ForwardedParamsFormat}}) 5 | } 6 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl: -------------------------------------------------------------------------------- 1 | {{.CommentWithoutT "a"}} 2 | func (a *Assertions) {{.DocInfo.Name}}({{.Params}}) bool { 3 | if h, ok := a.t.(tHelper); ok { h.Helper() } 4 | return {{.DocInfo.Name}}(a.t, {{.ForwardedParams}}) 5 | } 6 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/assert/doc.go: -------------------------------------------------------------------------------- 1 | // Package assert provides a set of comprehensive testing tools for use with the normal Go testing system. 2 | // 3 | // Example Usage 4 | // 5 | // The following is a complete example using assert in a standard test function: 6 | // import ( 7 | // "testing" 8 | // "github.com/stretchr/testify/assert" 9 | // ) 10 | // 11 | // func TestSomething(t *testing.T) { 12 | // 13 | // var a string = "Hello" 14 | // var b string = "Hello" 15 | // 16 | // assert.Equal(t, a, b, "The two words should be the same.") 17 | // 18 | // } 19 | // 20 | // if you assert many times, use the format below: 21 | // 22 | // import ( 23 | // "testing" 24 | // "github.com/stretchr/testify/assert" 25 | // ) 26 | // 27 | // func TestSomething(t *testing.T) { 28 | // assert := assert.New(t) 29 | // 30 | // var a string = "Hello" 31 | // var b string = "Hello" 32 | // 33 | // assert.Equal(a, b, "The two words should be the same.") 34 | // } 35 | // 36 | // Assertions 37 | // 38 | // Assertions allow you to easily write test code, and are global funcs in the `assert` package. 39 | // All assertion functions take, as the first argument, the `*testing.T` object provided by the 40 | // testing framework. This allows the assertion funcs to write the failings and other details to 41 | // the correct place. 42 | // 43 | // Every assertion function also takes an optional string message as the final argument, 44 | // allowing custom error messages to be appended to the message the assertion method outputs. 45 | package assert 46 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/assert/errors.go: -------------------------------------------------------------------------------- 1 | package assert 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | // AnError is an error instance useful for testing. If the code does not care 8 | // about error specifics, and only needs to return the error for example, this 9 | // error should be used to make the test code more readable. 10 | var AnError = errors.New("assert.AnError general error for testing") 11 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/assert/forward_assertions.go: -------------------------------------------------------------------------------- 1 | package assert 2 | 3 | // Assertions provides assertion methods around the 4 | // TestingT interface. 5 | type Assertions struct { 6 | t TestingT 7 | } 8 | 9 | // New makes a new Assertions object for the specified TestingT. 10 | func New(t TestingT) *Assertions { 11 | return &Assertions{ 12 | t: t, 13 | } 14 | } 15 | 16 | //go:generate go run ../_codegen/main.go -output-package=assert -template=assertion_forward.go.tmpl -include-format-funcs 17 | -------------------------------------------------------------------------------- /vendor/golang.org/x/net/AUTHORS: -------------------------------------------------------------------------------- 1 | # This source code refers to The Go Authors for copyright purposes. 2 | # The master list of authors is in the main Go distribution, 3 | # visible at http://tip.golang.org/AUTHORS. 4 | -------------------------------------------------------------------------------- /vendor/golang.org/x/net/CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # This source code was written by the Go contributors. 2 | # The master list of contributors is in the main Go distribution, 3 | # visible at http://tip.golang.org/CONTRIBUTORS. 4 | -------------------------------------------------------------------------------- /vendor/golang.org/x/net/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009 The Go Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /vendor/golang.org/x/net/PATENTS: -------------------------------------------------------------------------------- 1 | Additional IP Rights Grant (Patents) 2 | 3 | "This implementation" means the copyrightable works distributed by 4 | Google as part of the Go project. 5 | 6 | Google hereby grants to You a perpetual, worldwide, non-exclusive, 7 | no-charge, royalty-free, irrevocable (except as stated in this section) 8 | patent license to make, have made, use, offer to sell, sell, import, 9 | transfer and otherwise run, modify and propagate the contents of this 10 | implementation of Go, where such license applies only to those patent 11 | claims, both currently owned or controlled by Google and acquired in 12 | the future, licensable by Google that are necessarily infringed by this 13 | implementation of Go. This grant does not include claims that would be 14 | infringed only as a consequence of further modification of this 15 | implementation. If you or your agent or exclusive licensee institute or 16 | order or agree to the institution of patent litigation against any 17 | entity (including a cross-claim or counterclaim in a lawsuit) alleging 18 | that this implementation of Go or any code incorporated within this 19 | implementation of Go constitutes direct or contributory patent 20 | infringement, or inducement of patent infringement, then any patent 21 | rights granted to you under this License for this implementation of Go 22 | shall terminate as of the date such litigation is filed. 23 | -------------------------------------------------------------------------------- /vendor/golang.org/x/net/idna/trie.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | // Copyright 2016 The Go Authors. All rights reserved. 4 | // Use of this source code is governed by a BSD-style 5 | // license that can be found in the LICENSE file. 6 | 7 | package idna 8 | 9 | // appendMapping appends the mapping for the respective rune. isMapped must be 10 | // true. A mapping is a categorization of a rune as defined in UTS #46. 11 | func (c info) appendMapping(b []byte, s string) []byte { 12 | index := int(c >> indexShift) 13 | if c&xorBit == 0 { 14 | s := mappings[index:] 15 | return append(b, s[1:s[0]+1]...) 16 | } 17 | b = append(b, s...) 18 | if c&inlineXOR == inlineXOR { 19 | // TODO: support and handle two-byte inline masks 20 | b[len(b)-1] ^= byte(index) 21 | } else { 22 | for p := len(b) - int(xorData[index]); p < len(b); p++ { 23 | index++ 24 | b[p] ^= xorData[index] 25 | } 26 | } 27 | return b 28 | } 29 | 30 | // Sparse block handling code. 31 | 32 | type valueRange struct { 33 | value uint16 // header: value:stride 34 | lo, hi byte // header: lo:n 35 | } 36 | 37 | type sparseBlocks struct { 38 | values []valueRange 39 | offset []uint16 40 | } 41 | 42 | var idnaSparse = sparseBlocks{ 43 | values: idnaSparseValues[:], 44 | offset: idnaSparseOffset[:], 45 | } 46 | 47 | // Don't use newIdnaTrie to avoid unconditional linking in of the table. 48 | var trie = &idnaTrie{} 49 | 50 | // lookup determines the type of block n and looks up the value for b. 51 | // For n < t.cutoff, the block is a simple lookup table. Otherwise, the block 52 | // is a list of ranges with an accompanying value. Given a matching range r, 53 | // the value for b is by r.value + (b - r.lo) * stride. 54 | func (t *sparseBlocks) lookup(n uint32, b byte) uint16 { 55 | offset := t.offset[n] 56 | header := t.values[offset] 57 | lo := offset + 1 58 | hi := lo + uint16(header.lo) 59 | for lo < hi { 60 | m := lo + (hi-lo)/2 61 | r := t.values[m] 62 | if r.lo <= b && b <= r.hi { 63 | return r.value + uint16(b-r.lo)*header.value 64 | } 65 | if b < r.lo { 66 | hi = m 67 | } else { 68 | lo = m + 1 69 | } 70 | } 71 | return 0 72 | } 73 | -------------------------------------------------------------------------------- /vendor/golang.org/x/net/idna/trieval.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | package idna 4 | 5 | // This file contains definitions for interpreting the trie value of the idna 6 | // trie generated by "go run gen*.go". It is shared by both the generator 7 | // program and the resultant package. Sharing is achieved by the generator 8 | // copying gen_trieval.go to trieval.go and changing what's above this comment. 9 | 10 | // info holds information from the IDNA mapping table for a single rune. It is 11 | // the value returned by a trie lookup. In most cases, all information fits in 12 | // a 16-bit value. For mappings, this value may contain an index into a slice 13 | // with the mapped string. Such mappings can consist of the actual mapped value 14 | // or an XOR pattern to be applied to the bytes of the UTF8 encoding of the 15 | // input rune. This technique is used by the cases packages and reduces the 16 | // table size significantly. 17 | // 18 | // The per-rune values have the following format: 19 | // 20 | // if mapped { 21 | // if inlinedXOR { 22 | // 15..13 inline XOR marker 23 | // 12..11 unused 24 | // 10..3 inline XOR mask 25 | // } else { 26 | // 15..3 index into xor or mapping table 27 | // } 28 | // } else { 29 | // 15..14 unused 30 | // 13 mayNeedNorm 31 | // 12..11 attributes 32 | // 10..8 joining type 33 | // 7..3 category type 34 | // } 35 | // 2 use xor pattern 36 | // 1..0 mapped category 37 | // 38 | // See the definitions below for a more detailed description of the various 39 | // bits. 40 | type info uint16 41 | 42 | const ( 43 | catSmallMask = 0x3 44 | catBigMask = 0xF8 45 | indexShift = 3 46 | xorBit = 0x4 // interpret the index as an xor pattern 47 | inlineXOR = 0xE000 // These bits are set if the XOR pattern is inlined. 48 | 49 | joinShift = 8 50 | joinMask = 0x07 51 | 52 | // Attributes 53 | attributesMask = 0x1800 54 | viramaModifier = 0x1800 55 | modifier = 0x1000 56 | rtl = 0x0800 57 | 58 | mayNeedNorm = 0x2000 59 | ) 60 | 61 | // A category corresponds to a category defined in the IDNA mapping table. 62 | type category uint16 63 | 64 | const ( 65 | unknown category = 0 // not currently defined in unicode. 66 | mapped category = 1 67 | disallowedSTD3Mapped category = 2 68 | deviation category = 3 69 | ) 70 | 71 | const ( 72 | valid category = 0x08 73 | validNV8 category = 0x18 74 | validXV8 category = 0x28 75 | disallowed category = 0x40 76 | disallowedSTD3Valid category = 0x80 77 | ignored category = 0xC0 78 | ) 79 | 80 | // join types and additional rune information 81 | const ( 82 | joiningL = (iota + 1) 83 | joiningD 84 | joiningT 85 | joiningR 86 | 87 | //the following types are derived during processing 88 | joinZWJ 89 | joinZWNJ 90 | joinVirama 91 | numJoinTypes 92 | ) 93 | 94 | func (c info) isMapped() bool { 95 | return c&0x3 != 0 96 | } 97 | 98 | func (c info) category() category { 99 | small := c & catSmallMask 100 | if small != 0 { 101 | return category(small) 102 | } 103 | return category(c & catBigMask) 104 | } 105 | 106 | func (c info) joinType() info { 107 | if c.isMapped() { 108 | return 0 109 | } 110 | return (c >> joinShift) & joinMask 111 | } 112 | 113 | func (c info) isModifier() bool { 114 | return c&(modifier|catSmallMask) == modifier 115 | } 116 | 117 | func (c info) isViramaModifier() bool { 118 | return c&(attributesMask|catSmallMask) == viramaModifier 119 | } 120 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/AUTHORS: -------------------------------------------------------------------------------- 1 | # This source code refers to The Go Authors for copyright purposes. 2 | # The master list of authors is in the main Go distribution, 3 | # visible at http://tip.golang.org/AUTHORS. 4 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # This source code was written by the Go contributors. 2 | # The master list of contributors is in the main Go distribution, 3 | # visible at http://tip.golang.org/CONTRIBUTORS. 4 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009 The Go Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/PATENTS: -------------------------------------------------------------------------------- 1 | Additional IP Rights Grant (Patents) 2 | 3 | "This implementation" means the copyrightable works distributed by 4 | Google as part of the Go project. 5 | 6 | Google hereby grants to You a perpetual, worldwide, non-exclusive, 7 | no-charge, royalty-free, irrevocable (except as stated in this section) 8 | patent license to make, have made, use, offer to sell, sell, import, 9 | transfer and otherwise run, modify and propagate the contents of this 10 | implementation of Go, where such license applies only to those patent 11 | claims, both currently owned or controlled by Google and acquired in 12 | the future, licensable by Google that are necessarily infringed by this 13 | implementation of Go. This grant does not include claims that would be 14 | infringed only as a consequence of further modification of this 15 | implementation. If you or your agent or exclusive licensee institute or 16 | order or agree to the institution of patent litigation against any 17 | entity (including a cross-claim or counterclaim in a lawsuit) alleging 18 | that this implementation of Go or any code incorporated within this 19 | implementation of Go constitutes direct or contributory patent 20 | infringement, or inducement of patent infringement, then any patent 21 | rights granted to you under this License for this implementation of Go 22 | shall terminate as of the date such litigation is filed. 23 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/secure/bidirule/bidirule10.0.0.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build go1.10 6 | 7 | package bidirule 8 | 9 | func (t *Transformer) isFinal() bool { 10 | return t.state == ruleLTRFinal || t.state == ruleRTLFinal || t.state == ruleInitial 11 | } 12 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/secure/bidirule/bidirule9.0.0.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !go1.10 6 | 7 | package bidirule 8 | 9 | func (t *Transformer) isFinal() bool { 10 | if !t.isRTL() { 11 | return true 12 | } 13 | return t.state == ruleLTRFinal || t.state == ruleRTLFinal || t.state == ruleInitial 14 | } 15 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/unicode/bidi/gen.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build ignore 6 | 7 | package main 8 | 9 | import ( 10 | "flag" 11 | "log" 12 | 13 | "golang.org/x/text/internal/gen" 14 | "golang.org/x/text/internal/triegen" 15 | "golang.org/x/text/internal/ucd" 16 | ) 17 | 18 | var outputFile = flag.String("out", "tables.go", "output file") 19 | 20 | func main() { 21 | gen.Init() 22 | gen.Repackage("gen_trieval.go", "trieval.go", "bidi") 23 | gen.Repackage("gen_ranges.go", "ranges_test.go", "bidi") 24 | 25 | genTables() 26 | } 27 | 28 | // bidiClass names and codes taken from class "bc" in 29 | // http://www.unicode.org/Public/8.0.0/ucd/PropertyValueAliases.txt 30 | var bidiClass = map[string]Class{ 31 | "AL": AL, // ArabicLetter 32 | "AN": AN, // ArabicNumber 33 | "B": B, // ParagraphSeparator 34 | "BN": BN, // BoundaryNeutral 35 | "CS": CS, // CommonSeparator 36 | "EN": EN, // EuropeanNumber 37 | "ES": ES, // EuropeanSeparator 38 | "ET": ET, // EuropeanTerminator 39 | "L": L, // LeftToRight 40 | "NSM": NSM, // NonspacingMark 41 | "ON": ON, // OtherNeutral 42 | "R": R, // RightToLeft 43 | "S": S, // SegmentSeparator 44 | "WS": WS, // WhiteSpace 45 | 46 | "FSI": Control, 47 | "PDF": Control, 48 | "PDI": Control, 49 | "LRE": Control, 50 | "LRI": Control, 51 | "LRO": Control, 52 | "RLE": Control, 53 | "RLI": Control, 54 | "RLO": Control, 55 | } 56 | 57 | func genTables() { 58 | if numClass > 0x0F { 59 | log.Fatalf("Too many Class constants (%#x > 0x0F).", numClass) 60 | } 61 | w := gen.NewCodeWriter() 62 | defer w.WriteVersionedGoFile(*outputFile, "bidi") 63 | 64 | gen.WriteUnicodeVersion(w) 65 | 66 | t := triegen.NewTrie("bidi") 67 | 68 | // Build data about bracket mapping. These bits need to be or-ed with 69 | // any other bits. 70 | orMask := map[rune]uint64{} 71 | 72 | xorMap := map[rune]int{} 73 | xorMasks := []rune{0} // First value is no-op. 74 | 75 | ucd.Parse(gen.OpenUCDFile("BidiBrackets.txt"), func(p *ucd.Parser) { 76 | r1 := p.Rune(0) 77 | r2 := p.Rune(1) 78 | xor := r1 ^ r2 79 | if _, ok := xorMap[xor]; !ok { 80 | xorMap[xor] = len(xorMasks) 81 | xorMasks = append(xorMasks, xor) 82 | } 83 | entry := uint64(xorMap[xor]) << xorMaskShift 84 | switch p.String(2) { 85 | case "o": 86 | entry |= openMask 87 | case "c", "n": 88 | default: 89 | log.Fatalf("Unknown bracket class %q.", p.String(2)) 90 | } 91 | orMask[r1] = entry 92 | }) 93 | 94 | w.WriteComment(` 95 | xorMasks contains masks to be xor-ed with brackets to get the reverse 96 | version.`) 97 | w.WriteVar("xorMasks", xorMasks) 98 | 99 | done := map[rune]bool{} 100 | 101 | insert := func(r rune, c Class) { 102 | if !done[r] { 103 | t.Insert(r, orMask[r]|uint64(c)) 104 | done[r] = true 105 | } 106 | } 107 | 108 | // Insert the derived BiDi properties. 109 | ucd.Parse(gen.OpenUCDFile("extracted/DerivedBidiClass.txt"), func(p *ucd.Parser) { 110 | r := p.Rune(0) 111 | class, ok := bidiClass[p.String(1)] 112 | if !ok { 113 | log.Fatalf("%U: Unknown BiDi class %q", r, p.String(1)) 114 | } 115 | insert(r, class) 116 | }) 117 | visitDefaults(insert) 118 | 119 | // TODO: use sparse blocks. This would reduce table size considerably 120 | // from the looks of it. 121 | 122 | sz, err := t.Gen(w) 123 | if err != nil { 124 | log.Fatal(err) 125 | } 126 | w.Size += sz 127 | } 128 | 129 | // dummy values to make methods in gen_common compile. The real versions 130 | // will be generated by this file to tables.go. 131 | var ( 132 | xorMasks []rune 133 | ) 134 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/unicode/bidi/gen_ranges.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build ignore 6 | 7 | package main 8 | 9 | import ( 10 | "unicode" 11 | 12 | "golang.org/x/text/internal/gen" 13 | "golang.org/x/text/internal/ucd" 14 | "golang.org/x/text/unicode/rangetable" 15 | ) 16 | 17 | // These tables are hand-extracted from: 18 | // http://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedBidiClass.txt 19 | func visitDefaults(fn func(r rune, c Class)) { 20 | // first write default values for ranges listed above. 21 | visitRunes(fn, AL, []rune{ 22 | 0x0600, 0x07BF, // Arabic 23 | 0x08A0, 0x08FF, // Arabic Extended-A 24 | 0xFB50, 0xFDCF, // Arabic Presentation Forms 25 | 0xFDF0, 0xFDFF, 26 | 0xFE70, 0xFEFF, 27 | 0x0001EE00, 0x0001EEFF, // Arabic Mathematical Alpha Symbols 28 | }) 29 | visitRunes(fn, R, []rune{ 30 | 0x0590, 0x05FF, // Hebrew 31 | 0x07C0, 0x089F, // Nko et al. 32 | 0xFB1D, 0xFB4F, 33 | 0x00010800, 0x00010FFF, // Cypriot Syllabary et. al. 34 | 0x0001E800, 0x0001EDFF, 35 | 0x0001EF00, 0x0001EFFF, 36 | }) 37 | visitRunes(fn, ET, []rune{ // European Terminator 38 | 0x20A0, 0x20Cf, // Currency symbols 39 | }) 40 | rangetable.Visit(unicode.Noncharacter_Code_Point, func(r rune) { 41 | fn(r, BN) // Boundary Neutral 42 | }) 43 | ucd.Parse(gen.OpenUCDFile("DerivedCoreProperties.txt"), func(p *ucd.Parser) { 44 | if p.String(1) == "Default_Ignorable_Code_Point" { 45 | fn(p.Rune(0), BN) // Boundary Neutral 46 | } 47 | }) 48 | } 49 | 50 | func visitRunes(fn func(r rune, c Class), c Class, runes []rune) { 51 | for i := 0; i < len(runes); i += 2 { 52 | lo, hi := runes[i], runes[i+1] 53 | for j := lo; j <= hi; j++ { 54 | fn(j, c) 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/unicode/bidi/gen_trieval.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build ignore 6 | 7 | package main 8 | 9 | // Class is the Unicode BiDi class. Each rune has a single class. 10 | type Class uint 11 | 12 | const ( 13 | L Class = iota // LeftToRight 14 | R // RightToLeft 15 | EN // EuropeanNumber 16 | ES // EuropeanSeparator 17 | ET // EuropeanTerminator 18 | AN // ArabicNumber 19 | CS // CommonSeparator 20 | B // ParagraphSeparator 21 | S // SegmentSeparator 22 | WS // WhiteSpace 23 | ON // OtherNeutral 24 | BN // BoundaryNeutral 25 | NSM // NonspacingMark 26 | AL // ArabicLetter 27 | Control // Control LRO - PDI 28 | 29 | numClass 30 | 31 | LRO // LeftToRightOverride 32 | RLO // RightToLeftOverride 33 | LRE // LeftToRightEmbedding 34 | RLE // RightToLeftEmbedding 35 | PDF // PopDirectionalFormat 36 | LRI // LeftToRightIsolate 37 | RLI // RightToLeftIsolate 38 | FSI // FirstStrongIsolate 39 | PDI // PopDirectionalIsolate 40 | 41 | unknownClass = ^Class(0) 42 | ) 43 | 44 | var controlToClass = map[rune]Class{ 45 | 0x202D: LRO, // LeftToRightOverride, 46 | 0x202E: RLO, // RightToLeftOverride, 47 | 0x202A: LRE, // LeftToRightEmbedding, 48 | 0x202B: RLE, // RightToLeftEmbedding, 49 | 0x202C: PDF, // PopDirectionalFormat, 50 | 0x2066: LRI, // LeftToRightIsolate, 51 | 0x2067: RLI, // RightToLeftIsolate, 52 | 0x2068: FSI, // FirstStrongIsolate, 53 | 0x2069: PDI, // PopDirectionalIsolate, 54 | } 55 | 56 | // A trie entry has the following bits: 57 | // 7..5 XOR mask for brackets 58 | // 4 1: Bracket open, 0: Bracket close 59 | // 3..0 Class type 60 | 61 | const ( 62 | openMask = 0x10 63 | xorMaskShift = 5 64 | ) 65 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/unicode/bidi/trieval.go: -------------------------------------------------------------------------------- 1 | // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. 2 | 3 | package bidi 4 | 5 | // Class is the Unicode BiDi class. Each rune has a single class. 6 | type Class uint 7 | 8 | const ( 9 | L Class = iota // LeftToRight 10 | R // RightToLeft 11 | EN // EuropeanNumber 12 | ES // EuropeanSeparator 13 | ET // EuropeanTerminator 14 | AN // ArabicNumber 15 | CS // CommonSeparator 16 | B // ParagraphSeparator 17 | S // SegmentSeparator 18 | WS // WhiteSpace 19 | ON // OtherNeutral 20 | BN // BoundaryNeutral 21 | NSM // NonspacingMark 22 | AL // ArabicLetter 23 | Control // Control LRO - PDI 24 | 25 | numClass 26 | 27 | LRO // LeftToRightOverride 28 | RLO // RightToLeftOverride 29 | LRE // LeftToRightEmbedding 30 | RLE // RightToLeftEmbedding 31 | PDF // PopDirectionalFormat 32 | LRI // LeftToRightIsolate 33 | RLI // RightToLeftIsolate 34 | FSI // FirstStrongIsolate 35 | PDI // PopDirectionalIsolate 36 | 37 | unknownClass = ^Class(0) 38 | ) 39 | 40 | var controlToClass = map[rune]Class{ 41 | 0x202D: LRO, // LeftToRightOverride, 42 | 0x202E: RLO, // RightToLeftOverride, 43 | 0x202A: LRE, // LeftToRightEmbedding, 44 | 0x202B: RLE, // RightToLeftEmbedding, 45 | 0x202C: PDF, // PopDirectionalFormat, 46 | 0x2066: LRI, // LeftToRightIsolate, 47 | 0x2067: RLI, // RightToLeftIsolate, 48 | 0x2068: FSI, // FirstStrongIsolate, 49 | 0x2069: PDI, // PopDirectionalIsolate, 50 | } 51 | 52 | // A trie entry has the following bits: 53 | // 7..5 XOR mask for brackets 54 | // 4 1: Bracket open, 0: Bracket close 55 | // 3..0 Class type 56 | 57 | const ( 58 | openMask = 0x10 59 | xorMaskShift = 5 60 | ) 61 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/unicode/norm/input.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package norm 6 | 7 | import "unicode/utf8" 8 | 9 | type input struct { 10 | str string 11 | bytes []byte 12 | } 13 | 14 | func inputBytes(str []byte) input { 15 | return input{bytes: str} 16 | } 17 | 18 | func inputString(str string) input { 19 | return input{str: str} 20 | } 21 | 22 | func (in *input) setBytes(str []byte) { 23 | in.str = "" 24 | in.bytes = str 25 | } 26 | 27 | func (in *input) setString(str string) { 28 | in.str = str 29 | in.bytes = nil 30 | } 31 | 32 | func (in *input) _byte(p int) byte { 33 | if in.bytes == nil { 34 | return in.str[p] 35 | } 36 | return in.bytes[p] 37 | } 38 | 39 | func (in *input) skipASCII(p, max int) int { 40 | if in.bytes == nil { 41 | for ; p < max && in.str[p] < utf8.RuneSelf; p++ { 42 | } 43 | } else { 44 | for ; p < max && in.bytes[p] < utf8.RuneSelf; p++ { 45 | } 46 | } 47 | return p 48 | } 49 | 50 | func (in *input) skipContinuationBytes(p int) int { 51 | if in.bytes == nil { 52 | for ; p < len(in.str) && !utf8.RuneStart(in.str[p]); p++ { 53 | } 54 | } else { 55 | for ; p < len(in.bytes) && !utf8.RuneStart(in.bytes[p]); p++ { 56 | } 57 | } 58 | return p 59 | } 60 | 61 | func (in *input) appendSlice(buf []byte, b, e int) []byte { 62 | if in.bytes != nil { 63 | return append(buf, in.bytes[b:e]...) 64 | } 65 | for i := b; i < e; i++ { 66 | buf = append(buf, in.str[i]) 67 | } 68 | return buf 69 | } 70 | 71 | func (in *input) copySlice(buf []byte, b, e int) int { 72 | if in.bytes == nil { 73 | return copy(buf, in.str[b:e]) 74 | } 75 | return copy(buf, in.bytes[b:e]) 76 | } 77 | 78 | func (in *input) charinfoNFC(p int) (uint16, int) { 79 | if in.bytes == nil { 80 | return nfcData.lookupString(in.str[p:]) 81 | } 82 | return nfcData.lookup(in.bytes[p:]) 83 | } 84 | 85 | func (in *input) charinfoNFKC(p int) (uint16, int) { 86 | if in.bytes == nil { 87 | return nfkcData.lookupString(in.str[p:]) 88 | } 89 | return nfkcData.lookup(in.bytes[p:]) 90 | } 91 | 92 | func (in *input) hangul(p int) (r rune) { 93 | var size int 94 | if in.bytes == nil { 95 | if !isHangulString(in.str[p:]) { 96 | return 0 97 | } 98 | r, size = utf8.DecodeRuneInString(in.str[p:]) 99 | } else { 100 | if !isHangul(in.bytes[p:]) { 101 | return 0 102 | } 103 | r, size = utf8.DecodeRune(in.bytes[p:]) 104 | } 105 | if size != hangulUTF8Size { 106 | return 0 107 | } 108 | return r 109 | } 110 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/unicode/norm/readwriter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package norm 6 | 7 | import "io" 8 | 9 | type normWriter struct { 10 | rb reorderBuffer 11 | w io.Writer 12 | buf []byte 13 | } 14 | 15 | // Write implements the standard write interface. If the last characters are 16 | // not at a normalization boundary, the bytes will be buffered for the next 17 | // write. The remaining bytes will be written on close. 18 | func (w *normWriter) Write(data []byte) (n int, err error) { 19 | // Process data in pieces to keep w.buf size bounded. 20 | const chunk = 4000 21 | 22 | for len(data) > 0 { 23 | // Normalize into w.buf. 24 | m := len(data) 25 | if m > chunk { 26 | m = chunk 27 | } 28 | w.rb.src = inputBytes(data[:m]) 29 | w.rb.nsrc = m 30 | w.buf = doAppend(&w.rb, w.buf, 0) 31 | data = data[m:] 32 | n += m 33 | 34 | // Write out complete prefix, save remainder. 35 | // Note that lastBoundary looks back at most 31 runes. 36 | i := lastBoundary(&w.rb.f, w.buf) 37 | if i == -1 { 38 | i = 0 39 | } 40 | if i > 0 { 41 | if _, err = w.w.Write(w.buf[:i]); err != nil { 42 | break 43 | } 44 | bn := copy(w.buf, w.buf[i:]) 45 | w.buf = w.buf[:bn] 46 | } 47 | } 48 | return n, err 49 | } 50 | 51 | // Close forces data that remains in the buffer to be written. 52 | func (w *normWriter) Close() error { 53 | if len(w.buf) > 0 { 54 | _, err := w.w.Write(w.buf) 55 | if err != nil { 56 | return err 57 | } 58 | } 59 | return nil 60 | } 61 | 62 | // Writer returns a new writer that implements Write(b) 63 | // by writing f(b) to w. The returned writer may use an 64 | // an internal buffer to maintain state across Write calls. 65 | // Calling its Close method writes any buffered data to w. 66 | func (f Form) Writer(w io.Writer) io.WriteCloser { 67 | wr := &normWriter{rb: reorderBuffer{}, w: w} 68 | wr.rb.init(f, nil) 69 | return wr 70 | } 71 | 72 | type normReader struct { 73 | rb reorderBuffer 74 | r io.Reader 75 | inbuf []byte 76 | outbuf []byte 77 | bufStart int 78 | lastBoundary int 79 | err error 80 | } 81 | 82 | // Read implements the standard read interface. 83 | func (r *normReader) Read(p []byte) (int, error) { 84 | for { 85 | if r.lastBoundary-r.bufStart > 0 { 86 | n := copy(p, r.outbuf[r.bufStart:r.lastBoundary]) 87 | r.bufStart += n 88 | if r.lastBoundary-r.bufStart > 0 { 89 | return n, nil 90 | } 91 | return n, r.err 92 | } 93 | if r.err != nil { 94 | return 0, r.err 95 | } 96 | outn := copy(r.outbuf, r.outbuf[r.lastBoundary:]) 97 | r.outbuf = r.outbuf[0:outn] 98 | r.bufStart = 0 99 | 100 | n, err := r.r.Read(r.inbuf) 101 | r.rb.src = inputBytes(r.inbuf[0:n]) 102 | r.rb.nsrc, r.err = n, err 103 | if n > 0 { 104 | r.outbuf = doAppend(&r.rb, r.outbuf, 0) 105 | } 106 | if err == io.EOF { 107 | r.lastBoundary = len(r.outbuf) 108 | } else { 109 | r.lastBoundary = lastBoundary(&r.rb.f, r.outbuf) 110 | if r.lastBoundary == -1 { 111 | r.lastBoundary = 0 112 | } 113 | } 114 | } 115 | } 116 | 117 | // Reader returns a new reader that implements Read 118 | // by reading data from r and returning f(data). 119 | func (f Form) Reader(r io.Reader) io.Reader { 120 | const chunk = 4000 121 | buf := make([]byte, chunk) 122 | rr := &normReader{rb: reorderBuffer{}, r: r, inbuf: buf} 123 | rr.rb.init(f, buf) 124 | return rr 125 | } 126 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/unicode/norm/transform.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package norm 6 | 7 | import ( 8 | "unicode/utf8" 9 | 10 | "golang.org/x/text/transform" 11 | ) 12 | 13 | // Reset implements the Reset method of the transform.Transformer interface. 14 | func (Form) Reset() {} 15 | 16 | // Transform implements the Transform method of the transform.Transformer 17 | // interface. It may need to write segments of up to MaxSegmentSize at once. 18 | // Users should either catch ErrShortDst and allow dst to grow or have dst be at 19 | // least of size MaxTransformChunkSize to be guaranteed of progress. 20 | func (f Form) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { 21 | n := 0 22 | // Cap the maximum number of src bytes to check. 23 | b := src 24 | eof := atEOF 25 | if ns := len(dst); ns < len(b) { 26 | err = transform.ErrShortDst 27 | eof = false 28 | b = b[:ns] 29 | } 30 | i, ok := formTable[f].quickSpan(inputBytes(b), n, len(b), eof) 31 | n += copy(dst[n:], b[n:i]) 32 | if !ok { 33 | nDst, nSrc, err = f.transform(dst[n:], src[n:], atEOF) 34 | return nDst + n, nSrc + n, err 35 | } 36 | if n < len(src) && !atEOF { 37 | err = transform.ErrShortSrc 38 | } 39 | return n, n, err 40 | } 41 | 42 | func flushTransform(rb *reorderBuffer) bool { 43 | // Write out (must fully fit in dst, or else it is an ErrShortDst). 44 | if len(rb.out) < rb.nrune*utf8.UTFMax { 45 | return false 46 | } 47 | rb.out = rb.out[rb.flushCopy(rb.out):] 48 | return true 49 | } 50 | 51 | var errs = []error{nil, transform.ErrShortDst, transform.ErrShortSrc} 52 | 53 | // transform implements the transform.Transformer interface. It is only called 54 | // when quickSpan does not pass for a given string. 55 | func (f Form) transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { 56 | // TODO: get rid of reorderBuffer. See CL 23460044. 57 | rb := reorderBuffer{} 58 | rb.init(f, src) 59 | for { 60 | // Load segment into reorder buffer. 61 | rb.setFlusher(dst[nDst:], flushTransform) 62 | end := decomposeSegment(&rb, nSrc, atEOF) 63 | if end < 0 { 64 | return nDst, nSrc, errs[-end] 65 | } 66 | nDst = len(dst) - len(rb.out) 67 | nSrc = end 68 | 69 | // Next quickSpan. 70 | end = rb.nsrc 71 | eof := atEOF 72 | if n := nSrc + len(dst) - nDst; n < end { 73 | err = transform.ErrShortDst 74 | end = n 75 | eof = false 76 | } 77 | end, ok := rb.f.quickSpan(rb.src, nSrc, end, eof) 78 | n := copy(dst[nDst:], rb.src.bytes[nSrc:end]) 79 | nSrc += n 80 | nDst += n 81 | if ok { 82 | if n < rb.nsrc && !atEOF { 83 | err = transform.ErrShortSrc 84 | } 85 | return nDst, nSrc, err 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/unicode/norm/trie.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package norm 6 | 7 | type valueRange struct { 8 | value uint16 // header: value:stride 9 | lo, hi byte // header: lo:n 10 | } 11 | 12 | type sparseBlocks struct { 13 | values []valueRange 14 | offset []uint16 15 | } 16 | 17 | var nfcSparse = sparseBlocks{ 18 | values: nfcSparseValues[:], 19 | offset: nfcSparseOffset[:], 20 | } 21 | 22 | var nfkcSparse = sparseBlocks{ 23 | values: nfkcSparseValues[:], 24 | offset: nfkcSparseOffset[:], 25 | } 26 | 27 | var ( 28 | nfcData = newNfcTrie(0) 29 | nfkcData = newNfkcTrie(0) 30 | ) 31 | 32 | // lookupValue determines the type of block n and looks up the value for b. 33 | // For n < t.cutoff, the block is a simple lookup table. Otherwise, the block 34 | // is a list of ranges with an accompanying value. Given a matching range r, 35 | // the value for b is by r.value + (b - r.lo) * stride. 36 | func (t *sparseBlocks) lookup(n uint32, b byte) uint16 { 37 | offset := t.offset[n] 38 | header := t.values[offset] 39 | lo := offset + 1 40 | hi := lo + uint16(header.lo) 41 | for lo < hi { 42 | m := lo + (hi-lo)/2 43 | r := t.values[m] 44 | if r.lo <= b && b <= r.hi { 45 | return r.value + uint16(b-r.lo)*header.value 46 | } 47 | if b < r.lo { 48 | hi = m 49 | } else { 50 | lo = m + 1 51 | } 52 | } 53 | return 0 54 | } 55 | -------------------------------------------------------------------------------- /vendor/golang.org/x/text/unicode/norm/triegen.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build ignore 6 | 7 | // Trie table generator. 8 | // Used by make*tables tools to generate a go file with trie data structures 9 | // for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte 10 | // sequence are used to lookup offsets in the index table to be used for the 11 | // next byte. The last byte is used to index into a table with 16-bit values. 12 | 13 | package main 14 | 15 | import ( 16 | "fmt" 17 | "io" 18 | ) 19 | 20 | const maxSparseEntries = 16 21 | 22 | type normCompacter struct { 23 | sparseBlocks [][]uint64 24 | sparseOffset []uint16 25 | sparseCount int 26 | name string 27 | } 28 | 29 | func mostFrequentStride(a []uint64) int { 30 | counts := make(map[int]int) 31 | var v int 32 | for _, x := range a { 33 | if stride := int(x) - v; v != 0 && stride >= 0 { 34 | counts[stride]++ 35 | } 36 | v = int(x) 37 | } 38 | var maxs, maxc int 39 | for stride, cnt := range counts { 40 | if cnt > maxc || (cnt == maxc && stride < maxs) { 41 | maxs, maxc = stride, cnt 42 | } 43 | } 44 | return maxs 45 | } 46 | 47 | func countSparseEntries(a []uint64) int { 48 | stride := mostFrequentStride(a) 49 | var v, count int 50 | for _, tv := range a { 51 | if int(tv)-v != stride { 52 | if tv != 0 { 53 | count++ 54 | } 55 | } 56 | v = int(tv) 57 | } 58 | return count 59 | } 60 | 61 | func (c *normCompacter) Size(v []uint64) (sz int, ok bool) { 62 | if n := countSparseEntries(v); n <= maxSparseEntries { 63 | return (n+1)*4 + 2, true 64 | } 65 | return 0, false 66 | } 67 | 68 | func (c *normCompacter) Store(v []uint64) uint32 { 69 | h := uint32(len(c.sparseOffset)) 70 | c.sparseBlocks = append(c.sparseBlocks, v) 71 | c.sparseOffset = append(c.sparseOffset, uint16(c.sparseCount)) 72 | c.sparseCount += countSparseEntries(v) + 1 73 | return h 74 | } 75 | 76 | func (c *normCompacter) Handler() string { 77 | return c.name + "Sparse.lookup" 78 | } 79 | 80 | func (c *normCompacter) Print(w io.Writer) (retErr error) { 81 | p := func(f string, x ...interface{}) { 82 | if _, err := fmt.Fprintf(w, f, x...); retErr == nil && err != nil { 83 | retErr = err 84 | } 85 | } 86 | 87 | ls := len(c.sparseBlocks) 88 | p("// %sSparseOffset: %d entries, %d bytes\n", c.name, ls, ls*2) 89 | p("var %sSparseOffset = %#v\n\n", c.name, c.sparseOffset) 90 | 91 | ns := c.sparseCount 92 | p("// %sSparseValues: %d entries, %d bytes\n", c.name, ns, ns*4) 93 | p("var %sSparseValues = [%d]valueRange {", c.name, ns) 94 | for i, b := range c.sparseBlocks { 95 | p("\n// Block %#x, offset %#x", i, c.sparseOffset[i]) 96 | var v int 97 | stride := mostFrequentStride(b) 98 | n := countSparseEntries(b) 99 | p("\n{value:%#04x,lo:%#02x},", stride, uint8(n)) 100 | for i, nv := range b { 101 | if int(nv)-v != stride { 102 | if v != 0 { 103 | p(",hi:%#02x},", 0x80+i-1) 104 | } 105 | if nv != 0 { 106 | p("\n{value:%#04x,lo:%#02x", nv, 0x80+i) 107 | } 108 | } 109 | v = int(nv) 110 | } 111 | if v != 0 { 112 | p(",hi:%#02x},", 0x80+len(b)-1) 113 | } 114 | } 115 | p("\n}\n\n") 116 | return 117 | } 118 | -------------------------------------------------------------------------------- /vendor/modules.txt: -------------------------------------------------------------------------------- 1 | # github.com/Azure/azure-pipeline-go v0.1.8 2 | github.com/Azure/azure-pipeline-go/pipeline 3 | # github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804 4 | github.com/Azure/azure-storage-blob-go/azblob 5 | # github.com/davecgh/go-spew v1.1.0 6 | github.com/davecgh/go-spew/spew 7 | # github.com/go-ini/ini v1.37.0 8 | github.com/go-ini/ini 9 | # github.com/minio/minio-go v0.0.0-20180307105924-9e124ec59547 10 | github.com/minio/minio-go 11 | github.com/minio/minio-go/pkg/credentials 12 | github.com/minio/minio-go/pkg/encrypt 13 | github.com/minio/minio-go/pkg/policy 14 | github.com/minio/minio-go/pkg/s3signer 15 | github.com/minio/minio-go/pkg/s3utils 16 | github.com/minio/minio-go/pkg/set 17 | # github.com/mitchellh/go-homedir v0.0.0-20180523094522-3864e76763d9 18 | github.com/mitchellh/go-homedir 19 | # github.com/pmezard/go-difflib v1.0.0 20 | github.com/pmezard/go-difflib/difflib 21 | # github.com/stretchr/testify v1.2.2 22 | github.com/stretchr/testify/assert 23 | # golang.org/x/net v0.0.0-20180124060956-0ed95abb35c4 24 | golang.org/x/net/lex/httplex 25 | golang.org/x/net/idna 26 | # golang.org/x/text v0.3.0 27 | golang.org/x/text/secure/bidirule 28 | golang.org/x/text/unicode/bidi 29 | golang.org/x/text/unicode/norm 30 | golang.org/x/text/transform 31 | --------------------------------------------------------------------------------