├── .gitignore ├── LICENSE ├── NOTICE ├── README.md ├── blob_migrator.go ├── blob_migrator_test.go ├── blobstore ├── azblob.go ├── azblob_test.go ├── blobstore.go ├── blobstore_suite_test.go ├── blobstorefakes │ ├── fake_blobstore.go │ └── fake_bucket_iterator.go ├── bucket_iterator.go ├── nfs.go ├── nfs_bucket_iterator.go ├── nfs_bucket_iterator_test.go ├── nfs_test.go ├── nfs_testdata │ ├── .nfs_test │ ├── cc-buildpacks │ │ └── ea │ │ │ └── 07 │ │ │ └── ea07de9b-dd94-477c-b904-0f77d47dd111_a32d9ae40371d557c7c90eb2affc3d7bba6abe69 │ ├── cc-droplets │ │ ├── 61 │ │ │ └── 7c │ │ │ │ └── 617cf88e-da7b-4722-be53-dad7130514c1 │ │ │ │ └── 047363432e6b22f6cfc10be16de2d7479dfe2ee3 │ │ └── buildpack_cache │ │ │ └── f3 │ │ │ └── 9a │ │ │ └── f39a3f10-b3fd-4dc4-aa05-54db6c34c296 │ │ │ └── cflinuxfs2 │ ├── cc-packages │ │ ├── 1a │ │ │ └── 94 │ │ │ │ └── 1a94dd34-fb36-47b8-a0af-682a22a94874 │ │ └── f1 │ │ │ └── 2c │ │ │ └── f12c3b36-4626-4c4b-b441-8dc88407bfae │ └── cc-resources │ │ ├── 0a │ │ └── 1e │ │ │ └── 0a1eea6d7bb903d8c075688534480e87d4151470 │ │ └── 0d │ │ └── 78 │ │ └── 0d7896e2bb23f88e26e52b22a075350b354df447 ├── s3.go ├── s3_bucket_iterator.go ├── s3_bucket_iterator_test.go ├── s3_test.go └── s3_testdata │ └── test.txt ├── blobstore_migration_watcher.go ├── blobstore_migrator.go ├── blobstore_migrator_test.go ├── ci ├── build-darwin.yml ├── build-linux.yml ├── build-windows.yml ├── build.sh ├── create-pivnet-metadata.yml ├── create-release-notes.yml ├── install-dependencies.yml ├── pipeline.yml └── unit.yml ├── cmd └── goblob │ ├── main.go │ └── main_test.go ├── commands ├── goblob.go ├── migrate.go ├── migrate2azblob.go └── version.go ├── glide.lock ├── glide.yaml ├── goblob_suite_test.go ├── goblobfakes ├── fake_blob_migrator.go └── fake_blobstore_migration_watcher.go ├── testrunner ├── validation ├── checksum.go ├── checksum_test.go ├── fixtures │ ├── 013110a30e2a475551c801b4c45e497ce71c26fe │ └── testfile └── validation_suite_test.go ├── version.go └── wercker.yml /.gitignore: -------------------------------------------------------------------------------- 1 | /credentials 2 | /vendor 3 | /.vagrant 4 | /Vagrantfile 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ 2 | 3 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 4 | 5 | 1. Definitions. 6 | 7 | "License" shall mean the terms and conditions for use, reproduction, and 8 | distribution as defined by Sections 1 through 9 of this document. 9 | 10 | "Licensor" shall mean the copyright owner or entity authorized by the copyright 11 | owner that is granting the License. 12 | 13 | "Legal Entity" shall mean the union of the acting entity and all other entities 14 | that control, are controlled by, or are under common control with that entity. 15 | For the purposes of this definition, "control" means (i) the power, direct or 16 | indirect, to cause the direction or management of such entity, whether by 17 | contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the 18 | outstanding shares, or (iii) beneficial ownership of such entity. 19 | 20 | "You" (or "Your") shall mean an individual or Legal Entity exercising 21 | permissions granted by this License. 22 | 23 | "Source" form shall mean the preferred form for making modifications, including 24 | but not limited to software source code, documentation source, and 25 | configuration files. 26 | 27 | "Object" form shall mean any form resulting from mechanical transformation or 28 | translation of a Source form, including but not limited to compiled object 29 | code, generated documentation, and conversions to other media types. 30 | 31 | "Work" shall mean the work of authorship, whether in Source or Object form, 32 | made available under the License, as indicated by a copyright notice that is 33 | included in or attached to the work (an example is provided in the Appendix 34 | below). 35 | 36 | "Derivative Works" shall mean any work, whether in Source or Object form, that 37 | is based on (or derived from) the Work and for which the editorial revisions, 38 | annotations, elaborations, or other modifications represent, as a whole, an 39 | original work of authorship. For the purposes of this License, Derivative Works 40 | shall not include works that remain separable from, or merely link (or bind by 41 | name) to the interfaces of, the Work and Derivative Works thereof. 42 | 43 | "Contribution" shall mean any work of authorship, including the original 44 | version of the Work and any modifications or additions to that Work or 45 | Derivative Works thereof, that is intentionally submitted to Licensor for 46 | inclusion in the Work by the copyright owner or by an individual or Legal 47 | Entity authorized to submit on behalf of the copyright owner. For the purposes 48 | of this definition, "submitted" means any form of electronic, verbal, or 49 | written communication sent to the Licensor or its representatives, including 50 | but not limited to communication on electronic mailing lists, source code 51 | control systems, and issue tracking systems that are managed by, or on behalf 52 | of, the Licensor for the purpose of discussing and improving the Work, but 53 | excluding communication that is conspicuously marked or otherwise designated in 54 | writing by the copyright owner as "Not a Contribution." 55 | 56 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf 57 | of whom a Contribution has been received by Licensor and subsequently 58 | incorporated within the Work. 59 | 60 | 2. Grant of Copyright License. 61 | 62 | Subject to the terms and conditions of this License, each Contributor hereby 63 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 64 | irrevocable copyright license to reproduce, prepare Derivative Works of, 65 | publicly display, publicly perform, sublicense, and distribute the Work and 66 | such Derivative Works in Source or Object form. 67 | 68 | 3. Grant of Patent License. 69 | 70 | Subject to the terms and conditions of this License, each Contributor hereby 71 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 72 | irrevocable (except as stated in this section) patent license to make, have 73 | made, use, offer to sell, sell, import, and otherwise transfer the Work, where 74 | such license applies only to those patent claims licensable by such Contributor 75 | that are necessarily infringed by their Contribution(s) alone or by combination 76 | of their Contribution(s) with the Work to which such Contribution(s) was 77 | submitted. If You institute patent litigation against any entity (including a 78 | cross-claim or counterclaim in a lawsuit) alleging that the Work or a 79 | Contribution incorporated within the Work constitutes direct or contributory 80 | patent infringement, then any patent licenses granted to You under this License 81 | for that Work shall terminate as of the date such litigation is filed. 82 | 83 | 4. Redistribution. 84 | 85 | You may reproduce and distribute copies of the Work or Derivative Works thereof 86 | in any medium, with or without modifications, and in Source or Object form, 87 | provided that You meet the following conditions: 88 | 89 | You must give any other recipients of the Work or Derivative Works a copy of 90 | this License; and You must cause any modified files to carry prominent notices 91 | stating that You changed the files; and You must retain, in the Source form of 92 | any Derivative Works that You distribute, all copyright, patent, trademark, and 93 | attribution notices from the Source form of the Work, excluding those notices 94 | that do not pertain to any part of the Derivative Works; and If the Work 95 | includes a "NOTICE" text file as part of its distribution, then any Derivative 96 | Works that You distribute must include a readable copy of the attribution 97 | notices contained within such NOTICE file, excluding those notices that do not 98 | pertain to any part of the Derivative Works, in at least one of the following 99 | places: within a NOTICE text file distributed as part of the Derivative Works; 100 | within the Source form or documentation, if provided along with the Derivative 101 | Works; or, within a display generated by the Derivative Works, if and wherever 102 | such third-party notices normally appear. The contents of the NOTICE file are 103 | for informational purposes only and do not modify the License. You may add Your 104 | own attribution notices within Derivative Works that You distribute, alongside 105 | or as an addendum to the NOTICE text from the Work, provided that such 106 | additional attribution notices cannot be construed as modifying the License. 107 | You may add Your own copyright statement to Your modifications and may provide 108 | additional or different license terms and conditions for use, reproduction, or 109 | distribution of Your modifications, or for any such Derivative Works as a 110 | whole, provided Your use, reproduction, and distribution of the Work otherwise 111 | complies with the conditions stated in this License. 112 | 113 | 5. Submission of Contributions. 114 | 115 | Unless You explicitly state otherwise, any Contribution intentionally submitted 116 | for inclusion in the Work by You to the Licensor shall be under the terms and 117 | conditions of this License, without any additional terms or conditions. 118 | Notwithstanding the above, nothing herein shall supersede or modify the terms 119 | of any separate license agreement you may have executed with Licensor regarding 120 | such Contributions. 121 | 122 | 6. Trademarks. 123 | 124 | This License does not grant permission to use the trade names, trademarks, 125 | service marks, or product names of the Licensor, except as required for 126 | reasonable and customary use in describing the origin of the Work and 127 | reproducing the content of the NOTICE file. 128 | 129 | 7. Disclaimer of Warranty. 130 | 131 | Unless required by applicable law or agreed to in writing, Licensor provides 132 | the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, 133 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, 134 | including, without limitation, any warranties or conditions of TITLE, 135 | NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are 136 | solely responsible for determining the appropriateness of using or 137 | redistributing the Work and assume any risks associated with Your exercise of 138 | permissions under this License. 139 | 140 | 8. Limitation of Liability. 141 | 142 | In no event and under no legal theory, whether in tort (including negligence), 143 | contract, or otherwise, unless required by applicable law (such as deliberate 144 | and grossly negligent acts) or agreed to in writing, shall any Contributor be 145 | liable to You for damages, including any direct, indirect, special, incidental, 146 | or consequential damages of any character arising as a result of this License 147 | or out of the use or inability to use the Work (including but not limited to 148 | damages for loss of goodwill, work stoppage, computer failure or malfunction, 149 | or any and all other commercial damages or losses), even if such Contributor 150 | has been advised of the possibility of such damages. 151 | 152 | 9. Accepting Warranty or Additional Liability. 153 | 154 | While redistributing the Work or Derivative Works thereof, You may choose to 155 | offer, and charge a fee for, acceptance of support, warranty, indemnity, or 156 | other liability obligations and/or rights consistent with this License. 157 | However, in accepting such obligations, You may act only on Your own behalf and 158 | on Your sole responsibility, not on behalf of any other Contributor, and only 159 | if You agree to indemnify, defend, and hold each Contributor harmless for any 160 | liability incurred by, or claims asserted against, such Contributor by reason 161 | of your accepting any such warranty or additional liability. 162 | 163 | END OF TERMS AND CONDITIONS 164 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | goblob 2 | 3 | Copyright (c) 2017-Present Pivotal Software, Inc. All Rights Reserved. 4 | 5 | This product is licensed to you under the Apache License, Version 2.0 (the 6 | "License"). 7 | 8 | You may not use this product except in compliance with the License. 9 | 10 | This product may include a number of subcomponents with separate copyright 11 | notices and license terms. Your use of these subcomponents is subject to the 12 | terms and conditions of the subcomponent's license, as noted in the LICENSE 13 | file. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # goblob 2 | 3 | `goblob` is a tool for migrating Cloud Foundry blobs from one blobstore to 4 | another. Presently it only supports migrating from an NFS blobstore to an 5 | S3-compatible one. 6 | 7 | ## Installing 8 | 9 | Download the [latest release](https://github.com/pivotal-cf/goblob/releases/latest). 10 | 11 | ### Install from source 12 | 13 | Requirements: 14 | 15 | * [glide](https://github.com/masterminds/glide) 16 | * [go](https://golang.org) 17 | 18 | ``` 19 | mkdir -p $GOPATH/src/github.com/pivotal-cf/goblob 20 | git clone git@github.com:pivotal-cf/goblob.git $GOPATH/src/github.com/pivotal-cf/goblob 21 | cd $GOPATH/src/github.com/pivotal-cf/goblob 22 | glide install 23 | GOARCH=amd64 GOOS=linux go install github.com/pivotal-cf/goblob/cmd/goblob 24 | ``` 25 | 26 | ## Usage 27 | 28 | The tool is a Golang binary, which must be executed on the NFS VM that you intend to migrate. The commands of the tool are: 29 | 30 | 31 | 32 | | Command | Description | 33 | |-----------------------------------|---------------------------------------------------| 34 | | `goblob migrate [OPTIONS]` | Migrate NFS blobstore to S3-compatible blobstore | 35 | | `goblob migrate2azure [OPTIONS]` | Migrate NFS blobstore to Azure blob storage | 36 | 37 | For each option you use, add `--` before the option name in the command you want to execute. 38 | 39 | ### Migrate NFS blobstore to S3-compatible blobstore 40 | 41 | `goblob migrate [OPTIONS]` 42 | 43 | #### Options 44 | 45 | * `concurrent-uploads`: Number of concurrent uploads (default: 20) 46 | * `exclude`: Directory to exclude (may be given more than once) 47 | 48 | ##### NFS-specific Options 49 | 50 | * `blobstore-path`: The path to the root of the NFS blobstore, e.g. /var/vcap/store/shared 51 | 52 | ##### S3-specific Options 53 | 54 | * `s3-endpoint`: The endpoint of the S3-compatible blobstore 55 | * `s3-accesskey`: The access key to use with the S3-compatible blobstore 56 | * `s3-secretkey`: The secret key to use with the S3-compatible blobstore 57 | * `region`: The region to use with the S3-compatible blobstore 58 | * `buildpacks-bucket-name`: The bucket containing buildpacks 59 | * `droplets-bucket-name`: The bucket containing droplets 60 | * `packages-bucket-name`: The bucket containing packages 61 | * `resources-bucket-name`: The bucket containing resources 62 | * `use-multipart-uploads`: Whether to use multi-part uploads 63 | * `disable-ssl`: Whether to disable SSL when uploading blobs 64 | * `insecure-skip-verify`: Skip server SSL certificate verification 65 | 66 | ### Migrate NFS blobstore to Azure blob storage 67 | 68 | `goblob migrate2azure [OPTIONS]` 69 | 70 | #### Example 71 | 72 | ``` 73 | goblob migrate2azure --blobstore-path /var/vcap/store/shared \ 74 | --azure-storage-account $storage_account_name \ 75 | --azure-storage-account-key $storage_account_key \ 76 | --cloud-name AzureCloud \ 77 | --buildpacks-bucket-name cf-buildpacks \ 78 | --droplets-bucket-name cf-droplets \ 79 | --packages-bucket-name cf-packages \ 80 | --resources-bucket-name cf-resources 81 | ``` 82 | 83 | #### Options 84 | 85 | * `concurrent-uploads`: Number of concurrent uploads (default: 20) 86 | * `exclude`: Directory to exclude (may be given more than once) 87 | 88 | ##### NFS-specific Options 89 | 90 | * `blobstore-path`: The path to the root of the NFS blobstore, e.g. /var/vcap/store/shared 91 | 92 | ##### Azure-specific Options 93 | 94 | * `azure-storage-account`: Azure storage account name 95 | * `azure-storage-account-key`: Azure storage account key 96 | * `cloud-name`: cloud name, available names are: AzureCloud, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment 97 | * `buildpacks-bucket-name`: The container for buildpacks 98 | * `droplets-bucket-name`: The container for droplets 99 | * `packages-bucket-name`: The container for packages 100 | * `resources-bucket-name`: The container for resources 101 | 102 | ## Post-migration Tasks 103 | 104 | - If your S3 service uses an SSL certificate signed by your own CA: Before applying changes in Ops Manager to switch to S3, make sure the root CA cert that signed the endpoint cert is a BOSH-trusted-certificate. You will need to update Ops Manager ca-certs (place the CA cert in /usr/local/share/ca-certificates and run update-ca-certificates, and restart tempest-web). You will need to add this certificate back in each time you do an upgrade of Ops Manager. In PCF 1.9+, Ops Manager will let you replace its own SSL cert and have that persist across upgrades. 105 | - Update OpsManager File Storage Config to point at S3 blobstore using buckets (cc-buildpacks-, cc-droplets-, cc-packages-, cc-resources-) 106 | - Click `Apply Changes` in Ops Manager 107 | - Once changes are applied, re-run `goblob` to migrate any files which were created after the initial migration 108 | - Validate apps can be restaged and pushed 109 | 110 | ## Removing NFS post-migration 111 | 112 | - Turn off bosh resurrector (`bosh vm resurrection off`) 113 | - In the IaaS console (e.g. AWS EC2, vCenter console, etc.), terminate all the CC VM jobs (Cloud Controller, Cloud Controller Worker, and Clock Global) + NFS (ensure the attached disks are removed as well). Note that your CF API services will stop being available at this point (running apps should continue to be available though). This step is required to ensure the removal of the NFS mount from these jobs. 114 | - `bosh cck` the cf deployment to check for any errors with the bosh state. It should ask you if you want to delete references to the missing CC/NFS jobs, which you want to do. 115 | - Go back to Ops Mgr and update your ERT configurations to zero NFS instances and re-add your desired instance counts for the CC jobs. 116 | - Click `Apply Changes` in Ops Manager. After this deploy is finished, your CF API service availibility will resume. 117 | - Turn back on the bosh vm resurrector, if it isn’t turned back on after your re-deploy (`bosh vm resurrection on`). 118 | 119 | ## Known Issues 120 | - Starting with PCF 1.12, Ops Manager no longer allows you to select the S3 blobstorage configuration for ERT/PAS, instead of the internal NFS option, without also deleting the NFS server VM. This means that you should shut down the Cloud Controller VMs before you switch your configuration, as you will not have a chance to run a post-configuration-switch migration once the NFS server is gone. 121 | 122 | ## Developing 123 | 124 | * Install [Docker](https://www.docker.com/products/docker) 125 | * `docker pull minio/minio` 126 | 127 | To run all of the tests in a Docker container: 128 | 129 | `./testrunner` 130 | 131 | To continually run the tests during development: 132 | 133 | * `docker run -p 9000:9000 -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" minio/minio server /tmp` 134 | * (in a separate terminal) `MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY ginkgo watch -r` 135 | -------------------------------------------------------------------------------- /blob_migrator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package goblob 16 | 17 | import ( 18 | "fmt" 19 | 20 | "github.com/pivotal-cf/goblob/blobstore" 21 | ) 22 | 23 | //go:generate counterfeiter . BlobMigrator 24 | 25 | type BlobMigrator interface { 26 | Migrate(blob *blobstore.Blob) error 27 | } 28 | 29 | type blobMigrator struct { 30 | dst blobstore.Blobstore 31 | src blobstore.Blobstore 32 | } 33 | 34 | func NewBlobMigrator(dst blobstore.Blobstore, src blobstore.Blobstore) BlobMigrator { 35 | return &blobMigrator{ 36 | dst: dst, 37 | src: src, 38 | } 39 | } 40 | 41 | func (m *blobMigrator) Migrate(blob *blobstore.Blob) error { 42 | reader, err := m.src.Read(blob) 43 | if err != nil { 44 | return fmt.Errorf("error reading blob at %s: %s", blob.Path, err) 45 | } 46 | defer reader.Close() 47 | 48 | err = m.dst.Write(blob, reader) 49 | if err != nil { 50 | return fmt.Errorf("error writing blob at %s: %s", blob.Path, err) 51 | } 52 | 53 | checksum, err := m.dst.Checksum(blob) 54 | if err != nil { 55 | return fmt.Errorf("error checksumming blob at %s: %s", blob.Path, err) 56 | } 57 | 58 | if checksum != blob.Checksum { 59 | return fmt.Errorf( 60 | "error at %s: checksum [%s] does not match [%s]", 61 | blob.Path, 62 | checksum, 63 | blob.Checksum, 64 | ) 65 | } 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /blob_migrator_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package goblob_test 16 | 17 | import ( 18 | "errors" 19 | "io" 20 | "io/ioutil" 21 | "strings" 22 | 23 | . "github.com/onsi/ginkgo" 24 | . "github.com/onsi/gomega" 25 | "github.com/pivotal-cf/goblob" 26 | "github.com/pivotal-cf/goblob/blobstore" 27 | "github.com/pivotal-cf/goblob/blobstore/blobstorefakes" 28 | ) 29 | 30 | var _ = Describe("BlobMigrator", func() { 31 | var ( 32 | blobMigrator goblob.BlobMigrator 33 | dstStore *blobstorefakes.FakeBlobstore 34 | srcStore *blobstorefakes.FakeBlobstore 35 | ) 36 | 37 | BeforeEach(func() { 38 | dstStore = &blobstorefakes.FakeBlobstore{} 39 | srcStore = &blobstorefakes.FakeBlobstore{} 40 | blobMigrator = goblob.NewBlobMigrator(dstStore, srcStore) 41 | }) 42 | 43 | Describe("Migrate", func() { 44 | var ( 45 | controlBlob *blobstore.Blob 46 | expectedReader io.ReadCloser 47 | ) 48 | 49 | BeforeEach(func() { 50 | expectedReader = ioutil.NopCloser(strings.NewReader("some content")) 51 | srcStore.ReadReturns(expectedReader, nil) 52 | dstStore.ChecksumReturns("some-checksum", nil) 53 | controlBlob = &blobstore.Blob{ 54 | Checksum: "some-checksum", 55 | Path: "some-path/some-filename", 56 | } 57 | }) 58 | 59 | It("tries to read the source blob", func() { 60 | err := blobMigrator.Migrate(controlBlob) 61 | Expect(err).NotTo(HaveOccurred()) 62 | Expect(srcStore.ReadCallCount()).To(Equal(1)) 63 | 64 | Expect(srcStore.ReadArgsForCall(0)).To(Equal(controlBlob)) 65 | }) 66 | 67 | It("tries to write the destination blob", func() { 68 | err := blobMigrator.Migrate(controlBlob) 69 | Expect(err).NotTo(HaveOccurred()) 70 | 71 | Expect(dstStore.WriteCallCount()).To(Equal(1)) 72 | 73 | blob, r := dstStore.WriteArgsForCall(0) 74 | Expect(blob).To(Equal(controlBlob)) 75 | Expect(r).To(Equal(expectedReader)) 76 | }) 77 | 78 | It("tries to checksum the destination blob", func() { 79 | err := blobMigrator.Migrate(controlBlob) 80 | Expect(err).NotTo(HaveOccurred()) 81 | Expect(dstStore.ChecksumCallCount()).To(Equal(1)) 82 | 83 | Expect(dstStore.ChecksumArgsForCall(0)).To(Equal(controlBlob)) 84 | }) 85 | 86 | It("returns nil", func() { 87 | err := blobMigrator.Migrate(controlBlob) 88 | Expect(err).NotTo(HaveOccurred()) 89 | }) 90 | 91 | Context("when there is an error reading the source blob", func() { 92 | BeforeEach(func() { 93 | srcStore.ReadReturns(nil, errors.New("read-error")) 94 | }) 95 | 96 | It("returns an error", func() { 97 | err := blobMigrator.Migrate(controlBlob) 98 | Expect(err).To(HaveOccurred()) 99 | Expect(err.Error()).To(Equal("error reading blob at some-path/some-filename: read-error")) 100 | }) 101 | }) 102 | 103 | Context("when there is an error writing the destination blob", func() { 104 | BeforeEach(func() { 105 | dstStore.WriteReturns(errors.New("write-error")) 106 | }) 107 | 108 | It("returns an error", func() { 109 | err := blobMigrator.Migrate(controlBlob) 110 | Expect(err).To(HaveOccurred()) 111 | Expect(err.Error()).To(Equal("error writing blob at some-path/some-filename: write-error")) 112 | }) 113 | }) 114 | 115 | Context("when there is an error getting the destination checksum", func() { 116 | BeforeEach(func() { 117 | dstStore.ChecksumReturns("", errors.New("checksum-error")) 118 | }) 119 | 120 | It("returns an error", func() { 121 | err := blobMigrator.Migrate(controlBlob) 122 | Expect(err).To(HaveOccurred()) 123 | Expect(err.Error()).To(Equal("error checksumming blob at some-path/some-filename: checksum-error")) 124 | }) 125 | }) 126 | 127 | Context("when the checksums do not match", func() { 128 | BeforeEach(func() { 129 | dstStore.ChecksumReturns("other-checksum", nil) 130 | }) 131 | 132 | It("returns an error", func() { 133 | err := blobMigrator.Migrate(controlBlob) 134 | Expect(err).To(HaveOccurred()) 135 | Expect(err.Error()).To(Equal("error at some-path/some-filename: checksum [other-checksum] does not match [some-checksum]")) 136 | }) 137 | }) 138 | }) 139 | }) 140 | -------------------------------------------------------------------------------- /blobstore/azblob.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobstore 16 | 17 | import ( 18 | "context" 19 | "encoding/base64" 20 | "encoding/hex" 21 | "errors" 22 | "fmt" 23 | "io" 24 | "net/url" 25 | "path/filepath" 26 | "strings" 27 | 28 | "github.com/cheggaaa/pb" 29 | "github.com/pivotal-cf/goblob/validation" 30 | 31 | "github.com/Azure/azure-storage-blob-go/2018-03-28/azblob" 32 | ) 33 | 34 | var ( 35 | containers = []string{"cc-buildpacks", "cc-droplets", "cc-packages", "cc-resources"} 36 | cloudStorageEnpointsMap = map[string]string{ 37 | "AzureChinaCloud": "core.chinacloudapi.cn", 38 | "AzureCloud": "core.windows.net", 39 | "AzureGermanCloud": "core.cloudapi.de", 40 | "AzureUSGovernment": "core.usgovcloudapi.net", 41 | } 42 | ) 43 | 44 | type azblobStore struct { 45 | serviceURL *azblob.ServiceURL 46 | useMultipartUploads bool 47 | containerMapping map[string]string 48 | } 49 | 50 | func NewAzBlobStore( 51 | accountName string, 52 | accountKey string, 53 | cloudName string, 54 | buildpacksContainerName string, 55 | dropletsContainerName string, 56 | packagesContainerName string, 57 | resourcesContainerName string, 58 | ) Blobstore { 59 | credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) 60 | if err != nil { 61 | panic(err) 62 | } 63 | pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{}) 64 | 65 | primaryURL, _ := url.Parse( 66 | fmt.Sprintf("https://%s.blob.%s", accountName, cloudStorageEnpointsMap[cloudName])) 67 | serviceURL := azblob.NewServiceURL(*primaryURL, pipeline) 68 | 69 | return &azblobStore{ 70 | serviceURL: &serviceURL, 71 | containerMapping: map[string]string{ 72 | "cc-buildpacks": buildpacksContainerName, 73 | "cc-resources": resourcesContainerName, 74 | "cc-droplets": dropletsContainerName, 75 | "cc-packages": packagesContainerName, 76 | }, 77 | } 78 | } 79 | 80 | func (s *azblobStore) Name() string { 81 | return "azure blob store" 82 | } 83 | 84 | func (s *azblobStore) List() ([]*Blob, error) { 85 | var blobs []*Blob 86 | 87 | containersOnAccount, err := s.listContainers() 88 | if err != nil { 89 | return nil, err 90 | } 91 | 92 | for _, container := range containers { 93 | containerName := s.destContainerName(container) 94 | containerUrl := s.serviceURL.NewContainerURL(containerName) 95 | 96 | containerExists := stringInArray(containersOnAccount, containerName) 97 | if containerExists { 98 | for marker := (azblob.Marker{}); marker.NotDone(); { 99 | listBlob, err := containerUrl.ListBlobsFlatSegment(context.Background(), 100 | marker, 101 | azblob.ListBlobsSegmentOptions{}) 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | marker = listBlob.NextMarker 107 | 108 | bar := pb.StartNew(len(listBlob.Segment.BlobItems)) 109 | for _, blobInfo := range listBlob.Segment.BlobItems { 110 | md5, err := base64.StdEncoding.DecodeString(string(blobInfo.Properties.ContentMD5[:])) 111 | if err != nil { 112 | return nil, err 113 | } 114 | checksum := hex.EncodeToString(md5) 115 | 116 | blob := &Blob{ 117 | Path: filepath.Join(container, blobInfo.Name), 118 | Checksum: checksum, 119 | } 120 | blobs = append(blobs, blob) 121 | 122 | bar.Increment() 123 | } 124 | bar.FinishPrint(containerName) 125 | } 126 | } 127 | } 128 | return blobs, nil 129 | } 130 | 131 | func (s *azblobStore) Read(src *Blob) (io.ReadCloser, error) { 132 | containerName := s.containerName(src) 133 | path := s.path(src) 134 | 135 | containerURL := s.serviceURL.NewContainerURL(containerName) 136 | blobURL := containerURL.NewBlockBlobURL(path) 137 | response, err := blobURL.Download(context.Background(), 138 | 0, 139 | azblob.CountToEnd, 140 | azblob.BlobAccessConditions{}, 141 | false) 142 | if err != nil { 143 | return nil, err 144 | } 145 | 146 | return response.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3}), nil 147 | } 148 | 149 | func (s *azblobStore) Checksum(src *Blob) (string, error) { 150 | // large blob does not have Content-MD5 in Metadata, 151 | // otherwise it will be faster to get checksum from metadata. 152 | // return s.checksumFromMetadata(src) 153 | rc, err := s.Read(src) 154 | if err != nil { 155 | return "", nil 156 | } 157 | defer rc.Close() 158 | 159 | return validation.ChecksumReader(rc) 160 | } 161 | 162 | func (s *azblobStore) Write(dst *Blob, src io.Reader) error { 163 | containerName := s.containerName(dst) 164 | path := s.path(dst) 165 | if err := s.createContainer(containerName); err != nil { 166 | return err 167 | } 168 | 169 | containerURL := s.serviceURL.NewContainerURL(containerName) 170 | blobURL := containerURL.NewBlockBlobURL(path) 171 | 172 | _, err := azblob.UploadStreamToBlockBlob(context.Background(), 173 | src, 174 | blobURL, 175 | azblob.UploadStreamToBlockBlobOptions{ 176 | BufferSize: 10 * 1024 * 1024, // 10M buffer size 177 | MaxBuffers: 20, 178 | }) 179 | if err != nil { 180 | return err 181 | } 182 | return nil 183 | } 184 | 185 | func (s *azblobStore) Exists(blob *Blob) bool { 186 | checksum, err := s.Checksum(blob) 187 | if err != nil { 188 | return false 189 | } 190 | 191 | return checksum == blob.Checksum 192 | } 193 | 194 | func (s *azblobStore) NewBucketIterator(containerName string) (BucketIterator, error) { 195 | destContainerName := s.destContainerName(containerName) 196 | containerExists, err := s.doesContainerExist(destContainerName) 197 | if err != nil { 198 | return nil, err 199 | } 200 | 201 | if !containerExists { 202 | return nil, errors.New("bucket does not exist") 203 | } 204 | 205 | containerUrl := s.serviceURL.NewContainerURL(destContainerName) 206 | 207 | marker := azblob.Marker{} 208 | listBlob, err := containerUrl.ListBlobsFlatSegment(context.Background(), 209 | marker, 210 | azblob.ListBlobsSegmentOptions{}) 211 | 212 | if err != nil { 213 | return nil, err 214 | } 215 | 216 | if len(listBlob.Segment.BlobItems) == 0 { 217 | return &s3BucketIterator{}, nil 218 | } 219 | 220 | doneCh := make(chan struct{}) 221 | blobCh := make(chan *Blob) 222 | 223 | iterator := &s3BucketIterator{ 224 | doneCh: doneCh, 225 | blobCh: blobCh, 226 | } 227 | 228 | go func() { 229 | for marker.NotDone() { 230 | for _, blobInfo := range listBlob.Segment.BlobItems { 231 | select { 232 | case <-doneCh: 233 | doneCh = nil 234 | return 235 | default: 236 | blobCh <- &Blob{ 237 | Path: filepath.Join(containerName, blobInfo.Name), 238 | } 239 | } 240 | } 241 | marker = listBlob.NextMarker 242 | } 243 | close(blobCh) 244 | }() 245 | 246 | return iterator, nil 247 | } 248 | 249 | // helpers 250 | 251 | func (s *azblobStore) destContainerName(container string) string { 252 | return s.containerMapping[container] 253 | } 254 | 255 | func (s *azblobStore) containerName(blob *Blob) string { 256 | return s.destContainerName(blob.Path[:strings.Index(blob.Path, "/")]) 257 | } 258 | 259 | func (s *azblobStore) doesContainerExist(containerName string) (bool, error) { 260 | containers, err := s.listContainers() 261 | if err != nil { 262 | return false, err 263 | } 264 | return stringInArray(containers, containerName), nil 265 | } 266 | 267 | func (s *azblobStore) createContainer(containerName string) error { 268 | containerURL := s.serviceURL.NewContainerURL(containerName) 269 | 270 | _, err := containerURL.Create(context.Background(), 271 | azblob.Metadata{}, 272 | azblob.PublicAccessNone) 273 | if serr, ok := err.(azblob.StorageError); ok { 274 | switch serr.ServiceCode() { 275 | case azblob.ServiceCodeContainerAlreadyExists: 276 | default: 277 | return err 278 | } 279 | } 280 | 281 | return nil 282 | } 283 | 284 | func (s *azblobStore) listContainers() ([]string, error) { 285 | var containers []string 286 | for marker := (azblob.Marker{}); marker.NotDone(); { 287 | l, err := s.serviceURL.ListContainersSegment( 288 | context.Background(), 289 | marker, 290 | azblob.ListContainersSegmentOptions{}) 291 | if err != nil { 292 | return nil, err 293 | } 294 | 295 | marker = l.NextMarker 296 | 297 | for _, c := range l.ContainerItems { 298 | containers = append(containers, c.Name) 299 | } 300 | } 301 | 302 | return containers, nil 303 | } 304 | func (s *azblobStore) path(blob *Blob) string { 305 | return blob.Path[(strings.Index(blob.Path, "/") + 1):] 306 | } 307 | 308 | func (s *azblobStore) checksumFromMetadata(src *Blob) (string, error) { 309 | containerName := s.containerName(src) 310 | path := s.path(src) 311 | 312 | containerURL := s.serviceURL.NewContainerURL(containerName) 313 | 314 | blobURL := containerURL.NewBlobURL(path) 315 | r, err := blobURL.GetProperties(context.Background(), azblob.BlobAccessConditions{}) 316 | if err != nil { 317 | return "", err 318 | } 319 | md5 := r.ContentMD5() 320 | 321 | return hex.EncodeToString(md5), nil 322 | } 323 | 324 | func stringInArray(arr []string, str string) bool { 325 | for _, a := range arr { 326 | if a == str { 327 | return true 328 | } 329 | } 330 | return false 331 | } 332 | -------------------------------------------------------------------------------- /blobstore/azblob_test.go: -------------------------------------------------------------------------------- 1 | package blobstore_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/url" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/Azure/azure-storage-blob-go/2018-03-28/azblob" 11 | "github.com/pivotal-cf/goblob/blobstore" 12 | 13 | . "github.com/onsi/ginkgo" 14 | . "github.com/onsi/gomega" 15 | ) 16 | 17 | var _ = Describe("azblobStore", func() { 18 | var ( 19 | cloudStorageEnpointsMap = map[string]string{ 20 | "AzureChinaCloud": "core.chinacloudapi.cn", 21 | "AzureCloud": "core.windows.net", 22 | "AzureGermanCloud": "core.cloudapi.de", 23 | "AzureUSGovernment": "core.usgovcloudapi.net", 24 | } 25 | ) 26 | 27 | accountName := os.Getenv("AZURE_STORAGE_ACCOUNT") 28 | accountKey := os.Getenv("AZURE_STORAGE_ACCOUNT_KEY") 29 | cloudName := os.Getenv("AZURE_CLOUD") 30 | if os.Getenv("AZURE_CLOUD") == "" { 31 | cloudName = "AzureCloud" 32 | } 33 | controlContainer := "some-buildpacks" 34 | 35 | blobStore := blobstore.NewAzBlobStore(accountName, accountKey, cloudName, "some-buildpacks", "some-droplets", "some-packages", "some-resources") 36 | 37 | credential := azblob.NewSharedKeyCredential(accountName, accountKey) 38 | pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{}) 39 | primaryURL, _ := url.Parse( 40 | fmt.Sprintf("https://%s.blob.%s", accountName, cloudStorageEnpointsMap[cloudName])) 41 | serviceURL := azblob.NewServiceURL(*primaryURL, pipeline) 42 | 43 | AfterSuite(func() { 44 | containerURL := serviceURL.NewContainerURL(controlContainer) 45 | _, _ = containerURL.Delete(context.Background(), azblob.ContainerAccessConditions{}) 46 | }) 47 | 48 | Describe("Name()", func() { 49 | It("Should return the name", func() { 50 | name := blobStore.Name() 51 | Expect(name).Should(BeEquivalentTo("azure blob store")) 52 | }) 53 | }) 54 | 55 | Describe("List()", func() { 56 | It("Should return list of files", func() { 57 | fileReader, err := os.Open("./s3_testdata/test.txt") 58 | Expect(err).ShouldNot(HaveOccurred()) 59 | for _, path := range []string{"cc-buildpacks/aa/bb", "cc-buildpacks/aa/cc", "cc-buildpacks/aa/dd"} { 60 | err := blobStore.Write(&blobstore.Blob{ 61 | Path: filepath.Join(path, "test.txt"), 62 | }, fileReader) 63 | Expect(err).ShouldNot(HaveOccurred()) 64 | } 65 | blobs, err := blobStore.List() 66 | Expect(err).ShouldNot(HaveOccurred()) 67 | Expect(len(blobs)).Should(BeEquivalentTo(3)) 68 | }) 69 | }) 70 | 71 | Describe("Read()", func() { 72 | It("Should read the file", func() { 73 | fileReader, err := os.Open("./s3_testdata/test.txt") 74 | Expect(err).ShouldNot(HaveOccurred()) 75 | writeErr := blobStore.Write(&blobstore.Blob{ 76 | Path: "cc-buildpacks/aa/bb/test.txt", 77 | }, fileReader) 78 | Expect(writeErr).ShouldNot(HaveOccurred()) 79 | reader, err := blobStore.Read(&blobstore.Blob{ 80 | Path: "cc-buildpacks/aa/bb/test.txt", 81 | }) 82 | Expect(err).ShouldNot(HaveOccurred()) 83 | Expect(reader).ShouldNot(BeNil()) 84 | }) 85 | }) 86 | 87 | Describe("Write()", func() { 88 | It("Should write to azure blob store with correct checksum", func() { 89 | reader, err := os.Open("./s3_testdata/test.txt") 90 | Expect(err).ShouldNot(HaveOccurred()) 91 | blob := &blobstore.Blob{ 92 | Path: "cc-buildpacks/aa/bb/test.txt", 93 | } 94 | err = blobStore.Write(blob, reader) 95 | Expect(err).ShouldNot(HaveOccurred()) 96 | 97 | checksum, err := blobStore.Checksum(blob) 98 | Expect(err).ShouldNot(HaveOccurred()) 99 | Expect(checksum).Should(BeEquivalentTo("d8e8fca2dc0f896fd7cb4cb0031ba249")) 100 | }) 101 | }) 102 | }) 103 | -------------------------------------------------------------------------------- /blobstore/blobstore.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobstore 16 | 17 | import "io" 18 | 19 | // Blob is a file in a blob store 20 | type Blob struct { 21 | Checksum string 22 | Path string 23 | } 24 | 25 | //go:generate counterfeiter . Blobstore 26 | 27 | type Blobstore interface { 28 | //Returns logical name of blobstore (S3, NFS) 29 | Name() string 30 | //Returns a list of all the "blobs" in blobstore 31 | List() ([]*Blob, error) 32 | //For a given blob will return io.ReadCloser with contents 33 | Read(src *Blob) (io.ReadCloser, error) 34 | //Returns md5 checksum for the given blob 35 | Checksum(src *Blob) (string, error) 36 | //Writes the blob to the blobstore 37 | Write(dst *Blob, src io.Reader) error 38 | //Determins if blob exists 39 | Exists(*Blob) bool 40 | //Returns an interator for all the blobs in the given bucket (or folder for NFS) 41 | NewBucketIterator(string) (BucketIterator, error) 42 | } 43 | -------------------------------------------------------------------------------- /blobstore/blobstore_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobstore_test 16 | 17 | import ( 18 | . "github.com/onsi/ginkgo" 19 | . "github.com/onsi/gomega" 20 | 21 | "testing" 22 | ) 23 | 24 | func TestBlobstore(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "Blobstore Suite") 27 | } 28 | -------------------------------------------------------------------------------- /blobstore/blobstorefakes/fake_blobstore.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This file was generated by counterfeiter 16 | package blobstorefakes 17 | 18 | import ( 19 | "io" 20 | "sync" 21 | 22 | "github.com/pivotal-cf/goblob/blobstore" 23 | ) 24 | 25 | type FakeBlobstore struct { 26 | NameStub func() string 27 | nameMutex sync.RWMutex 28 | nameArgsForCall []struct{} 29 | nameReturns struct { 30 | result1 string 31 | } 32 | ListStub func() ([]*blobstore.Blob, error) 33 | listMutex sync.RWMutex 34 | listArgsForCall []struct{} 35 | listReturns struct { 36 | result1 []*blobstore.Blob 37 | result2 error 38 | } 39 | ReadStub func(src *blobstore.Blob) (io.ReadCloser, error) 40 | readMutex sync.RWMutex 41 | readArgsForCall []struct { 42 | src *blobstore.Blob 43 | } 44 | readReturns struct { 45 | result1 io.ReadCloser 46 | result2 error 47 | } 48 | ChecksumStub func(src *blobstore.Blob) (string, error) 49 | checksumMutex sync.RWMutex 50 | checksumArgsForCall []struct { 51 | src *blobstore.Blob 52 | } 53 | checksumReturns struct { 54 | result1 string 55 | result2 error 56 | } 57 | WriteStub func(dst *blobstore.Blob, src io.Reader) error 58 | writeMutex sync.RWMutex 59 | writeArgsForCall []struct { 60 | dst *blobstore.Blob 61 | src io.Reader 62 | } 63 | writeReturns struct { 64 | result1 error 65 | } 66 | ExistsStub func(*blobstore.Blob) bool 67 | existsMutex sync.RWMutex 68 | existsArgsForCall []struct { 69 | arg1 *blobstore.Blob 70 | } 71 | existsReturns struct { 72 | result1 bool 73 | } 74 | NewBucketIteratorStub func(string) (blobstore.BucketIterator, error) 75 | newBucketIteratorMutex sync.RWMutex 76 | newBucketIteratorArgsForCall []struct { 77 | arg1 string 78 | } 79 | newBucketIteratorReturns struct { 80 | result1 blobstore.BucketIterator 81 | result2 error 82 | } 83 | invocations map[string][][]interface{} 84 | invocationsMutex sync.RWMutex 85 | } 86 | 87 | func (fake *FakeBlobstore) Name() string { 88 | fake.nameMutex.Lock() 89 | fake.nameArgsForCall = append(fake.nameArgsForCall, struct{}{}) 90 | fake.recordInvocation("Name", []interface{}{}) 91 | fake.nameMutex.Unlock() 92 | if fake.NameStub != nil { 93 | return fake.NameStub() 94 | } else { 95 | return fake.nameReturns.result1 96 | } 97 | } 98 | 99 | func (fake *FakeBlobstore) NameCallCount() int { 100 | fake.nameMutex.RLock() 101 | defer fake.nameMutex.RUnlock() 102 | return len(fake.nameArgsForCall) 103 | } 104 | 105 | func (fake *FakeBlobstore) NameReturns(result1 string) { 106 | fake.NameStub = nil 107 | fake.nameReturns = struct { 108 | result1 string 109 | }{result1} 110 | } 111 | 112 | func (fake *FakeBlobstore) List() ([]*blobstore.Blob, error) { 113 | fake.listMutex.Lock() 114 | fake.listArgsForCall = append(fake.listArgsForCall, struct{}{}) 115 | fake.recordInvocation("List", []interface{}{}) 116 | fake.listMutex.Unlock() 117 | if fake.ListStub != nil { 118 | return fake.ListStub() 119 | } else { 120 | return fake.listReturns.result1, fake.listReturns.result2 121 | } 122 | } 123 | 124 | func (fake *FakeBlobstore) ListCallCount() int { 125 | fake.listMutex.RLock() 126 | defer fake.listMutex.RUnlock() 127 | return len(fake.listArgsForCall) 128 | } 129 | 130 | func (fake *FakeBlobstore) ListReturns(result1 []*blobstore.Blob, result2 error) { 131 | fake.ListStub = nil 132 | fake.listReturns = struct { 133 | result1 []*blobstore.Blob 134 | result2 error 135 | }{result1, result2} 136 | } 137 | 138 | func (fake *FakeBlobstore) Read(src *blobstore.Blob) (io.ReadCloser, error) { 139 | fake.readMutex.Lock() 140 | fake.readArgsForCall = append(fake.readArgsForCall, struct { 141 | src *blobstore.Blob 142 | }{src}) 143 | fake.recordInvocation("Read", []interface{}{src}) 144 | fake.readMutex.Unlock() 145 | if fake.ReadStub != nil { 146 | return fake.ReadStub(src) 147 | } else { 148 | return fake.readReturns.result1, fake.readReturns.result2 149 | } 150 | } 151 | 152 | func (fake *FakeBlobstore) ReadCallCount() int { 153 | fake.readMutex.RLock() 154 | defer fake.readMutex.RUnlock() 155 | return len(fake.readArgsForCall) 156 | } 157 | 158 | func (fake *FakeBlobstore) ReadArgsForCall(i int) *blobstore.Blob { 159 | fake.readMutex.RLock() 160 | defer fake.readMutex.RUnlock() 161 | return fake.readArgsForCall[i].src 162 | } 163 | 164 | func (fake *FakeBlobstore) ReadReturns(result1 io.ReadCloser, result2 error) { 165 | fake.ReadStub = nil 166 | fake.readReturns = struct { 167 | result1 io.ReadCloser 168 | result2 error 169 | }{result1, result2} 170 | } 171 | 172 | func (fake *FakeBlobstore) Checksum(src *blobstore.Blob) (string, error) { 173 | fake.checksumMutex.Lock() 174 | fake.checksumArgsForCall = append(fake.checksumArgsForCall, struct { 175 | src *blobstore.Blob 176 | }{src}) 177 | fake.recordInvocation("Checksum", []interface{}{src}) 178 | fake.checksumMutex.Unlock() 179 | if fake.ChecksumStub != nil { 180 | return fake.ChecksumStub(src) 181 | } else { 182 | return fake.checksumReturns.result1, fake.checksumReturns.result2 183 | } 184 | } 185 | 186 | func (fake *FakeBlobstore) ChecksumCallCount() int { 187 | fake.checksumMutex.RLock() 188 | defer fake.checksumMutex.RUnlock() 189 | return len(fake.checksumArgsForCall) 190 | } 191 | 192 | func (fake *FakeBlobstore) ChecksumArgsForCall(i int) *blobstore.Blob { 193 | fake.checksumMutex.RLock() 194 | defer fake.checksumMutex.RUnlock() 195 | return fake.checksumArgsForCall[i].src 196 | } 197 | 198 | func (fake *FakeBlobstore) ChecksumReturns(result1 string, result2 error) { 199 | fake.ChecksumStub = nil 200 | fake.checksumReturns = struct { 201 | result1 string 202 | result2 error 203 | }{result1, result2} 204 | } 205 | 206 | func (fake *FakeBlobstore) Write(dst *blobstore.Blob, src io.Reader) error { 207 | fake.writeMutex.Lock() 208 | fake.writeArgsForCall = append(fake.writeArgsForCall, struct { 209 | dst *blobstore.Blob 210 | src io.Reader 211 | }{dst, src}) 212 | fake.recordInvocation("Write", []interface{}{dst, src}) 213 | fake.writeMutex.Unlock() 214 | if fake.WriteStub != nil { 215 | return fake.WriteStub(dst, src) 216 | } else { 217 | return fake.writeReturns.result1 218 | } 219 | } 220 | 221 | func (fake *FakeBlobstore) WriteCallCount() int { 222 | fake.writeMutex.RLock() 223 | defer fake.writeMutex.RUnlock() 224 | return len(fake.writeArgsForCall) 225 | } 226 | 227 | func (fake *FakeBlobstore) WriteArgsForCall(i int) (*blobstore.Blob, io.Reader) { 228 | fake.writeMutex.RLock() 229 | defer fake.writeMutex.RUnlock() 230 | return fake.writeArgsForCall[i].dst, fake.writeArgsForCall[i].src 231 | } 232 | 233 | func (fake *FakeBlobstore) WriteReturns(result1 error) { 234 | fake.WriteStub = nil 235 | fake.writeReturns = struct { 236 | result1 error 237 | }{result1} 238 | } 239 | 240 | func (fake *FakeBlobstore) Exists(arg1 *blobstore.Blob) bool { 241 | fake.existsMutex.Lock() 242 | fake.existsArgsForCall = append(fake.existsArgsForCall, struct { 243 | arg1 *blobstore.Blob 244 | }{arg1}) 245 | fake.recordInvocation("Exists", []interface{}{arg1}) 246 | fake.existsMutex.Unlock() 247 | if fake.ExistsStub != nil { 248 | return fake.ExistsStub(arg1) 249 | } else { 250 | return fake.existsReturns.result1 251 | } 252 | } 253 | 254 | func (fake *FakeBlobstore) ExistsCallCount() int { 255 | fake.existsMutex.RLock() 256 | defer fake.existsMutex.RUnlock() 257 | return len(fake.existsArgsForCall) 258 | } 259 | 260 | func (fake *FakeBlobstore) ExistsArgsForCall(i int) *blobstore.Blob { 261 | fake.existsMutex.RLock() 262 | defer fake.existsMutex.RUnlock() 263 | return fake.existsArgsForCall[i].arg1 264 | } 265 | 266 | func (fake *FakeBlobstore) ExistsReturns(result1 bool) { 267 | fake.ExistsStub = nil 268 | fake.existsReturns = struct { 269 | result1 bool 270 | }{result1} 271 | } 272 | 273 | func (fake *FakeBlobstore) NewBucketIterator(arg1 string) (blobstore.BucketIterator, error) { 274 | fake.newBucketIteratorMutex.Lock() 275 | fake.newBucketIteratorArgsForCall = append(fake.newBucketIteratorArgsForCall, struct { 276 | arg1 string 277 | }{arg1}) 278 | fake.recordInvocation("NewBucketIterator", []interface{}{arg1}) 279 | fake.newBucketIteratorMutex.Unlock() 280 | if fake.NewBucketIteratorStub != nil { 281 | return fake.NewBucketIteratorStub(arg1) 282 | } else { 283 | return fake.newBucketIteratorReturns.result1, fake.newBucketIteratorReturns.result2 284 | } 285 | } 286 | 287 | func (fake *FakeBlobstore) NewBucketIteratorCallCount() int { 288 | fake.newBucketIteratorMutex.RLock() 289 | defer fake.newBucketIteratorMutex.RUnlock() 290 | return len(fake.newBucketIteratorArgsForCall) 291 | } 292 | 293 | func (fake *FakeBlobstore) NewBucketIteratorArgsForCall(i int) string { 294 | fake.newBucketIteratorMutex.RLock() 295 | defer fake.newBucketIteratorMutex.RUnlock() 296 | return fake.newBucketIteratorArgsForCall[i].arg1 297 | } 298 | 299 | func (fake *FakeBlobstore) NewBucketIteratorReturns(result1 blobstore.BucketIterator, result2 error) { 300 | fake.NewBucketIteratorStub = nil 301 | fake.newBucketIteratorReturns = struct { 302 | result1 blobstore.BucketIterator 303 | result2 error 304 | }{result1, result2} 305 | } 306 | 307 | func (fake *FakeBlobstore) Invocations() map[string][][]interface{} { 308 | fake.invocationsMutex.RLock() 309 | defer fake.invocationsMutex.RUnlock() 310 | fake.nameMutex.RLock() 311 | defer fake.nameMutex.RUnlock() 312 | fake.listMutex.RLock() 313 | defer fake.listMutex.RUnlock() 314 | fake.readMutex.RLock() 315 | defer fake.readMutex.RUnlock() 316 | fake.checksumMutex.RLock() 317 | defer fake.checksumMutex.RUnlock() 318 | fake.writeMutex.RLock() 319 | defer fake.writeMutex.RUnlock() 320 | fake.existsMutex.RLock() 321 | defer fake.existsMutex.RUnlock() 322 | fake.newBucketIteratorMutex.RLock() 323 | defer fake.newBucketIteratorMutex.RUnlock() 324 | return fake.invocations 325 | } 326 | 327 | func (fake *FakeBlobstore) recordInvocation(key string, args []interface{}) { 328 | fake.invocationsMutex.Lock() 329 | defer fake.invocationsMutex.Unlock() 330 | if fake.invocations == nil { 331 | fake.invocations = map[string][][]interface{}{} 332 | } 333 | if fake.invocations[key] == nil { 334 | fake.invocations[key] = [][]interface{}{} 335 | } 336 | fake.invocations[key] = append(fake.invocations[key], args) 337 | } 338 | 339 | var _ blobstore.Blobstore = new(FakeBlobstore) 340 | -------------------------------------------------------------------------------- /blobstore/blobstorefakes/fake_bucket_iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This file was generated by counterfeiter 16 | package blobstorefakes 17 | 18 | import ( 19 | "sync" 20 | 21 | "github.com/pivotal-cf/goblob/blobstore" 22 | ) 23 | 24 | type FakeBucketIterator struct { 25 | NextStub func() (*blobstore.Blob, error) 26 | nextMutex sync.RWMutex 27 | nextArgsForCall []struct{} 28 | nextReturns struct { 29 | result1 *blobstore.Blob 30 | result2 error 31 | } 32 | DoneStub func() 33 | doneMutex sync.RWMutex 34 | doneArgsForCall []struct{} 35 | invocations map[string][][]interface{} 36 | invocationsMutex sync.RWMutex 37 | } 38 | 39 | func (fake *FakeBucketIterator) Next() (*blobstore.Blob, error) { 40 | fake.nextMutex.Lock() 41 | fake.nextArgsForCall = append(fake.nextArgsForCall, struct{}{}) 42 | fake.recordInvocation("Next", []interface{}{}) 43 | fake.nextMutex.Unlock() 44 | if fake.NextStub != nil { 45 | return fake.NextStub() 46 | } else { 47 | return fake.nextReturns.result1, fake.nextReturns.result2 48 | } 49 | } 50 | 51 | func (fake *FakeBucketIterator) NextCallCount() int { 52 | fake.nextMutex.RLock() 53 | defer fake.nextMutex.RUnlock() 54 | return len(fake.nextArgsForCall) 55 | } 56 | 57 | func (fake *FakeBucketIterator) NextReturns(result1 *blobstore.Blob, result2 error) { 58 | fake.NextStub = nil 59 | fake.nextReturns = struct { 60 | result1 *blobstore.Blob 61 | result2 error 62 | }{result1, result2} 63 | } 64 | 65 | func (fake *FakeBucketIterator) Done() { 66 | fake.doneMutex.Lock() 67 | fake.doneArgsForCall = append(fake.doneArgsForCall, struct{}{}) 68 | fake.recordInvocation("Done", []interface{}{}) 69 | fake.doneMutex.Unlock() 70 | if fake.DoneStub != nil { 71 | fake.DoneStub() 72 | } 73 | } 74 | 75 | func (fake *FakeBucketIterator) DoneCallCount() int { 76 | fake.doneMutex.RLock() 77 | defer fake.doneMutex.RUnlock() 78 | return len(fake.doneArgsForCall) 79 | } 80 | 81 | func (fake *FakeBucketIterator) Invocations() map[string][][]interface{} { 82 | fake.invocationsMutex.RLock() 83 | defer fake.invocationsMutex.RUnlock() 84 | fake.nextMutex.RLock() 85 | defer fake.nextMutex.RUnlock() 86 | fake.doneMutex.RLock() 87 | defer fake.doneMutex.RUnlock() 88 | return fake.invocations 89 | } 90 | 91 | func (fake *FakeBucketIterator) recordInvocation(key string, args []interface{}) { 92 | fake.invocationsMutex.Lock() 93 | defer fake.invocationsMutex.Unlock() 94 | if fake.invocations == nil { 95 | fake.invocations = map[string][][]interface{}{} 96 | } 97 | if fake.invocations[key] == nil { 98 | fake.invocations[key] = [][]interface{}{} 99 | } 100 | fake.invocations[key] = append(fake.invocations[key], args) 101 | } 102 | 103 | var _ blobstore.BucketIterator = new(FakeBucketIterator) 104 | -------------------------------------------------------------------------------- /blobstore/bucket_iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobstore 16 | 17 | import "errors" 18 | 19 | var ErrIteratorDone = errors.New("no more items in iterator") 20 | var ErrIteratorAborted = errors.New("iterator aborted") 21 | 22 | //go:generate counterfeiter . BucketIterator 23 | 24 | type BucketIterator interface { 25 | Next() (*Blob, error) 26 | Done() 27 | } 28 | -------------------------------------------------------------------------------- /blobstore/nfs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobstore 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "io" 21 | "io/ioutil" 22 | "os" 23 | "path" 24 | "path/filepath" 25 | "strings" 26 | 27 | "github.com/cheggaaa/pb" 28 | "github.com/pivotal-cf/goblob/validation" 29 | "golang.org/x/sync/errgroup" 30 | ) 31 | 32 | type nfsStore struct { 33 | path string 34 | } 35 | 36 | // NewNFS creates an NFS blobstore 37 | func NewNFS(path string) Blobstore { 38 | return &nfsStore{ 39 | path: path, 40 | } 41 | } 42 | 43 | func (s *nfsStore) Name() string { 44 | return "NFS" 45 | } 46 | 47 | // List fetches a list of files with checksums 48 | func (s *nfsStore) List() ([]*Blob, error) { 49 | var blobs []*Blob 50 | walk := func(path string, info os.FileInfo, e error) error { 51 | if !info.IsDir() && info.Name() != ".nfs_test" { 52 | relPath := path[len(s.path)+1:] 53 | blobs = append(blobs, &Blob{ 54 | Path: relPath, 55 | }) 56 | } 57 | return e 58 | } 59 | if err := filepath.Walk(s.path, walk); err != nil { 60 | return nil, err 61 | } 62 | if err := s.processBlobsForChecksums(blobs); err != nil { 63 | return nil, err 64 | } 65 | return blobs, nil 66 | } 67 | 68 | func (s *nfsStore) processBlobsForChecksums(blobs []*Blob) error { 69 | 70 | fmt.Println("Getting list of files from NFS") 71 | bar := pb.StartNew(len(blobs)) 72 | bar.Format("<.- >") 73 | 74 | var g errgroup.Group 75 | for _, blob := range blobs { 76 | blob := blob 77 | g.Go(func() error { 78 | checksum, err := s.Checksum(blob) 79 | if (err) != nil { 80 | return err 81 | } 82 | blob.Checksum = checksum 83 | bar.Increment() 84 | return nil 85 | }) 86 | } 87 | if err := g.Wait(); err != nil { 88 | return err 89 | } 90 | 91 | bar.FinishPrint("Done Getting list of files from NFS") 92 | return nil 93 | } 94 | 95 | func (s *nfsStore) Checksum(src *Blob) (string, error) { 96 | return validation.Checksum(path.Join(s.path, src.Path)) 97 | } 98 | 99 | func (s *nfsStore) Read(src *Blob) (io.ReadCloser, error) { 100 | return os.Open(path.Join(s.path, src.Path)) 101 | } 102 | func (s *nfsStore) Write(dst *Blob, src io.Reader) error { 103 | return errors.New("writing to the NFS store is not supported") 104 | } 105 | 106 | func (s *nfsStore) Exists(blob *Blob) bool { 107 | checksum, err := s.Checksum(blob) 108 | if err != nil { 109 | return false 110 | } 111 | 112 | return checksum == blob.Checksum 113 | } 114 | 115 | func (s *nfsStore) NewBucketIterator(folder string) (BucketIterator, error) { 116 | blobCh := make(chan *Blob) 117 | doneCh := make(chan struct{}) 118 | errCh := make(chan error) 119 | 120 | actualPath := filepath.Join(s.path, folder) 121 | 122 | files, err := ioutil.ReadDir(actualPath) 123 | if err != nil { 124 | return nil, err 125 | } 126 | 127 | if len(files) == 0 { 128 | return &nfsBucketIterator{}, nil 129 | } 130 | 131 | iterator := &nfsBucketIterator{ 132 | blobCh: blobCh, 133 | doneCh: doneCh, 134 | errCh: errCh, 135 | } 136 | 137 | walkFn := func(path string, info os.FileInfo, err error) error { 138 | if err != nil { 139 | return err 140 | } 141 | 142 | if info.IsDir() || info.Name() == ".nfs_test" { 143 | return nil 144 | } 145 | 146 | select { 147 | case <-doneCh: 148 | doneCh = nil 149 | return ErrIteratorAborted 150 | default: 151 | blob := &Blob{ 152 | Path: strings.TrimPrefix(path, s.path+string(os.PathSeparator)), 153 | } 154 | 155 | blobCh <- blob 156 | 157 | return nil 158 | } 159 | } 160 | 161 | go func() { 162 | errCh <- filepath.Walk(actualPath, walkFn) 163 | close(blobCh) 164 | }() 165 | 166 | return iterator, nil 167 | } 168 | -------------------------------------------------------------------------------- /blobstore/nfs_bucket_iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobstore 16 | 17 | type nfsBucketIterator struct { 18 | blobCh chan *Blob 19 | doneCh chan struct{} 20 | errCh chan error 21 | } 22 | 23 | func (i *nfsBucketIterator) Next() (*Blob, error) { 24 | if i.blobCh == nil { 25 | return nil, ErrIteratorDone 26 | } 27 | 28 | select { 29 | case blob, ok := <-i.blobCh: 30 | if !ok { 31 | i.blobCh = nil 32 | return nil, ErrIteratorDone 33 | } 34 | 35 | return blob, nil 36 | case err := <-i.errCh: 37 | if err != nil { 38 | return nil, err 39 | } 40 | return nil, ErrIteratorDone 41 | } 42 | } 43 | 44 | func (i *nfsBucketIterator) Done() { 45 | i.blobCh = nil 46 | close(i.doneCh) 47 | } 48 | -------------------------------------------------------------------------------- /blobstore/nfs_bucket_iterator_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobstore_test 16 | 17 | import ( 18 | "io/ioutil" 19 | "os" 20 | "path/filepath" 21 | 22 | "github.com/pivotal-cf/goblob/blobstore" 23 | . "github.com/onsi/ginkgo" 24 | . "github.com/onsi/gomega" 25 | ) 26 | 27 | var _ = Describe("NFSBucketIterator", func() { 28 | var ( 29 | iterator blobstore.BucketIterator 30 | store blobstore.Blobstore 31 | baseDir string 32 | ) 33 | 34 | BeforeEach(func() { 35 | var err error 36 | baseDir, err = ioutil.TempDir("", "nfs-bucket-iterator-test") 37 | Expect(err).NotTo(HaveOccurred()) 38 | 39 | store = blobstore.NewNFS(baseDir) 40 | 41 | err = os.MkdirAll(filepath.Join(baseDir, "some-bucket"), os.ModePerm) 42 | Expect(err).NotTo(HaveOccurred()) 43 | 44 | iterator, err = store.NewBucketIterator("some-bucket") 45 | Expect(err).NotTo(HaveOccurred()) 46 | }) 47 | 48 | AfterEach(func() { 49 | os.RemoveAll(baseDir) 50 | }) 51 | 52 | Describe("Next", func() { 53 | It("returns an error", func() { 54 | _, err := iterator.Next() 55 | Expect(err).To(HaveOccurred()) 56 | Expect(err.Error()).To(Equal("no more items in iterator")) 57 | }) 58 | 59 | Context("when a blob exists in the bucket", func() { 60 | var expectedBlob blobstore.Blob 61 | 62 | BeforeEach(func() { 63 | expectedBlob = blobstore.Blob{ 64 | Path: "some-bucket/some-path/some-file", 65 | } 66 | 67 | err := os.MkdirAll(filepath.Join(baseDir, "some-bucket", "some-path"), os.ModePerm) 68 | Expect(err).NotTo(HaveOccurred()) 69 | 70 | err = ioutil.WriteFile( 71 | filepath.Join(baseDir, "some-bucket", "some-path", "some-file"), 72 | []byte("content"), 73 | os.ModePerm, 74 | ) 75 | Expect(err).NotTo(HaveOccurred()) 76 | 77 | iterator, err = store.NewBucketIterator("some-bucket") 78 | Expect(err).NotTo(HaveOccurred()) 79 | }) 80 | 81 | It("returns the blob", func() { 82 | blob, err := iterator.Next() 83 | Expect(err).NotTo(HaveOccurred()) 84 | Expect(*blob).To(Equal(expectedBlob)) 85 | }) 86 | 87 | It("returns an error when all blobs have been listed", func() { 88 | _, err := iterator.Next() 89 | Expect(err).NotTo(HaveOccurred()) 90 | 91 | _, err = iterator.Next() 92 | Expect(err).To(HaveOccurred()) 93 | Expect(err.Error()).To(Equal("no more items in iterator")) 94 | }) 95 | }) 96 | 97 | Context("when a file named .nfs_test exists in the bucket", func() { 98 | BeforeEach(func() { 99 | err := os.MkdirAll(filepath.Join(baseDir, "some-bucket", "some-path"), os.ModePerm) 100 | Expect(err).NotTo(HaveOccurred()) 101 | 102 | err = ioutil.WriteFile( 103 | filepath.Join(baseDir, "some-bucket", "some-path", ".nfs_test"), 104 | []byte{}, 105 | os.ModePerm, 106 | ) 107 | Expect(err).NotTo(HaveOccurred()) 108 | 109 | err = ioutil.WriteFile( 110 | filepath.Join(baseDir, "some-bucket", "some-path", "some-file"), 111 | []byte("content"), 112 | os.ModePerm, 113 | ) 114 | Expect(err).NotTo(HaveOccurred()) 115 | 116 | iterator, err = store.NewBucketIterator("some-bucket") 117 | Expect(err).NotTo(HaveOccurred()) 118 | }) 119 | 120 | It("skips the file", func() { 121 | _, err := iterator.Next() 122 | Expect(err).NotTo(HaveOccurred()) 123 | 124 | _, err = iterator.Next() 125 | Expect(err).To(HaveOccurred()) 126 | Expect(err.Error()).To(Equal("no more items in iterator")) 127 | }) 128 | }) 129 | }) 130 | 131 | Describe("Done", func() { 132 | Context("when blobs exist in the bucket", func() { 133 | BeforeEach(func() { 134 | err := os.MkdirAll(filepath.Join(baseDir, "some-bucket", "some-path"), os.ModePerm) 135 | Expect(err).NotTo(HaveOccurred()) 136 | 137 | err = ioutil.WriteFile( 138 | filepath.Join(baseDir, "some-bucket", "some-path", "some-file"), 139 | []byte("content"), 140 | os.ModePerm, 141 | ) 142 | Expect(err).NotTo(HaveOccurred()) 143 | 144 | err = ioutil.WriteFile( 145 | filepath.Join(baseDir, "some-bucket", "some-path", "some-other-file"), 146 | []byte("content"), 147 | os.ModePerm, 148 | ) 149 | Expect(err).NotTo(HaveOccurred()) 150 | 151 | iterator, err = store.NewBucketIterator("some-bucket") 152 | Expect(err).NotTo(HaveOccurred()) 153 | }) 154 | 155 | It("causes Next to return an error", func() { 156 | iterator.Done() 157 | 158 | _, err := iterator.Next() 159 | Expect(err).To(HaveOccurred()) 160 | Expect(err.Error()).To(Equal("no more items in iterator")) 161 | }) 162 | }) 163 | }) 164 | }) 165 | -------------------------------------------------------------------------------- /blobstore/nfs_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobstore_test 16 | 17 | import ( 18 | "errors" 19 | 20 | "github.com/pivotal-cf/goblob/blobstore" 21 | 22 | . "github.com/onsi/ginkgo" 23 | . "github.com/onsi/gomega" 24 | ) 25 | 26 | var _ = Describe("NFS", func() { 27 | var store blobstore.Blobstore 28 | BeforeEach(func() { 29 | store = blobstore.NewNFS("nfs_testdata") 30 | }) 31 | 32 | Describe("List()", func() { 33 | It("Should return a list of blobs", func() { 34 | blobs, err := store.List() 35 | Ω(err).ShouldNot(HaveOccurred()) 36 | Ω(len(blobs)).Should(BeEquivalentTo(7)) 37 | Ω(blobs[0].Checksum).Should(BeEquivalentTo("b026324c6904b2a9cb4b88d6d61c81d1")) 38 | Ω(blobs[0].Path).Should(BeEquivalentTo("cc-buildpacks/ea/07/ea07de9b-dd94-477c-b904-0f77d47dd111_a32d9ae40371d557c7c90eb2affc3d7bba6abe69")) 39 | }) 40 | }) 41 | Describe("Read()", func() { 42 | It("Given a file it should return a reader", func() { 43 | blobs, err := store.List() 44 | Ω(err).ShouldNot(HaveOccurred()) 45 | 46 | reader, err := store.Read(blobs[0]) 47 | Ω(err).ShouldNot(HaveOccurred()) 48 | Ω(reader).ShouldNot(BeNil()) 49 | }) 50 | }) 51 | Describe("Write()", func() { 52 | It("Should return an error", func() { 53 | err := errors.New("writing to the NFS store is not supported") 54 | Ω(store.Write(&blobstore.Blob{}, nil)).Should(BeEquivalentTo(err)) 55 | }) 56 | }) 57 | }) 58 | -------------------------------------------------------------------------------- /blobstore/nfs_testdata/.nfs_test: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware-archive/goblob/78c90e7779bab204ecb7c8c05b204c52cf8af2da/blobstore/nfs_testdata/.nfs_test -------------------------------------------------------------------------------- /blobstore/nfs_testdata/cc-buildpacks/ea/07/ea07de9b-dd94-477c-b904-0f77d47dd111_a32d9ae40371d557c7c90eb2affc3d7bba6abe69: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /blobstore/nfs_testdata/cc-droplets/61/7c/617cf88e-da7b-4722-be53-dad7130514c1/047363432e6b22f6cfc10be16de2d7479dfe2ee3: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /blobstore/nfs_testdata/cc-droplets/buildpack_cache/f3/9a/f39a3f10-b3fd-4dc4-aa05-54db6c34c296/cflinuxfs2: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /blobstore/nfs_testdata/cc-packages/1a/94/1a94dd34-fb36-47b8-a0af-682a22a94874: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /blobstore/nfs_testdata/cc-packages/f1/2c/f12c3b36-4626-4c4b-b441-8dc88407bfae: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /blobstore/nfs_testdata/cc-resources/0a/1e/0a1eea6d7bb903d8c075688534480e87d4151470: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /blobstore/nfs_testdata/cc-resources/0d/78/0d7896e2bb23f88e26e52b22a075350b354df447: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /blobstore/s3.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobstore 16 | 17 | import ( 18 | "crypto/tls" 19 | "errors" 20 | "fmt" 21 | "io" 22 | "net/http" 23 | "path/filepath" 24 | "strings" 25 | 26 | "github.com/aws/aws-sdk-go/aws" 27 | "github.com/aws/aws-sdk-go/aws/credentials" 28 | "github.com/aws/aws-sdk-go/aws/session" 29 | awss3 "github.com/aws/aws-sdk-go/service/s3" 30 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 31 | "github.com/cheggaaa/pb" 32 | "github.com/pivotal-cf/goblob/validation" 33 | "github.com/xchapter7x/lo" 34 | ) 35 | 36 | var ( 37 | buckets = []string{"cc-buildpacks", "cc-droplets", "cc-packages", "cc-resources"} 38 | ) 39 | 40 | type s3Store struct { 41 | session *session.Session 42 | useMultipartUploads bool 43 | bucketMapping map[string]string 44 | } 45 | 46 | func NewS3( 47 | awsAccessKey string, 48 | awsSecretKey string, 49 | region string, 50 | endpoint string, 51 | useMultipartUploads bool, 52 | disableSSL bool, 53 | insecureSkipVerify bool, 54 | buildpacksBucketName string, 55 | dropletsBucketName string, 56 | packagesBucketName string, 57 | resourcesBucketName string, 58 | ) Blobstore { 59 | httpClient := &http.Client{ 60 | Transport: &http.Transport{ 61 | TLSClientConfig: &tls.Config{ 62 | InsecureSkipVerify: insecureSkipVerify, 63 | }, 64 | }, 65 | } 66 | 67 | return &s3Store{ 68 | session: session.New(&aws.Config{ 69 | HTTPClient: httpClient, 70 | Region: aws.String(region), 71 | Credentials: credentials.NewStaticCredentials(awsAccessKey, awsSecretKey, ""), 72 | Endpoint: aws.String(endpoint), 73 | DisableSSL: aws.Bool(disableSSL), 74 | S3ForcePathStyle: aws.Bool(true), 75 | }), 76 | useMultipartUploads: useMultipartUploads, 77 | bucketMapping: map[string]string{ 78 | "cc-buildpacks": buildpacksBucketName, 79 | "cc-resources": resourcesBucketName, 80 | "cc-droplets": dropletsBucketName, 81 | "cc-packages": packagesBucketName, 82 | }, 83 | } 84 | } 85 | 86 | func (s *s3Store) Name() string { 87 | return "S3" 88 | } 89 | 90 | func (s *s3Store) destBucketName(bucket string) string { 91 | return s.bucketMapping[bucket] 92 | } 93 | 94 | func (s *s3Store) List() ([]*Blob, error) { 95 | var blobs []*Blob 96 | s3Service := awss3.New(s.session) 97 | for _, bucket := range buckets { 98 | bucketName := s.destBucketName(bucket) 99 | bucketExists, err := s.doesBucketExist(bucketName) 100 | if err != nil { 101 | return nil, err 102 | } 103 | if bucketExists { 104 | listObjectsOutput, err := s3Service.ListObjects(&awss3.ListObjectsInput{ 105 | Bucket: aws.String(bucketName), 106 | }) 107 | if err != nil { 108 | return nil, err 109 | } 110 | fmt.Println("Getting list of files from S3", bucketName) 111 | bar := pb.StartNew(len(listObjectsOutput.Contents)) 112 | bar.Format("<.- >") 113 | for _, item := range listObjectsOutput.Contents { 114 | blob := &Blob{ 115 | Path: filepath.Join(bucket, *item.Key), 116 | } 117 | 118 | checksum, err := s.checksumFromMetadata(blob) 119 | if err != nil { 120 | return nil, err 121 | } 122 | blob.Checksum = checksum 123 | blobs = append(blobs, blob) 124 | bar.Increment() 125 | } 126 | bar.FinishPrint(fmt.Sprintf("Done Getting list of files from S3 %s", bucketName)) 127 | } 128 | } 129 | return blobs, nil 130 | } 131 | 132 | func (s *s3Store) bucketName(blob *Blob) string { 133 | return s.destBucketName(blob.Path[:strings.Index(blob.Path, "/")]) 134 | } 135 | 136 | func (s *s3Store) path(blob *Blob) string { 137 | bucketName := blob.Path[:strings.Index(blob.Path, "/")] 138 | return blob.Path[len(bucketName)+1:] 139 | } 140 | 141 | func (s *s3Store) Checksum(src *Blob) (string, error) { 142 | if s.useMultipartUploads { 143 | getObjectOutput, err := awss3.New(s.session).GetObject(&awss3.GetObjectInput{ 144 | Bucket: aws.String(s.bucketName(src)), 145 | Key: aws.String(s.path(src)), 146 | }) 147 | if err != nil { 148 | return "", err 149 | } 150 | defer getObjectOutput.Body.Close() 151 | return validation.ChecksumReader(getObjectOutput.Body) 152 | } 153 | 154 | return s.checksumFromETAG(src) 155 | } 156 | 157 | func (s *s3Store) checksumFromETAG(src *Blob) (string, error) { 158 | headObjectOutput, err := awss3.New(s.session).HeadObject(&awss3.HeadObjectInput{ 159 | Bucket: aws.String(s.bucketName(src)), 160 | Key: aws.String(s.path(src)), 161 | }) 162 | if err != nil { 163 | return "", err 164 | } 165 | return strings.Replace(*headObjectOutput.ETag, "\"", "", -1), nil 166 | } 167 | 168 | func (s *s3Store) checksumFromMetadata(src *Blob) (string, error) { 169 | headObjectOutput, err := awss3.New(s.session).HeadObject(&awss3.HeadObjectInput{ 170 | Bucket: aws.String(s.bucketName(src)), 171 | Key: aws.String(s.path(src)), 172 | }) 173 | if err != nil { 174 | return "", err 175 | } 176 | value, ok := headObjectOutput.Metadata["Checksum"] 177 | if ok { 178 | return *value, nil 179 | } 180 | 181 | return "", nil 182 | } 183 | 184 | func (s *s3Store) Read(src *Blob) (io.ReadCloser, error) { 185 | bucketName := s.bucketName(src) 186 | path := s.path(src) 187 | lo.G.Debug("Getting", path, "from bucket", bucketName) 188 | getObjectOutput, err := awss3.New(s.session).GetObject(&awss3.GetObjectInput{ 189 | Bucket: aws.String(bucketName), 190 | Key: aws.String(path), 191 | }) 192 | if err != nil { 193 | return nil, err 194 | } 195 | return getObjectOutput.Body, nil 196 | 197 | } 198 | 199 | func (s *s3Store) Write(dst *Blob, src io.Reader) error { 200 | bucketName := s.bucketName(dst) 201 | path := s.path(dst) 202 | if err := s.createBucket(bucketName); err != nil { 203 | return err 204 | } 205 | metadataMap := map[string]*string{ 206 | "Checksum": &dst.Checksum, 207 | } 208 | if s.useMultipartUploads { 209 | uploader := s3manager.NewUploader(s.session) 210 | _, err := uploader.Upload(&s3manager.UploadInput{ 211 | Body: src, 212 | Bucket: aws.String(bucketName), 213 | Key: aws.String(path), 214 | Metadata: metadataMap, 215 | }, func(u *s3manager.Uploader) { 216 | u.PartSize = 10 * 1024 * 1024 // 10MB part size 217 | u.Concurrency = 20 218 | }) 219 | if err != nil { 220 | return err 221 | } 222 | } else { 223 | _, err := awss3.New(s.session).PutObject(&awss3.PutObjectInput{ 224 | Body: aws.ReadSeekCloser(src), 225 | Bucket: aws.String(bucketName), 226 | Key: aws.String(path), 227 | Metadata: metadataMap, 228 | }) 229 | if err != nil { 230 | return err 231 | } 232 | } 233 | 234 | return nil 235 | } 236 | 237 | func (s *s3Store) doesBucketExist(bucketName string) (bool, error) { 238 | var listBucketOutput *awss3.ListBucketsOutput 239 | var err error 240 | s3Service := awss3.New(s.session) 241 | if listBucketOutput, err = s3Service.ListBuckets(&awss3.ListBucketsInput{}); err != nil { 242 | return false, err 243 | } 244 | for _, bucket := range listBucketOutput.Buckets { 245 | if *bucket.Name == bucketName { 246 | return true, nil 247 | } 248 | } 249 | return false, nil 250 | } 251 | 252 | func (s *s3Store) createBucket(bucketName string) error { 253 | 254 | s3Service := awss3.New(s.session) 255 | bucketExists, err := s.doesBucketExist(bucketName) 256 | if err != nil { 257 | return err 258 | } 259 | if bucketExists { 260 | return nil 261 | } 262 | _, err = s3Service.CreateBucket(&awss3.CreateBucketInput{ 263 | Bucket: aws.String(bucketName), 264 | }) 265 | 266 | return err 267 | 268 | } 269 | 270 | func (s *s3Store) Exists(blob *Blob) bool { 271 | checksum, err := s.Checksum(blob) 272 | if err != nil { 273 | return false 274 | } 275 | 276 | return checksum == blob.Checksum 277 | } 278 | 279 | func (s *s3Store) NewBucketIterator(bucketName string) (BucketIterator, error) { 280 | s3Client := awss3.New(s.session) 281 | 282 | bucketExists, err := s.doesBucketExist(bucketName) 283 | if err != nil { 284 | return nil, err 285 | } 286 | 287 | if !bucketExists { 288 | return nil, errors.New("bucket does not exist") 289 | } 290 | 291 | listObjectsOutput, err := s3Client.ListObjects( 292 | &awss3.ListObjectsInput{ 293 | Bucket: aws.String(bucketName), 294 | }, 295 | ) 296 | if err != nil { 297 | return nil, err 298 | } 299 | 300 | if len(listObjectsOutput.Contents) == 0 { 301 | return &s3BucketIterator{}, nil 302 | } 303 | 304 | doneCh := make(chan struct{}) 305 | blobCh := make(chan *Blob) 306 | 307 | iterator := &s3BucketIterator{ 308 | doneCh: doneCh, 309 | blobCh: blobCh, 310 | } 311 | 312 | go func() { 313 | for _, item := range listObjectsOutput.Contents { 314 | select { 315 | case <-doneCh: 316 | doneCh = nil 317 | return 318 | default: 319 | blobCh <- &Blob{ 320 | Path: filepath.Join( 321 | bucketName, 322 | strings.TrimPrefix(*item.Key, bucketName), 323 | ), 324 | } 325 | } 326 | } 327 | 328 | close(blobCh) 329 | }() 330 | 331 | return iterator, nil 332 | } 333 | -------------------------------------------------------------------------------- /blobstore/s3_bucket_iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobstore 16 | 17 | type s3BucketIterator struct { 18 | blobCh chan *Blob 19 | doneCh chan struct{} 20 | } 21 | 22 | func (i *s3BucketIterator) Next() (*Blob, error) { 23 | if i.blobCh == nil { 24 | return nil, ErrIteratorDone 25 | } 26 | 27 | blob, ok := <-i.blobCh 28 | if !ok { 29 | i.blobCh = nil 30 | return nil, ErrIteratorDone 31 | } 32 | 33 | return blob, nil 34 | } 35 | 36 | func (i *s3BucketIterator) Done() { 37 | i.blobCh = nil 38 | close(i.doneCh) 39 | } 40 | -------------------------------------------------------------------------------- /blobstore/s3_bucket_iterator_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobstore_test 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "strings" 21 | 22 | "github.com/aws/aws-sdk-go/aws" 23 | "github.com/aws/aws-sdk-go/aws/credentials" 24 | "github.com/aws/aws-sdk-go/aws/session" 25 | awss3 "github.com/aws/aws-sdk-go/service/s3" 26 | . "github.com/onsi/ginkgo" 27 | . "github.com/onsi/gomega" 28 | "github.com/pivotal-cf/goblob/blobstore" 29 | ) 30 | 31 | var _ = Describe("S3BucketIterator", func() { 32 | var ( 33 | iterator blobstore.BucketIterator 34 | store blobstore.Blobstore 35 | bucketName string 36 | s3Client *awss3.S3 37 | minioAccessKey string 38 | minioSecretKey string 39 | ) 40 | 41 | const ( 42 | s3Region = "us-east-1" 43 | ) 44 | 45 | BeforeEach(func() { 46 | var s3Endpoint string 47 | 48 | if os.Getenv("MINIO_ACCESS_KEY") == "" { 49 | minioAccessKey = "example-access-key" 50 | } else { 51 | minioAccessKey = os.Getenv("MINIO_ACCESS_KEY") 52 | } 53 | 54 | if os.Getenv("MINIO_SECRET_KEY") == "" { 55 | minioSecretKey = "example-secret-key" 56 | } else { 57 | minioSecretKey = os.Getenv("MINIO_SECRET_KEY") 58 | } 59 | 60 | if os.Getenv("MINIO_PORT_9000_TCP_ADDR") == "" { 61 | s3Endpoint = "http://127.0.0.1:9000" 62 | } else { 63 | s3Endpoint = fmt.Sprintf("http://%s:9000", os.Getenv("MINIO_PORT_9000_TCP_ADDR")) 64 | } 65 | 66 | session := session.New(&aws.Config{ 67 | Region: aws.String(s3Region), 68 | Credentials: credentials.NewStaticCredentials( 69 | minioAccessKey, 70 | minioSecretKey, 71 | "example-token", 72 | ), 73 | Endpoint: aws.String(s3Endpoint), 74 | DisableSSL: aws.Bool(true), 75 | S3ForcePathStyle: aws.Bool(true), 76 | }) 77 | 78 | s3Client = awss3.New(session) 79 | 80 | bucketName = fmt.Sprintf("some-bucket-%d", GinkgoParallelNode()) 81 | 82 | _, err := s3Client.CreateBucket(&awss3.CreateBucketInput{ 83 | Bucket: aws.String(bucketName), 84 | }) 85 | Expect(err).NotTo(HaveOccurred()) 86 | 87 | store = blobstore.NewS3( 88 | minioAccessKey, 89 | minioSecretKey, 90 | s3Region, 91 | s3Endpoint, 92 | true, 93 | true, 94 | true, 95 | "some-buildpacks", 96 | "some-droplets", 97 | "some-packages", 98 | "some-resources", 99 | ) 100 | 101 | iterator, err = store.NewBucketIterator(bucketName) 102 | Expect(err).NotTo(HaveOccurred()) 103 | }) 104 | 105 | AfterEach(func() { 106 | listObjectsOutput, err := s3Client.ListObjects(&awss3.ListObjectsInput{ 107 | Bucket: aws.String(bucketName), 108 | }) 109 | Expect(err).NotTo(HaveOccurred()) 110 | 111 | for _, item := range listObjectsOutput.Contents { 112 | _, err := s3Client.DeleteObject(&awss3.DeleteObjectInput{ 113 | Bucket: aws.String(bucketName), 114 | Key: item.Key, 115 | }) 116 | Expect(err).NotTo(HaveOccurred()) 117 | } 118 | 119 | _, err = s3Client.DeleteBucket(&awss3.DeleteBucketInput{ 120 | Bucket: aws.String(bucketName), 121 | }) 122 | Expect(err).NotTo(HaveOccurred()) 123 | }) 124 | 125 | Describe("Next", func() { 126 | It("returns an error", func() { 127 | _, err := iterator.Next() 128 | Expect(err).To(HaveOccurred()) 129 | Expect(err.Error()).To(Equal("no more items in iterator")) 130 | }) 131 | 132 | Context("when a blob exists in the bucket", func() { 133 | var expectedBlob blobstore.Blob 134 | 135 | BeforeEach(func() { 136 | expectedBlob = blobstore.Blob{ 137 | Path: fmt.Sprintf("%s/some-path/some-file", bucketName), 138 | } 139 | 140 | _, err := s3Client.PutObject(&awss3.PutObjectInput{ 141 | Body: strings.NewReader("content"), 142 | Bucket: aws.String(bucketName), 143 | Key: aws.String("some-path/some-file"), 144 | Metadata: map[string]*string{ 145 | "Checksum": aws.String("some-checksum"), 146 | }, 147 | }) 148 | Expect(err).NotTo(HaveOccurred()) 149 | 150 | iterator, err = store.NewBucketIterator(bucketName) 151 | Expect(err).NotTo(HaveOccurred()) 152 | }) 153 | 154 | It("returns the blob", func() { 155 | blob, err := iterator.Next() 156 | Expect(err).NotTo(HaveOccurred()) 157 | Expect(*blob).To(Equal(expectedBlob)) 158 | }) 159 | 160 | It("returns an error when all blobs have been listed", func() { 161 | _, err := iterator.Next() 162 | Expect(err).NotTo(HaveOccurred()) 163 | 164 | _, err = iterator.Next() 165 | Expect(err).To(HaveOccurred()) 166 | Expect(err.Error()).To(Equal("no more items in iterator")) 167 | }) 168 | }) 169 | }) 170 | 171 | Describe("Done", func() { 172 | Context("when blobs exist in the bucket", func() { 173 | BeforeEach(func() { 174 | _, err := s3Client.PutObject(&awss3.PutObjectInput{ 175 | Body: strings.NewReader("content"), 176 | Bucket: aws.String(bucketName), 177 | Key: aws.String("some-path/some-file"), 178 | Metadata: map[string]*string{ 179 | "Checksum": aws.String("some-checksum"), 180 | }, 181 | }) 182 | Expect(err).NotTo(HaveOccurred()) 183 | 184 | _, err = s3Client.PutObject(&awss3.PutObjectInput{ 185 | Body: strings.NewReader("content"), 186 | Bucket: aws.String(bucketName), 187 | Key: aws.String("some-path/some-other-file"), 188 | Metadata: map[string]*string{ 189 | "Checksum": aws.String("some-checksum"), 190 | }, 191 | }) 192 | Expect(err).NotTo(HaveOccurred()) 193 | 194 | iterator, err = store.NewBucketIterator(bucketName) 195 | Expect(err).NotTo(HaveOccurred()) 196 | }) 197 | 198 | It("causes Next to return an error", func() { 199 | iterator.Done() 200 | 201 | _, err := iterator.Next() 202 | Expect(err).To(HaveOccurred()) 203 | Expect(err.Error()).To(Equal("no more items in iterator")) 204 | }) 205 | }) 206 | }) 207 | }) 208 | -------------------------------------------------------------------------------- /blobstore/s3_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package blobstore_test 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | "path/filepath" 21 | 22 | "github.com/pivotal-cf/goblob/blobstore" 23 | 24 | . "github.com/onsi/ginkgo" 25 | . "github.com/onsi/gomega" 26 | 27 | "github.com/aws/aws-sdk-go/aws" 28 | "github.com/aws/aws-sdk-go/aws/credentials" 29 | 30 | "github.com/aws/aws-sdk-go/aws/session" 31 | 32 | awss3 "github.com/aws/aws-sdk-go/service/s3" 33 | ) 34 | 35 | var _ = Describe("S3Store", func() { 36 | var ( 37 | s3Endpoint string 38 | minioAccessKey string 39 | minioSecretKey string 40 | ) 41 | 42 | region := "us-east-1" 43 | 44 | if os.Getenv("MINIO_ACCESS_KEY") == "" { 45 | minioAccessKey = "example-access-key" 46 | } else { 47 | minioAccessKey = os.Getenv("MINIO_ACCESS_KEY") 48 | } 49 | 50 | if os.Getenv("MINIO_SECRET_KEY") == "" { 51 | minioSecretKey = "example-secret-key" 52 | } else { 53 | minioSecretKey = os.Getenv("MINIO_SECRET_KEY") 54 | } 55 | 56 | if os.Getenv("MINIO_PORT_9000_TCP_ADDR") == "" { 57 | s3Endpoint = "http://127.0.0.1:9000" 58 | } else { 59 | s3Endpoint = fmt.Sprintf("http://%s:9000", os.Getenv("MINIO_PORT_9000_TCP_ADDR")) 60 | } 61 | 62 | config := &aws.Config{ 63 | Region: aws.String(region), 64 | Credentials: credentials.NewStaticCredentials(minioAccessKey, minioSecretKey, ""), 65 | Endpoint: aws.String(s3Endpoint), 66 | DisableSSL: aws.Bool(true), 67 | S3ForcePathStyle: aws.Bool(true), 68 | } 69 | controlBucket := "cc-buildpacks-identifier" 70 | 71 | testsToRun("Multi-part", config, controlBucket, blobstore.NewS3(minioAccessKey, minioSecretKey, region, s3Endpoint, true, true, true, "some-buildpacks", "some-droplets", "some-packages", "some-resources")) 72 | testsToRun("non Multi-part", config, controlBucket, blobstore.NewS3(minioAccessKey, minioSecretKey, region, s3Endpoint, false, true, true, "some-buildpacks", "some-droplets", "some-packages", "some-resources")) 73 | }) 74 | 75 | func testsToRun(testSuiteName string, config *aws.Config, controlBucket string, store blobstore.Blobstore) { 76 | AfterEach(func() { 77 | session := session.New(config) 78 | s3Service := awss3.New(session) 79 | listObjectsOutput, err := s3Service.ListObjects(&awss3.ListObjectsInput{ 80 | Bucket: aws.String(controlBucket), 81 | }) 82 | 83 | if err == nil { 84 | for _, item := range listObjectsOutput.Contents { 85 | _, _ = s3Service.DeleteObject(&awss3.DeleteObjectInput{ 86 | Bucket: aws.String(controlBucket), 87 | Key: item.Key, 88 | }) 89 | } 90 | _, _ = s3Service.DeleteBucket(&awss3.DeleteBucketInput{ 91 | Bucket: aws.String(controlBucket), 92 | }) 93 | } 94 | }) 95 | Describe(testSuiteName, func() { 96 | Describe("List()", func() { 97 | It("Should return list of files", func() { 98 | fileReader, err := os.Open("./s3_testdata/test.txt") 99 | Ω(err).ShouldNot(HaveOccurred()) 100 | for _, path := range []string{"cc-buildpacks/aa/bb", "cc-buildpacks/aa/cc", "cc-buildpacks/aa/dd"} { 101 | err := store.Write(&blobstore.Blob{ 102 | Path: filepath.Join(path, "test.txt"), 103 | }, fileReader) 104 | Ω(err).ShouldNot(HaveOccurred()) 105 | } 106 | blobs, err := store.List() 107 | Ω(err).ShouldNot(HaveOccurred()) 108 | Ω(len(blobs)).Should(BeEquivalentTo(3)) 109 | }) 110 | }) 111 | Describe("Read()", func() { 112 | It("Should read the file", func() { 113 | fileReader, err := os.Open("./s3_testdata/test.txt") 114 | Ω(err).ShouldNot(HaveOccurred()) 115 | writeErr := store.Write(&blobstore.Blob{ 116 | Path: "cc-buildpacks/aa/bb/test.txt", 117 | }, fileReader) 118 | Ω(writeErr).ShouldNot(HaveOccurred()) 119 | reader, err := store.Read(&blobstore.Blob{ 120 | Path: "cc-buildpacks/aa/bb/test.txt", 121 | }) 122 | Ω(err).ShouldNot(HaveOccurred()) 123 | Ω(reader).ShouldNot(BeNil()) 124 | }) 125 | }) 126 | Describe("Write()", func() { 127 | It("Should write to s3 blob store with correct checksum", func() { 128 | reader, err := os.Open("./s3_testdata/test.txt") 129 | Ω(err).ShouldNot(HaveOccurred()) 130 | blob := &blobstore.Blob{ 131 | Path: "cc-buildpacks/aa/bb/test.txt", 132 | } 133 | err = store.Write(blob, reader) 134 | Ω(err).ShouldNot(HaveOccurred()) 135 | 136 | checksum, err := store.Checksum(blob) 137 | Ω(err).ShouldNot(HaveOccurred()) 138 | Ω(checksum).Should(BeEquivalentTo("d8e8fca2dc0f896fd7cb4cb0031ba249")) 139 | }) 140 | }) 141 | }) 142 | } 143 | -------------------------------------------------------------------------------- /blobstore/s3_testdata/test.txt: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /blobstore_migration_watcher.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package goblob 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "html/template" 21 | "os" 22 | "sync" 23 | "sync/atomic" 24 | "time" 25 | 26 | "github.com/mgutz/ansi" 27 | "github.com/pivotal-cf/goblob/blobstore" 28 | ) 29 | 30 | var red = ansi.ColorFunc("red+b") 31 | var yellow = ansi.ColorFunc("yellow+b") 32 | var green = ansi.ColorFunc("green+b") 33 | 34 | type BlobstoreMigrationWatcher interface { 35 | MigrationDidStart(blobstore.Blobstore, blobstore.Blobstore) 36 | MigrationDidFinish() 37 | 38 | MigrateBucketDidStart(string) 39 | MigrateBucketDidFinish() 40 | 41 | MigrateBlobDidFailWithError(error) 42 | MigrateBlobDidFinish() 43 | MigrateBlobAlreadyFinished() 44 | } 45 | 46 | //go:generate counterfeiter . BlobstoreMigrationWatcher 47 | 48 | func NewBlobstoreMigrationWatcher() BlobstoreMigrationWatcher { 49 | return &blobstoreMigrationWatcher{ 50 | stats: &migrateStats{}, 51 | errorsMutex: &sync.Mutex{}, 52 | } 53 | } 54 | 55 | type blobstoreMigrationWatcher struct { 56 | stats *migrateStats 57 | errors []error 58 | errorsMutex *sync.Mutex 59 | } 60 | 61 | func (w *blobstoreMigrationWatcher) MigrationDidStart(dst, src blobstore.Blobstore) { 62 | fmt.Printf("Migrating from %s to %s\n\n", src.Name(), dst.Name()) 63 | w.stats.Start() 64 | } 65 | 66 | func (w *blobstoreMigrationWatcher) MigrationDidFinish() { 67 | w.stats.Finish() 68 | fmt.Println(w.stats) 69 | for i := range w.errors { 70 | fmt.Fprintln(os.Stderr, w.errors[i]) 71 | } 72 | } 73 | 74 | func (w *blobstoreMigrationWatcher) MigrateBucketDidStart(bucket string) { 75 | fmt.Printf("%s ", bucket) 76 | } 77 | 78 | func (w *blobstoreMigrationWatcher) MigrateBucketDidFinish() { 79 | fmt.Println(" done.") 80 | } 81 | 82 | func (w *blobstoreMigrationWatcher) MigrateBlobDidFailWithError(err error) { 83 | w.errorsMutex.Lock() 84 | defer w.errorsMutex.Unlock() 85 | w.stats.AddFailed() 86 | w.errors = append(w.errors, err) 87 | fmt.Print(red(".")) 88 | } 89 | 90 | func (w *blobstoreMigrationWatcher) MigrateBlobDidFinish() { 91 | w.stats.AddSuccess() 92 | fmt.Print(green(".")) 93 | } 94 | 95 | func (w *blobstoreMigrationWatcher) MigrateBlobAlreadyFinished() { 96 | w.stats.AddSkipped() 97 | fmt.Print(yellow(".")) 98 | } 99 | 100 | type migrateStats struct { 101 | startTime time.Time 102 | Duration time.Duration 103 | Migrated int64 104 | Skipped int64 105 | Failed int64 106 | } 107 | 108 | func (m *migrateStats) Start() { 109 | m.startTime = time.Now() 110 | } 111 | 112 | func (m *migrateStats) Finish() { 113 | m.Duration = time.Since(m.startTime) 114 | } 115 | 116 | func (m *migrateStats) AddSuccess() { 117 | atomic.AddInt64(&m.Migrated, 1) 118 | } 119 | 120 | func (m *migrateStats) AddSkipped() { 121 | atomic.AddInt64(&m.Skipped, 1) 122 | } 123 | 124 | func (m *migrateStats) AddFailed() { 125 | atomic.AddInt64(&m.Failed, 1) 126 | } 127 | 128 | func (m *migrateStats) String() string { 129 | t := template.Must(template.New("stats").Parse(` 130 | Took {{.Duration}} 131 | 132 | Migrated files: {{.Migrated}} 133 | Already migrated: {{.Skipped}} 134 | Failed to migrate: {{.Failed}} 135 | `)) 136 | 137 | buf := new(bytes.Buffer) 138 | err := t.Execute(buf, m) 139 | if err != nil { 140 | panic(err.Error()) 141 | } 142 | 143 | return buf.String() 144 | } 145 | -------------------------------------------------------------------------------- /blobstore_migrator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package goblob 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "sync" 21 | 22 | "code.cloudfoundry.org/workpool" 23 | 24 | "github.com/pivotal-cf/goblob/blobstore" 25 | ) 26 | 27 | var ( 28 | buckets = []string{"cc-buildpacks", "cc-droplets", "cc-packages", "cc-resources"} 29 | ) 30 | 31 | // BlobstoreMigrator moves blobs from one blobstore to another 32 | type BlobstoreMigrator interface { 33 | Migrate(dst blobstore.Blobstore, src blobstore.Blobstore) error 34 | } 35 | 36 | type blobstoreMigrator struct { 37 | pool *workpool.WorkPool 38 | blobMigrator BlobMigrator 39 | skip map[string]struct{} 40 | watcher BlobstoreMigrationWatcher 41 | } 42 | 43 | func NewBlobstoreMigrator( 44 | pool *workpool.WorkPool, 45 | blobMigrator BlobMigrator, 46 | exclusions []string, 47 | watcher BlobstoreMigrationWatcher, 48 | ) BlobstoreMigrator { 49 | skip := make(map[string]struct{}) 50 | for i := range exclusions { 51 | skip[exclusions[i]] = struct{}{} 52 | } 53 | 54 | return &blobstoreMigrator{ 55 | pool: pool, 56 | blobMigrator: blobMigrator, 57 | skip: skip, 58 | watcher: watcher, 59 | } 60 | } 61 | 62 | func (m *blobstoreMigrator) Migrate(dst blobstore.Blobstore, src blobstore.Blobstore) error { 63 | if src == nil { 64 | return errors.New("src is an empty store") 65 | } 66 | 67 | if dst == nil { 68 | return errors.New("dst is an empty store") 69 | } 70 | 71 | m.watcher.MigrationDidStart(dst, src) 72 | 73 | migrateWG := &sync.WaitGroup{} 74 | for _, bucket := range buckets { 75 | if _, ok := m.skip[bucket]; ok { 76 | continue 77 | } 78 | 79 | iterator, err := src.NewBucketIterator(bucket) 80 | if err != nil { 81 | return fmt.Errorf("could not create bucket iterator for bucket %s: %s", bucket, err) 82 | } 83 | 84 | m.watcher.MigrateBucketDidStart(bucket) 85 | 86 | bucketWG := &sync.WaitGroup{} 87 | for { 88 | blob, err := iterator.Next() 89 | if err == blobstore.ErrIteratorDone { 90 | break 91 | } 92 | 93 | if err != nil { 94 | return err 95 | } 96 | 97 | migrateWG.Add(1) 98 | bucketWG.Add(1) 99 | m.pool.Submit(func() { 100 | defer bucketWG.Done() 101 | defer migrateWG.Done() 102 | 103 | checksum, err := src.Checksum(blob) 104 | if err != nil { 105 | checksumErr := fmt.Errorf("could not checksum blob: %s", err) 106 | m.watcher.MigrateBlobDidFailWithError(checksumErr) 107 | return 108 | } 109 | 110 | blob.Checksum = checksum 111 | 112 | if dst.Exists(blob) { 113 | m.watcher.MigrateBlobAlreadyFinished() 114 | return 115 | } 116 | 117 | err = m.blobMigrator.Migrate(blob) 118 | if err != nil { 119 | m.watcher.MigrateBlobDidFailWithError(err) 120 | return 121 | } 122 | 123 | m.watcher.MigrateBlobDidFinish() 124 | }) 125 | } 126 | 127 | bucketWG.Wait() 128 | m.watcher.MigrateBucketDidFinish() 129 | } 130 | 131 | migrateWG.Wait() 132 | m.watcher.MigrationDidFinish() 133 | 134 | return nil 135 | } 136 | -------------------------------------------------------------------------------- /blobstore_migrator_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package goblob_test 16 | 17 | import ( 18 | "errors" 19 | 20 | "code.cloudfoundry.org/workpool" 21 | 22 | "github.com/pivotal-cf/goblob" 23 | "github.com/pivotal-cf/goblob/blobstore" 24 | "github.com/pivotal-cf/goblob/blobstore/blobstorefakes" 25 | "github.com/pivotal-cf/goblob/goblobfakes" 26 | 27 | . "github.com/onsi/ginkgo" 28 | . "github.com/onsi/gomega" 29 | ) 30 | 31 | var _ = Describe("BlobstoreMigrator", func() { 32 | var ( 33 | migrator goblob.BlobstoreMigrator 34 | blobMigrator *goblobfakes.FakeBlobMigrator 35 | dstStore *blobstorefakes.FakeBlobstore 36 | srcStore *blobstorefakes.FakeBlobstore 37 | iterator *blobstorefakes.FakeBucketIterator 38 | pool *workpool.WorkPool 39 | watcher *goblobfakes.FakeBlobstoreMigrationWatcher 40 | ) 41 | 42 | BeforeEach(func() { 43 | dstStore = &blobstorefakes.FakeBlobstore{} 44 | srcStore = &blobstorefakes.FakeBlobstore{} 45 | blobMigrator = &goblobfakes.FakeBlobMigrator{} 46 | 47 | var err error 48 | pool, err = workpool.NewWorkPool(1) 49 | Expect(err).NotTo(HaveOccurred()) 50 | 51 | exclusions := []string{} 52 | 53 | watcher = &goblobfakes.FakeBlobstoreMigrationWatcher{} 54 | 55 | migrator = goblob.NewBlobstoreMigrator(pool, blobMigrator, exclusions, watcher) 56 | 57 | iterator = &blobstorefakes.FakeBucketIterator{} 58 | srcStore.NewBucketIteratorReturns(iterator, nil) 59 | }) 60 | 61 | Describe("Migrate", func() { 62 | var firstBlob, secondBlob, thirdBlob *blobstore.Blob 63 | 64 | BeforeEach(func() { 65 | firstBlob = &blobstore.Blob{ 66 | Checksum: "some-file-checksum", 67 | Path: "some-file-path/some-file", 68 | } 69 | 70 | secondBlob = &blobstore.Blob{ 71 | Checksum: "some-other-file-checksum", 72 | Path: "some-other-path/some-other-file", 73 | } 74 | 75 | thirdBlob = &blobstore.Blob{ 76 | Checksum: "yet-another-file-checksum", 77 | Path: "yet-another-path/yet-another-file", 78 | } 79 | 80 | iterator.NextStub = func() (*blobstore.Blob, error) { 81 | switch iterator.NextCallCount() { 82 | case 1: 83 | return firstBlob, nil 84 | case 2: 85 | return secondBlob, nil 86 | case 3: 87 | return thirdBlob, nil 88 | default: 89 | return nil, blobstore.ErrIteratorDone 90 | } 91 | } 92 | }) 93 | 94 | It("uploads all the files from the source", func() { 95 | err := migrator.Migrate(dstStore, srcStore) 96 | Expect(err).NotTo(HaveOccurred()) 97 | Expect(blobMigrator.MigrateCallCount()).To(Equal(3)) 98 | Expect(blobMigrator.MigrateArgsForCall(0)).To(Equal(firstBlob)) 99 | Expect(blobMigrator.MigrateArgsForCall(1)).To(Equal(secondBlob)) 100 | Expect(blobMigrator.MigrateArgsForCall(2)).To(Equal(thirdBlob)) 101 | }) 102 | 103 | Context("when an exclusion list is given", func() { 104 | BeforeEach(func() { 105 | exclusions := []string{"cc-resources", "cc-buildpacks"} 106 | migrator = goblob.NewBlobstoreMigrator(pool, blobMigrator, exclusions, watcher) 107 | }) 108 | 109 | It("does not migrate those paths", func() { 110 | err := migrator.Migrate(dstStore, srcStore) 111 | Expect(err).NotTo(HaveOccurred()) 112 | 113 | var dirs []string 114 | for i := 0; i < srcStore.NewBucketIteratorCallCount(); i++ { 115 | dirs = append(dirs, srcStore.NewBucketIteratorArgsForCall(i)) 116 | } 117 | 118 | Expect(dirs).NotTo(ContainElement("cc-resources")) 119 | Expect(dirs).NotTo(ContainElement("cc-buildpacks")) 120 | }) 121 | }) 122 | 123 | Context("when a file already exists", func() { 124 | BeforeEach(func() { 125 | dstStore.ExistsStub = func(blob *blobstore.Blob) bool { 126 | return blob.Path == "some-other-path/some-other-file" 127 | } 128 | }) 129 | 130 | It("uploads only the new files", func() { 131 | err := migrator.Migrate(dstStore, srcStore) 132 | Expect(err).NotTo(HaveOccurred()) 133 | Expect(blobMigrator.MigrateCallCount()).To(Equal(2)) 134 | Expect(blobMigrator.MigrateArgsForCall(0)).To(Equal(firstBlob)) 135 | Expect(blobMigrator.MigrateArgsForCall(1)).To(Equal(thirdBlob)) 136 | }) 137 | }) 138 | 139 | Context("when there is an error uploading one blob", func() { 140 | BeforeEach(func() { 141 | blobMigrator.MigrateStub = func(blob *blobstore.Blob) error { 142 | if blob.Path == "some-other-path/some-other-file" { 143 | return errors.New("migrate-err") 144 | } 145 | return nil 146 | } 147 | }) 148 | 149 | It("continues uploading", func() { 150 | err := migrator.Migrate(dstStore, srcStore) 151 | Expect(err).NotTo(HaveOccurred()) 152 | 153 | Expect(blobMigrator.MigrateCallCount()).To(Equal(3)) 154 | Expect(blobMigrator.MigrateArgsForCall(0)).To(Equal(firstBlob)) 155 | Expect(blobMigrator.MigrateArgsForCall(1)).To(Equal(secondBlob)) 156 | Expect(blobMigrator.MigrateArgsForCall(2)).To(Equal(thirdBlob)) 157 | }) 158 | }) 159 | 160 | It("returns an error when the source store is nil", func() { 161 | err := migrator.Migrate(dstStore, nil) 162 | Expect(err).To(HaveOccurred()) 163 | Expect(err.Error()).To(Equal("src is an empty store")) 164 | }) 165 | 166 | It("returns an error when the destination store is nil", func() { 167 | err := migrator.Migrate(nil, srcStore) 168 | Expect(err).To(HaveOccurred()) 169 | Expect(err.Error()).To(Equal("dst is an empty store")) 170 | }) 171 | 172 | Context("when the source store has no files", func() { 173 | BeforeEach(func() { 174 | iterator.NextStub = func() (*blobstore.Blob, error) { 175 | return nil, errors.New("no more files!") // <-- this needs to be an exported err 176 | } 177 | }) 178 | 179 | It("returns an error", func() { 180 | err := migrator.Migrate(dstStore, srcStore) 181 | Expect(err).To(HaveOccurred()) 182 | Expect(err.Error()).To(Equal("no more files!")) 183 | }) 184 | }) 185 | 186 | XContext("when there is an error listing the source's files", func() { 187 | BeforeEach(func() { 188 | // srcStore.ListReturns(nil, errors.New("list-error")) 189 | }) 190 | 191 | It("returns an error", func() { 192 | err := migrator.Migrate(dstStore, srcStore) 193 | Expect(err).To(HaveOccurred()) 194 | Expect(err.Error()).To(Equal("list-error")) 195 | }) 196 | }) 197 | }) 198 | }) 199 | -------------------------------------------------------------------------------- /ci/build-darwin.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | platform: linux 17 | 18 | image_resource: 19 | type: docker-image 20 | source: 21 | repository: golang 22 | tag: latest 23 | 24 | inputs: 25 | - name: goblob-with-dependencies 26 | path: go/src/github.com/pivotal-cf/goblob 27 | - name: version 28 | 29 | outputs: 30 | - name: darwin-binary 31 | 32 | params: 33 | OUTPUT_PATH: darwin-binary/goblob-darwin 34 | 35 | run: 36 | path: go/src/github.com/pivotal-cf/goblob/ci/build.sh 37 | -------------------------------------------------------------------------------- /ci/build-linux.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | platform: linux 17 | 18 | image_resource: 19 | type: docker-image 20 | source: 21 | repository: golang 22 | tag: latest 23 | 24 | inputs: 25 | - name: goblob-with-dependencies 26 | path: go/src/github.com/pivotal-cf/goblob 27 | - name: version 28 | 29 | outputs: 30 | - name: linux-binary 31 | 32 | params: 33 | OUTPUT_PATH: linux-binary/goblob-linux 34 | 35 | run: 36 | path: go/src/github.com/pivotal-cf/goblob/ci/build.sh 37 | -------------------------------------------------------------------------------- /ci/build-windows.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | platform: linux 17 | 18 | image_resource: 19 | type: docker-image 20 | source: 21 | repository: golang 22 | tag: latest 23 | 24 | inputs: 25 | - name: goblob-with-dependencies 26 | path: go/src/github.com/pivotal-cf/goblob 27 | - name: version 28 | 29 | outputs: 30 | - name: windows-binary 31 | 32 | params: 33 | OUTPUT_PATH: windows-binary/goblob.exe 34 | 35 | run: 36 | path: go/src/github.com/pivotal-cf/goblob/ci/build.sh 37 | -------------------------------------------------------------------------------- /ci/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | export GOPATH=$PWD/go 6 | 7 | root=$PWD 8 | output_path=$root/$OUTPUT_PATH 9 | version=$(cat $root/version/version) 10 | 11 | cd go/src/github.com/pivotal-cf/goblob 12 | go build \ 13 | -o $output_path \ 14 | -ldflags "-s -w -X goblob.Version=${version}" \ 15 | github.com/pivotal-cf/goblob/cmd/goblob 16 | cd - 17 | -------------------------------------------------------------------------------- /ci/create-pivnet-metadata.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | platform: linux 17 | 18 | image_resource: 19 | type: docker-image 20 | source: 21 | repository: busybox 22 | version: latest 23 | 24 | inputs: 25 | - name: version 26 | 27 | outputs: 28 | - name: pivnet-metadata 29 | 30 | run: 31 | path: sh 32 | args: 33 | - -exc 34 | - | 35 | version=v$(cat version/version) 36 | release_date=$(date +%Y-%m-%d) 37 | 38 | cat >> pivnet-metadata/metadata.yml < release-notes/name 34 | echo v${version} > release-notes/tag 35 | -------------------------------------------------------------------------------- /ci/install-dependencies.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | platform: linux 17 | 18 | image_resource: 19 | type: docker-image 20 | source: 21 | repository: golang 22 | tag: latest 23 | 24 | inputs: 25 | - name: goblob 26 | path: go/src/github.com/pivotal-cf/goblob 27 | 28 | outputs: 29 | - name: goblob-with-dependencies 30 | 31 | run: 32 | path: sh 33 | args: 34 | - -exc 35 | - | 36 | export GOPATH=$PWD/go 37 | export PATH=$GOPATH/bin:$PATH 38 | 39 | ROOT=$(cd $(dirname $0) && pwd) 40 | 41 | go get -u github.com/Masterminds/glide 42 | 43 | cd go/src/github.com/pivotal-cf/goblob 44 | glide install 45 | cp -R . "${ROOT}/goblob-with-dependencies" 46 | cd - 47 | -------------------------------------------------------------------------------- /ci/pipeline.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | groups: [] 16 | 17 | resource_types: 18 | - name: gcs-resource 19 | type: docker-image 20 | source: 21 | repository: frodenas/gcs-resource 22 | 23 | - name: pivnet 24 | type: docker-image 25 | source: 26 | repository: pivotalcf/pivnet-resource 27 | tag: latest-final 28 | 29 | resources: 30 | - name: goblob 31 | type: git 32 | source: 33 | branch: master 34 | private_key: {{git-private-key}} 35 | uri: git@github.com:pivotal-cf/goblob.git 36 | ignore_paths: 37 | - ci 38 | 39 | - name: pivnet-release 40 | type: pivnet 41 | source: 42 | api_token: {{pivnet-api-token}} 43 | product_slug: goblob 44 | access_key_id: {{pivnet-aws-access-key}} 45 | secret_access_key: {{pivnet-aws-secret-key}} 46 | 47 | - name: github-rc 48 | type: github-release 49 | source: 50 | release: false 51 | pre_release: true 52 | user: pivotal-cf 53 | repository: goblob 54 | access_token: {{github-access-token}} 55 | 56 | - name: github-release 57 | type: github-release 58 | source: 59 | user: pivotal-cf 60 | repository: goblob 61 | access_token: {{github-access-token}} 62 | 63 | - name: version 64 | type: semver 65 | source: 66 | driver: git 67 | uri: git@github.com:pivotal-cf/goblob.git 68 | branch: version 69 | file: version 70 | private_key: {{git-private-key}} 71 | 72 | jobs: 73 | - name: create-rc 74 | serial_groups: [version] 75 | plan: 76 | - aggregate: 77 | - get: goblob 78 | trigger: true 79 | - get: version 80 | params: {pre: rc} 81 | - task: install-dependencies 82 | file: goblob/ci/install-dependencies.yml 83 | - aggregate: 84 | - task: unit 85 | file: goblob/ci/unit.yml 86 | params: 87 | MINIO_ACCESS_KEY: example-access-key 88 | MINIO_SECRET_KEY: example-secret-key 89 | - task: build-linux 90 | file: goblob/ci/build-linux.yml 91 | - task: build-darwin 92 | file: goblob/ci/build-darwin.yml 93 | - task: build-windows 94 | file: goblob/ci/build-windows.yml 95 | - aggregate: 96 | - put: version 97 | params: {pre: rc} 98 | - do: 99 | - task: create-release-notes 100 | file: goblob/ci/create-release-notes.yml 101 | - put: github-rc 102 | params: 103 | name: release-notes/name 104 | tag: release-notes/tag 105 | globs: 106 | - linux-binary/* 107 | - darwin-binary/* 108 | - windows-binary/* 109 | 110 | - name: shipit 111 | serial_groups: [version] 112 | plan: 113 | - aggregate: 114 | - get: github-rc 115 | passed: [create-rc] 116 | - get: version 117 | passed: [create-rc] 118 | params: {bump: final} 119 | - get: goblob 120 | passed: [create-rc] 121 | - aggregate: 122 | - put: goblob 123 | params: 124 | repository: goblob 125 | only_tag: true 126 | tag: version/version 127 | tag_prefix: v 128 | - do: 129 | - task: create-pivnet-metadata 130 | file: goblob/ci/create-pivnet-metadata.yml 131 | - put: pivnet-release 132 | params: 133 | metadata_file: pivnet-metadata/metadata.yml 134 | file_glob: github-rc/goblob* 135 | s3_filepath_prefix: {{pivnet-filepath-prefix}} 136 | - do: 137 | - task: create-release-notes 138 | file: goblob/ci/create-release-notes.yml 139 | - put: github-release 140 | params: 141 | name: release-notes/name 142 | tag: release-notes/tag 143 | globs: 144 | - github-rc/* 145 | - put: version 146 | params: 147 | bump: minor 148 | pre: rc 149 | -------------------------------------------------------------------------------- /ci/unit.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | platform: linux 17 | 18 | image_resource: 19 | type: docker-image 20 | source: 21 | repository: golang 22 | tag: latest 23 | 24 | inputs: 25 | - name: goblob-with-dependencies 26 | path: go/src/github.com/pivotal-cf/goblob 27 | 28 | params: 29 | MINIO_ACCESS_KEY: 30 | MINIO_SECRET_KEY: 31 | 32 | run: 33 | path: sh 34 | args: 35 | - -exc 36 | - | 37 | export GOPATH=$PWD/go 38 | export PATH=$GOPATH/bin:$PATH 39 | 40 | wget https://dl.minio.io/server/minio/release/linux-amd64/minio 41 | chmod +x minio 42 | 43 | tmpdir=$(mktemp -d) 44 | ./minio server $tmpdir & 45 | 46 | go get -u github.com/onsi/ginkgo/ginkgo 47 | 48 | cd go/src/github.com/pivotal-cf/goblob 49 | ginkgo -r -race -randomizeAllSpecs 50 | cd - 51 | -------------------------------------------------------------------------------- /cmd/goblob/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | 21 | flags "github.com/jessevdk/go-flags" 22 | "github.com/pivotal-cf/goblob/commands" 23 | ) 24 | 25 | func main() { 26 | parser := flags.NewParser(&commands.Goblob, flags.HelpFlag) 27 | parser.NamespaceDelimiter = "-" 28 | 29 | _, err := parser.Parse() 30 | if err != nil { 31 | fmt.Fprintf(os.Stderr, "error: %s\n", err) 32 | os.Exit(1) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /cmd/goblob/main_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main_test 16 | 17 | import ( 18 | . "github.com/onsi/ginkgo" 19 | . "github.com/onsi/gomega" 20 | "github.com/onsi/gomega/gexec" 21 | 22 | "testing" 23 | ) 24 | 25 | func TestMain(t *testing.T) { 26 | RegisterFailHandler(Fail) 27 | RunSpecs(t, "Main Suite") 28 | } 29 | 30 | var _ = Describe("Main", func() { 31 | AfterEach(func() { 32 | gexec.CleanupBuildArtifacts() 33 | }) 34 | 35 | It("builds", func() { 36 | _, err := gexec.Build("github.com/pivotal-cf/goblob/cmd/goblob") 37 | Expect(err).NotTo(HaveOccurred()) 38 | }) 39 | }) 40 | -------------------------------------------------------------------------------- /commands/goblob.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package commands 16 | 17 | type GoblobCommand struct { 18 | Version func() `command:"version" description:"Print version information and exit"` 19 | 20 | Migrate MigrateCommand `command:"migrate" description:"Migrate blobs from one blobstore to another"` 21 | MigrateToAzure MigrateToAzureBlobCommand `command:"migrate2azure" description:"Migrate blobs from NFS blobstore to Azure blobstore"` 22 | } 23 | 24 | var Goblob GoblobCommand 25 | -------------------------------------------------------------------------------- /commands/migrate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package commands 16 | 17 | import ( 18 | "fmt" 19 | 20 | "code.cloudfoundry.org/workpool" 21 | "github.com/pivotal-cf/goblob" 22 | "github.com/pivotal-cf/goblob/blobstore" 23 | ) 24 | 25 | type MigrateCommand struct { 26 | ConcurrentUploads int `long:"concurrent-uploads" env:"CONCURRENT_UPLOADS" default:"20"` 27 | Exclusions []string `long:"exclude" description:"blobstore directories to exclude, e.g. cc-resources"` 28 | 29 | NFS struct { 30 | Path string `long:"blobstore-path" env:"BLOBSTORE_PATH" description:"path to root of blobstore" default:"/var/vcap/store/shared"` 31 | } `group:"NFS"` 32 | 33 | S3 struct { 34 | AccessKey string `long:"s3-accesskey" env:"S3_ACCESSKEY" description:"S3 access key"` 35 | SecretKey string `long:"s3-secretkey" env:"S3_SECRETKEY" description:"S3 secret access key"` 36 | Region string `long:"region" default:"us-east-1" env:"S3_REGION" description:"S3 region"` 37 | Endpoint string `long:"s3-endpoint" default:"https://s3.amazonaws.com" env:"S3_ENDPOINT"` 38 | UseMultipartUploads bool `long:"use-multipart-uploads" env:"USE_MULTIPART_UPLOADS"` 39 | DisableSSL bool `long:"disable-ssl" description:"disable SSL connections to S3 endpoint"` 40 | InsecureSkipVerify bool `long:"insecure-skip-verify" description:"disable verification of server certificate chain"` 41 | BuildpacksBucketName string `long:"buildpacks-bucket-name" default:"cc-buildpacks" description:"name of bucket to store buildpacks in"` 42 | DropletsBucketName string `long:"droplets-bucket-name" default:"cc-droplets" description:"name of bucket to store droplets in"` 43 | PackagesBucketName string `long:"packages-bucket-name" default:"cc-packages" description:"name of bucket to store packages in"` 44 | ResourcesBucketName string `long:"resources-bucket-name" default:"cc-resources" description:"name of bucket to store resources in"` 45 | } `group:"S3"` 46 | } 47 | 48 | func (c *MigrateCommand) Execute([]string) error { 49 | nfsStore := blobstore.NewNFS(c.NFS.Path) 50 | s3Store := blobstore.NewS3( 51 | c.S3.AccessKey, 52 | c.S3.SecretKey, 53 | c.S3.Region, 54 | c.S3.Endpoint, 55 | c.S3.UseMultipartUploads, 56 | c.S3.DisableSSL, 57 | c.S3.InsecureSkipVerify, 58 | c.S3.BuildpacksBucketName, 59 | c.S3.DropletsBucketName, 60 | c.S3.PackagesBucketName, 61 | c.S3.ResourcesBucketName, 62 | ) 63 | 64 | blobMigrator := goblob.NewBlobMigrator(s3Store, nfsStore) 65 | pool, err := workpool.NewWorkPool(c.ConcurrentUploads) 66 | if err != nil { 67 | return fmt.Errorf("error creating workpool: %s", err) 68 | } 69 | 70 | watcher := goblob.NewBlobstoreMigrationWatcher() 71 | 72 | blobStoreMigrator := goblob.NewBlobstoreMigrator(pool, blobMigrator, c.Exclusions, watcher) 73 | 74 | return blobStoreMigrator.Migrate(s3Store, nfsStore) 75 | } 76 | -------------------------------------------------------------------------------- /commands/migrate2azblob.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package commands 16 | 17 | import ( 18 | "fmt" 19 | 20 | "code.cloudfoundry.org/workpool" 21 | "github.com/pivotal-cf/goblob" 22 | "github.com/pivotal-cf/goblob/blobstore" 23 | ) 24 | 25 | type MigrateToAzureBlobCommand struct { 26 | ConcurrentUploads int `long:"concurrent-uploads" env:"CONCURRENT_UPLOADS" default:"20"` 27 | Exclusions []string `long:"exclude" description:"blobstore directories to exclude, e.g. cc-resources"` 28 | 29 | NFS struct { 30 | Path string `long:"blobstore-path" env:"BLOBSTORE_PATH" description:"path to root of blobstore" default:"/var/vcap/store/shared"` 31 | } `group:"NFS"` 32 | 33 | AzStore struct { 34 | AccountName string `long:"azure-storage-account" env:"AZURE_STORAGE_ACCOUNT" description:"Azure storage account name"` 35 | AccountKey string `long:"azure-storage-account-key" env:"AZURE_STORAGE_ACCOUNT_KEY" description:"Azure storage account key"` 36 | CloudName string `long:"cloud-name" default:"AzureCloud" env:"AZURE_CLOUD" description:"cloud name, available names are: AzureCloud, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment"` 37 | BuildpacksBucketName string `long:"buildpacks-bucket-name" default:"cc-buildpacks" description:"name of bucket to store buildpacks in"` 38 | DropletsBucketName string `long:"droplets-bucket-name" default:"cc-droplets" description:"name of bucket to store droplets in"` 39 | PackagesBucketName string `long:"packages-bucket-name" default:"cc-packages" description:"name of bucket to store packages in"` 40 | ResourcesBucketName string `long:"resources-bucket-name" default:"cc-resources" description:"name of bucket to store resources in"` 41 | } `group:"AzureBlob"` 42 | } 43 | 44 | func (c *MigrateToAzureBlobCommand) Execute([]string) error { 45 | nfsStore := blobstore.NewNFS(c.NFS.Path) 46 | azblobStore := blobstore.NewAzBlobStore( 47 | c.AzStore.AccountName, 48 | c.AzStore.AccountKey, 49 | c.AzStore.CloudName, 50 | c.AzStore.BuildpacksBucketName, 51 | c.AzStore.DropletsBucketName, 52 | c.AzStore.PackagesBucketName, 53 | c.AzStore.ResourcesBucketName, 54 | ) 55 | 56 | blobMigrator := goblob.NewBlobMigrator(azblobStore, nfsStore) 57 | pool, err := workpool.NewWorkPool(c.ConcurrentUploads) 58 | if err != nil { 59 | return fmt.Errorf("error creating workpool: %s", err) 60 | } 61 | 62 | watcher := goblob.NewBlobstoreMigrationWatcher() 63 | 64 | blobStoreMigrator := goblob.NewBlobstoreMigrator(pool, blobMigrator, c.Exclusions, watcher) 65 | 66 | return blobStoreMigrator.Migrate(azblobStore, nfsStore) 67 | } 68 | -------------------------------------------------------------------------------- /commands/version.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package commands 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | 21 | "github.com/pivotal-cf/goblob" 22 | ) 23 | 24 | func init() { 25 | Goblob.Version = func() { 26 | fmt.Println(goblob.Version) 27 | os.Exit(0) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: 1a1decfbf4b1ef74533aa3ee83cbd36011eaf37fbc032fb20bb0764970ce7129 2 | updated: 2018-09-12T04:06:03.398114585Z 3 | imports: 4 | - name: code.cloudfoundry.org/workpool 5 | version: 24756b8d25e8a6a601284a5900005c8bfdcaa766 6 | - name: github.com/aws/aws-sdk-go 7 | version: 08356ca48321d30b25ff97530d4a58fa3754322d 8 | subpackages: 9 | - aws 10 | - aws/awserr 11 | - aws/awsutil 12 | - aws/client 13 | - aws/client/metadata 14 | - aws/corehandlers 15 | - aws/credentials 16 | - aws/credentials/ec2rolecreds 17 | - aws/credentials/endpointcreds 18 | - aws/credentials/stscreds 19 | - aws/csm 20 | - aws/defaults 21 | - aws/ec2metadata 22 | - aws/endpoints 23 | - aws/request 24 | - aws/session 25 | - aws/signer/v4 26 | - internal/sdkio 27 | - internal/sdkrand 28 | - internal/sdkuri 29 | - internal/shareddefaults 30 | - private/protocol 31 | - private/protocol/eventstream 32 | - private/protocol/eventstream/eventstreamapi 33 | - private/protocol/query 34 | - private/protocol/query/queryutil 35 | - private/protocol/rest 36 | - private/protocol/restxml 37 | - private/protocol/xml/xmlutil 38 | - service/s3 39 | - service/s3/s3iface 40 | - service/s3/s3manager 41 | - service/sts 42 | - name: github.com/Azure/azure-pipeline-go 43 | version: 7571e8eb0876932ab505918ff7ed5107773e5ee2 44 | subpackages: 45 | - pipeline 46 | - name: github.com/Azure/azure-storage-blob-go 47 | version: bb46532f68b79e9e1baca8fb19a382ef5d40ed33 48 | subpackages: 49 | - 2018-03-28/azblob 50 | - name: github.com/cheggaaa/pb 51 | version: d7e6ca3010b6f084d8056847f55d7f572f180678 52 | - name: github.com/go-ini/ini 53 | version: 6f66b0e091edb3c7b380f7c4f0f884274d550b67 54 | - name: github.com/jessevdk/go-flags 55 | version: 4e64e4a4e2552194cf594243e23aa9baf3b4297e 56 | - name: github.com/jmespath/go-jmespath 57 | version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d 58 | - name: github.com/mattn/go-runewidth 59 | version: ce7b0b5c7b45a81508558cd1dba6bb1e4ddb51bb 60 | - name: github.com/mgutz/ansi 61 | version: c286dcecd19ff979eeb73ea444e479b903f2cfcb 62 | - name: github.com/op/go-logging 63 | version: 970db520ece77730c7e4724c61121037378659d9 64 | - name: github.com/xchapter7x/lo 65 | version: e33b245fc7a8186582208abc2458c2691bff681c 66 | - name: golang.org/x/net 67 | version: 07b51741c1d6423d4a6abab1c49940ec09cb1aaf 68 | subpackages: 69 | - context 70 | - html 71 | - html/atom 72 | - html/charset 73 | - name: golang.org/x/sync 74 | version: 450f422ab23cf9881c94e2db30cac0eb1b7cf80c 75 | subpackages: 76 | - errgroup 77 | testImports: 78 | - name: github.com/hpcloud/tail 79 | version: a1dbeea552b7c8df4b542c66073e393de198a800 80 | subpackages: 81 | - ratelimiter 82 | - util 83 | - watch 84 | - winfile 85 | - name: github.com/onsi/ginkgo 86 | version: 3774a09d95489ccaa16032e0770d08ea77ba6184 87 | subpackages: 88 | - config 89 | - internal/codelocation 90 | - internal/containernode 91 | - internal/failer 92 | - internal/leafnodes 93 | - internal/remote 94 | - internal/spec 95 | - internal/spec_iterator 96 | - internal/specrunner 97 | - internal/suite 98 | - internal/testingtproxy 99 | - internal/writer 100 | - reporters 101 | - reporters/stenographer 102 | - types 103 | - name: github.com/onsi/gomega 104 | version: 7615b9433f86a8bdf29709bf288bc4fd0636a369 105 | subpackages: 106 | - format 107 | - gbytes 108 | - gexec 109 | - internal/assertion 110 | - internal/asyncassertion 111 | - internal/oraclematcher 112 | - internal/testingtsupport 113 | - matchers 114 | - matchers/support/goraph/bipartitegraph 115 | - matchers/support/goraph/edge 116 | - matchers/support/goraph/node 117 | - matchers/support/goraph/util 118 | - types 119 | - name: golang.org/x/sys 120 | version: d75a52659825e75fff6158388dddc6a5b04f9ba5 121 | subpackages: 122 | - unix 123 | - name: golang.org/x/text 124 | version: 905a57155faa8230500121607930ebb9dd8e139c 125 | subpackages: 126 | - encoding 127 | - encoding/charmap 128 | - encoding/htmlindex 129 | - encoding/internal 130 | - encoding/internal/identifier 131 | - encoding/japanese 132 | - encoding/korean 133 | - encoding/simplifiedchinese 134 | - encoding/traditionalchinese 135 | - encoding/unicode 136 | - internal/language 137 | - internal/language/compact 138 | - internal/tag 139 | - internal/utf8internal 140 | - language 141 | - runes 142 | - transform 143 | - name: gopkg.in/fsnotify/fsnotify.v1 144 | version: c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9 145 | - name: gopkg.in/tomb.v1 146 | version: c131134a1947e9afd9cecfe11f4c6dff0732ae58 147 | - name: gopkg.in/yaml.v2 148 | version: a83829b6f1293c91addabc89d0571c246397bbf4 149 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/pivotal-cf/goblob 2 | import: 3 | - package: github.com/aws/aws-sdk-go 4 | version: master 5 | subpackages: 6 | - service/s3 7 | - package: github.com/xchapter7x/lo 8 | - package: github.com/op/go-logging 9 | version: master 10 | - package: github.com/mattn/go-runewidth 11 | version: master 12 | - package: golang.org/x/sync 13 | - package: golang.org/x/net 14 | subpackages: 15 | - context 16 | - package: code.cloudfoundry.org/workpool 17 | - package: github.com/jessevdk/go-flags 18 | - package: github.com/mgutz/ansi 19 | - package: github.com/Azure/azure-storage-blob-go 20 | version: 0.2.0 21 | subpackages: 22 | - 2018-03-28/azblob 23 | testImport: 24 | - package: github.com/onsi/ginkgo 25 | version: master 26 | - package: github.com/onsi/gomega 27 | version: master 28 | -------------------------------------------------------------------------------- /goblob_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package goblob_test 16 | 17 | import ( 18 | . "github.com/onsi/ginkgo" 19 | . "github.com/onsi/gomega" 20 | 21 | "testing" 22 | ) 23 | 24 | func TestGoblob(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "Goblob Suite") 27 | } 28 | -------------------------------------------------------------------------------- /goblobfakes/fake_blob_migrator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This file was generated by counterfeiter 16 | package goblobfakes 17 | 18 | import ( 19 | "sync" 20 | 21 | "github.com/pivotal-cf/goblob" 22 | "github.com/pivotal-cf/goblob/blobstore" 23 | ) 24 | 25 | type FakeBlobMigrator struct { 26 | MigrateStub func(blob *blobstore.Blob) error 27 | migrateMutex sync.RWMutex 28 | migrateArgsForCall []struct { 29 | blob *blobstore.Blob 30 | } 31 | migrateReturns struct { 32 | result1 error 33 | } 34 | invocations map[string][][]interface{} 35 | invocationsMutex sync.RWMutex 36 | } 37 | 38 | func (fake *FakeBlobMigrator) Migrate(blob *blobstore.Blob) error { 39 | fake.migrateMutex.Lock() 40 | fake.migrateArgsForCall = append(fake.migrateArgsForCall, struct { 41 | blob *blobstore.Blob 42 | }{blob}) 43 | fake.recordInvocation("Migrate", []interface{}{blob}) 44 | fake.migrateMutex.Unlock() 45 | if fake.MigrateStub != nil { 46 | return fake.MigrateStub(blob) 47 | } else { 48 | return fake.migrateReturns.result1 49 | } 50 | } 51 | 52 | func (fake *FakeBlobMigrator) MigrateCallCount() int { 53 | fake.migrateMutex.RLock() 54 | defer fake.migrateMutex.RUnlock() 55 | return len(fake.migrateArgsForCall) 56 | } 57 | 58 | func (fake *FakeBlobMigrator) MigrateArgsForCall(i int) *blobstore.Blob { 59 | fake.migrateMutex.RLock() 60 | defer fake.migrateMutex.RUnlock() 61 | return fake.migrateArgsForCall[i].blob 62 | } 63 | 64 | func (fake *FakeBlobMigrator) MigrateReturns(result1 error) { 65 | fake.MigrateStub = nil 66 | fake.migrateReturns = struct { 67 | result1 error 68 | }{result1} 69 | } 70 | 71 | func (fake *FakeBlobMigrator) Invocations() map[string][][]interface{} { 72 | fake.invocationsMutex.RLock() 73 | defer fake.invocationsMutex.RUnlock() 74 | fake.migrateMutex.RLock() 75 | defer fake.migrateMutex.RUnlock() 76 | return fake.invocations 77 | } 78 | 79 | func (fake *FakeBlobMigrator) recordInvocation(key string, args []interface{}) { 80 | fake.invocationsMutex.Lock() 81 | defer fake.invocationsMutex.Unlock() 82 | if fake.invocations == nil { 83 | fake.invocations = map[string][][]interface{}{} 84 | } 85 | if fake.invocations[key] == nil { 86 | fake.invocations[key] = [][]interface{}{} 87 | } 88 | fake.invocations[key] = append(fake.invocations[key], args) 89 | } 90 | 91 | var _ goblob.BlobMigrator = new(FakeBlobMigrator) 92 | -------------------------------------------------------------------------------- /goblobfakes/fake_blobstore_migration_watcher.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // This file was generated by counterfeiter 16 | package goblobfakes 17 | 18 | import ( 19 | "sync" 20 | 21 | "github.com/pivotal-cf/goblob" 22 | "github.com/pivotal-cf/goblob/blobstore" 23 | ) 24 | 25 | type FakeBlobstoreMigrationWatcher struct { 26 | MigrationDidStartStub func(blobstore.Blobstore, blobstore.Blobstore) 27 | migrationDidStartMutex sync.RWMutex 28 | migrationDidStartArgsForCall []struct { 29 | arg1 blobstore.Blobstore 30 | arg2 blobstore.Blobstore 31 | } 32 | MigrationDidFinishStub func() 33 | migrationDidFinishMutex sync.RWMutex 34 | migrationDidFinishArgsForCall []struct{} 35 | MigrateBucketDidStartStub func(string) 36 | migrateBucketDidStartMutex sync.RWMutex 37 | migrateBucketDidStartArgsForCall []struct { 38 | arg1 string 39 | } 40 | MigrateBucketDidFinishStub func() 41 | migrateBucketDidFinishMutex sync.RWMutex 42 | migrateBucketDidFinishArgsForCall []struct{} 43 | MigrateBlobDidFailWithErrorStub func(error) 44 | migrateBlobDidFailWithErrorMutex sync.RWMutex 45 | migrateBlobDidFailWithErrorArgsForCall []struct { 46 | arg1 error 47 | } 48 | MigrateBlobDidFinishStub func() 49 | migrateBlobDidFinishMutex sync.RWMutex 50 | migrateBlobDidFinishArgsForCall []struct{} 51 | MigrateBlobDidFinishPreviouslyStub func() 52 | migrateBlobDidFinishPreviouslyMutex sync.RWMutex 53 | migrateBlobDidFinishPreviouslyArgsForCall []struct{} 54 | invocations map[string][][]interface{} 55 | invocationsMutex sync.RWMutex 56 | } 57 | 58 | func (fake *FakeBlobstoreMigrationWatcher) MigrationDidStart(arg1 blobstore.Blobstore, arg2 blobstore.Blobstore) { 59 | fake.migrationDidStartMutex.Lock() 60 | fake.migrationDidStartArgsForCall = append(fake.migrationDidStartArgsForCall, struct { 61 | arg1 blobstore.Blobstore 62 | arg2 blobstore.Blobstore 63 | }{arg1, arg2}) 64 | fake.recordInvocation("MigrationDidStart", []interface{}{arg1, arg2}) 65 | fake.migrationDidStartMutex.Unlock() 66 | if fake.MigrationDidStartStub != nil { 67 | fake.MigrationDidStartStub(arg1, arg2) 68 | } 69 | } 70 | 71 | func (fake *FakeBlobstoreMigrationWatcher) MigrationDidStartCallCount() int { 72 | fake.migrationDidStartMutex.RLock() 73 | defer fake.migrationDidStartMutex.RUnlock() 74 | return len(fake.migrationDidStartArgsForCall) 75 | } 76 | 77 | func (fake *FakeBlobstoreMigrationWatcher) MigrationDidStartArgsForCall(i int) (blobstore.Blobstore, blobstore.Blobstore) { 78 | fake.migrationDidStartMutex.RLock() 79 | defer fake.migrationDidStartMutex.RUnlock() 80 | return fake.migrationDidStartArgsForCall[i].arg1, fake.migrationDidStartArgsForCall[i].arg2 81 | } 82 | 83 | func (fake *FakeBlobstoreMigrationWatcher) MigrationDidFinish() { 84 | fake.migrationDidFinishMutex.Lock() 85 | fake.migrationDidFinishArgsForCall = append(fake.migrationDidFinishArgsForCall, struct{}{}) 86 | fake.recordInvocation("MigrationDidFinish", []interface{}{}) 87 | fake.migrationDidFinishMutex.Unlock() 88 | if fake.MigrationDidFinishStub != nil { 89 | fake.MigrationDidFinishStub() 90 | } 91 | } 92 | 93 | func (fake *FakeBlobstoreMigrationWatcher) MigrationDidFinishCallCount() int { 94 | fake.migrationDidFinishMutex.RLock() 95 | defer fake.migrationDidFinishMutex.RUnlock() 96 | return len(fake.migrationDidFinishArgsForCall) 97 | } 98 | 99 | func (fake *FakeBlobstoreMigrationWatcher) MigrateBucketDidStart(arg1 string) { 100 | fake.migrateBucketDidStartMutex.Lock() 101 | fake.migrateBucketDidStartArgsForCall = append(fake.migrateBucketDidStartArgsForCall, struct { 102 | arg1 string 103 | }{arg1}) 104 | fake.recordInvocation("MigrateBucketDidStart", []interface{}{arg1}) 105 | fake.migrateBucketDidStartMutex.Unlock() 106 | if fake.MigrateBucketDidStartStub != nil { 107 | fake.MigrateBucketDidStartStub(arg1) 108 | } 109 | } 110 | 111 | func (fake *FakeBlobstoreMigrationWatcher) MigrateBucketDidStartCallCount() int { 112 | fake.migrateBucketDidStartMutex.RLock() 113 | defer fake.migrateBucketDidStartMutex.RUnlock() 114 | return len(fake.migrateBucketDidStartArgsForCall) 115 | } 116 | 117 | func (fake *FakeBlobstoreMigrationWatcher) MigrateBucketDidStartArgsForCall(i int) string { 118 | fake.migrateBucketDidStartMutex.RLock() 119 | defer fake.migrateBucketDidStartMutex.RUnlock() 120 | return fake.migrateBucketDidStartArgsForCall[i].arg1 121 | } 122 | 123 | func (fake *FakeBlobstoreMigrationWatcher) MigrateBucketDidFinish() { 124 | fake.migrateBucketDidFinishMutex.Lock() 125 | fake.migrateBucketDidFinishArgsForCall = append(fake.migrateBucketDidFinishArgsForCall, struct{}{}) 126 | fake.recordInvocation("MigrateBucketDidFinish", []interface{}{}) 127 | fake.migrateBucketDidFinishMutex.Unlock() 128 | if fake.MigrateBucketDidFinishStub != nil { 129 | fake.MigrateBucketDidFinishStub() 130 | } 131 | } 132 | 133 | func (fake *FakeBlobstoreMigrationWatcher) MigrateBucketDidFinishCallCount() int { 134 | fake.migrateBucketDidFinishMutex.RLock() 135 | defer fake.migrateBucketDidFinishMutex.RUnlock() 136 | return len(fake.migrateBucketDidFinishArgsForCall) 137 | } 138 | 139 | func (fake *FakeBlobstoreMigrationWatcher) MigrateBlobDidFailWithError(arg1 error) { 140 | fake.migrateBlobDidFailWithErrorMutex.Lock() 141 | fake.migrateBlobDidFailWithErrorArgsForCall = append(fake.migrateBlobDidFailWithErrorArgsForCall, struct { 142 | arg1 error 143 | }{arg1}) 144 | fake.recordInvocation("MigrateBlobDidFailWithError", []interface{}{arg1}) 145 | fake.migrateBlobDidFailWithErrorMutex.Unlock() 146 | if fake.MigrateBlobDidFailWithErrorStub != nil { 147 | fake.MigrateBlobDidFailWithErrorStub(arg1) 148 | } 149 | } 150 | 151 | func (fake *FakeBlobstoreMigrationWatcher) MigrateBlobDidFailWithErrorCallCount() int { 152 | fake.migrateBlobDidFailWithErrorMutex.RLock() 153 | defer fake.migrateBlobDidFailWithErrorMutex.RUnlock() 154 | return len(fake.migrateBlobDidFailWithErrorArgsForCall) 155 | } 156 | 157 | func (fake *FakeBlobstoreMigrationWatcher) MigrateBlobDidFailWithErrorArgsForCall(i int) error { 158 | fake.migrateBlobDidFailWithErrorMutex.RLock() 159 | defer fake.migrateBlobDidFailWithErrorMutex.RUnlock() 160 | return fake.migrateBlobDidFailWithErrorArgsForCall[i].arg1 161 | } 162 | 163 | func (fake *FakeBlobstoreMigrationWatcher) MigrateBlobDidFinish() { 164 | fake.migrateBlobDidFinishMutex.Lock() 165 | fake.migrateBlobDidFinishArgsForCall = append(fake.migrateBlobDidFinishArgsForCall, struct{}{}) 166 | fake.recordInvocation("MigrateBlobDidFinish", []interface{}{}) 167 | fake.migrateBlobDidFinishMutex.Unlock() 168 | if fake.MigrateBlobDidFinishStub != nil { 169 | fake.MigrateBlobDidFinishStub() 170 | } 171 | } 172 | 173 | func (fake *FakeBlobstoreMigrationWatcher) MigrateBlobDidFinishCallCount() int { 174 | fake.migrateBlobDidFinishMutex.RLock() 175 | defer fake.migrateBlobDidFinishMutex.RUnlock() 176 | return len(fake.migrateBlobDidFinishArgsForCall) 177 | } 178 | 179 | func (fake *FakeBlobstoreMigrationWatcher) MigrateBlobAlreadyFinished() { 180 | fake.migrateBlobDidFinishPreviouslyMutex.Lock() 181 | fake.migrateBlobDidFinishPreviouslyArgsForCall = append(fake.migrateBlobDidFinishPreviouslyArgsForCall, struct{}{}) 182 | fake.recordInvocation("MigrateBlobDidFinishPreviously", []interface{}{}) 183 | fake.migrateBlobDidFinishPreviouslyMutex.Unlock() 184 | if fake.MigrateBlobDidFinishPreviouslyStub != nil { 185 | fake.MigrateBlobDidFinishPreviouslyStub() 186 | } 187 | } 188 | 189 | func (fake *FakeBlobstoreMigrationWatcher) MigrateBlobDidFinishPreviouslyCallCount() int { 190 | fake.migrateBlobDidFinishPreviouslyMutex.RLock() 191 | defer fake.migrateBlobDidFinishPreviouslyMutex.RUnlock() 192 | return len(fake.migrateBlobDidFinishPreviouslyArgsForCall) 193 | } 194 | 195 | func (fake *FakeBlobstoreMigrationWatcher) Invocations() map[string][][]interface{} { 196 | fake.invocationsMutex.RLock() 197 | defer fake.invocationsMutex.RUnlock() 198 | fake.migrationDidStartMutex.RLock() 199 | defer fake.migrationDidStartMutex.RUnlock() 200 | fake.migrationDidFinishMutex.RLock() 201 | defer fake.migrationDidFinishMutex.RUnlock() 202 | fake.migrateBucketDidStartMutex.RLock() 203 | defer fake.migrateBucketDidStartMutex.RUnlock() 204 | fake.migrateBucketDidFinishMutex.RLock() 205 | defer fake.migrateBucketDidFinishMutex.RUnlock() 206 | fake.migrateBlobDidFailWithErrorMutex.RLock() 207 | defer fake.migrateBlobDidFailWithErrorMutex.RUnlock() 208 | fake.migrateBlobDidFinishMutex.RLock() 209 | defer fake.migrateBlobDidFinishMutex.RUnlock() 210 | fake.migrateBlobDidFinishPreviouslyMutex.RLock() 211 | defer fake.migrateBlobDidFinishPreviouslyMutex.RUnlock() 212 | return fake.invocations 213 | } 214 | 215 | func (fake *FakeBlobstoreMigrationWatcher) recordInvocation(key string, args []interface{}) { 216 | fake.invocationsMutex.Lock() 217 | defer fake.invocationsMutex.Unlock() 218 | if fake.invocations == nil { 219 | fake.invocations = map[string][][]interface{}{} 220 | } 221 | if fake.invocations[key] == nil { 222 | fake.invocations[key] = [][]interface{}{} 223 | } 224 | fake.invocations[key] = append(fake.invocations[key], args) 225 | } 226 | 227 | var _ goblob.BlobstoreMigrationWatcher = new(FakeBlobstoreMigrationWatcher) 228 | -------------------------------------------------------------------------------- /testrunner: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Check if images are available locally 4 | DOCKER_LOCAL="" 5 | if [[ $(docker images|grep golang) ]]; then 6 | echo "Using previously cached docker images" 7 | DOCKER_LOCAL="--docker-local" 8 | fi 9 | rm -fR _builds _steps _projects _cache _temp 10 | wercker --verbose --environment ".testrunner_env_defaults" build --git-domain github.com --git-owner pivotal-cf --git-repository goblob ${DOCKER_LOCAL} 11 | rm -fR _builds _steps _projects _cache _temp .wercker 12 | -------------------------------------------------------------------------------- /validation/checksum.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package validation 16 | 17 | import ( 18 | "crypto/md5" 19 | "encoding/hex" 20 | "io" 21 | "os" 22 | ) 23 | 24 | // Checksum calculates a checksum for a file 25 | func Checksum(filePath string) (string, error) { 26 | file, err := os.Open(filePath) 27 | if err != nil { 28 | return "", err 29 | } 30 | defer file.Close() 31 | 32 | return ChecksumReader(file) 33 | } 34 | 35 | // ChecksumReader calculates a checksum for a file 36 | func ChecksumReader(reader io.Reader) (string, error) { 37 | hash := md5.New() 38 | if _, err := io.Copy(hash, reader); err != nil { 39 | return "", err 40 | } 41 | 42 | hashInBytes := hash.Sum(nil)[:16] 43 | return hex.EncodeToString(hashInBytes), nil 44 | } 45 | -------------------------------------------------------------------------------- /validation/checksum_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package validation_test 16 | 17 | import ( 18 | "path" 19 | 20 | . "github.com/pivotal-cf/goblob/validation" 21 | 22 | . "github.com/onsi/ginkgo" 23 | . "github.com/onsi/gomega" 24 | ) 25 | 26 | var _ = Describe("Md5", func() { 27 | It("Generates correct checksums", func() { 28 | checksum, err := Checksum(path.Join(".", "fixtures", "testfile")) 29 | Ω(err).Should(BeNil()) 30 | Ω(checksum).Should(BeEquivalentTo("b026324c6904b2a9cb4b88d6d61c81d1")) 31 | }) 32 | 33 | It("Generates correct checksums", func() { 34 | checksum, err := Checksum(path.Join(".", "fixtures", "013110a30e2a475551c801b4c45e497ce71c26fe")) 35 | Ω(err).Should(BeNil()) 36 | Ω(checksum).Should(BeEquivalentTo("9e63a667623321944e174d3d3ea16e9e")) 37 | }) 38 | 39 | It("Returns an error for a missing filename", func() { 40 | checksum, err := Checksum(path.Join(".", "fixtures", "testmissing")) 41 | Ω(err).ShouldNot(BeNil()) 42 | Ω(checksum).Should(BeEquivalentTo("")) 43 | }) 44 | }) 45 | -------------------------------------------------------------------------------- /validation/fixtures/013110a30e2a475551c801b4c45e497ce71c26fe: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | -------------------------------------------------------------------------------- /validation/fixtures/testfile: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /validation/validation_suite_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package validation_test 16 | 17 | import ( 18 | . "github.com/onsi/ginkgo" 19 | . "github.com/onsi/gomega" 20 | 21 | "testing" 22 | ) 23 | 24 | func TestValidation(t *testing.T) { 25 | RegisterFailHandler(Fail) 26 | RunSpecs(t, "Validation Suite") 27 | } 28 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http:#www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package goblob 16 | 17 | var Version = "0.0.0-dev" 18 | -------------------------------------------------------------------------------- /wercker.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2017-Present Pivotal Software, Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | box: golang 16 | services: 17 | - name: minio 18 | id: minio/minio 19 | cmd: server --address 0.0.0.0:9000 /export 20 | env: 21 | MINIO_ACCESS_KEY: AKIAIOSFODNN7EXAMPLE 22 | MINIO_SECRET_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY 23 | build: 24 | # The steps that will be executed on build 25 | steps: 26 | 27 | - script: 28 | name: env 29 | code: env 30 | # Sets the go workspace and places you package 31 | # at the right place in the workspace tree 32 | - setup-go-workspace 33 | 34 | # Get the dependencies 35 | - script: 36 | name: go get 37 | code: | 38 | export GO15VENDOREXPERIMENT=1 39 | cd $WERCKER_SOURCE_DIR 40 | go version 41 | wget https://github.com/Masterminds/glide/releases/download/v0.12.3/glide-v0.12.3-linux-amd64.tar.gz 42 | tar xvzf glide-v0.12.3-linux-amd64.tar.gz 43 | export PATH=./linux-amd64:$PATH 44 | export PATH=$WERCKER_SOURCE_DIR/bin:$PATH 45 | glide install 46 | 47 | # Test the project 48 | - script: 49 | name: go test 50 | code: | 51 | export MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE 52 | export MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY 53 | LOG_LEVEL=debug go test $(glide novendor) -v -cover -race 54 | 55 | crosscompile: 56 | steps: 57 | - setup-go-workspace 58 | 59 | # Get the dependencies 60 | - script: 61 | name: go get 62 | code: | 63 | export GO15VENDOREXPERIMENT=1 64 | cd $WERCKER_SOURCE_DIR 65 | go version 66 | wget https://github.com/Masterminds/glide/releases/download/v0.11.1/glide-v0.11.1-linux-amd64.tar.gz 67 | tar xvzf glide-v0.11.1-linux-amd64.tar.gz 68 | export PATH=./linux-amd64:$PATH 69 | export PATH=$WERCKER_SOURCE_DIR/bin:$PATH 70 | glide install 71 | - script: 72 | name: set release id variable and version 73 | code: | 74 | go get github.com/xchapter7x/versioning 75 | export NEXT_VERSION=`versioning bump_patch`-`git rev-parse HEAD | cut -c1-6` 76 | echo "next version should be: ${NEXT_VERSION}" 77 | 78 | - script: 79 | name: cross platform release 80 | code: | 81 | (GOOS=linux GOARCH=amd64 go build -o goblob-linux -ldflags "-X goblob.Version=${NEXT_VERSION}" ./cmd/goblob/main.go) 82 | (GOOS=darwin GOARCH=amd64 go build -o goblob-osx -ldflags "-X goblob.Version=${NEXT_VERSION}" ./cmd/goblob/main.go) 83 | (GOOS=windows GOARCH=amd64 go build -o goblob.exe -ldflags "-X goblob.Version=${NEXT_VERSION}" ./cmd/goblob/main.go) 84 | 85 | - script: 86 | name: add repo to artifact 87 | code: | 88 | cp -R ./ ${WERCKER_OUTPUT_DIR} 89 | 90 | - script: 91 | name: list output dir 92 | code: | 93 | ls -al ${WERCKER_OUTPUT_DIR} 94 | deploy: 95 | steps: 96 | - script: 97 | name: install-packages 98 | code: | 99 | sudo apt-get install -y openssh-client wget 100 | ls -la 101 | pwd 102 | echo ${WERCKER_OUTPUT_DIR} 103 | ls -la ${WERCKER_OUTPUT_DIR} 104 | 105 | - wercker/add-ssh-key@1.0.2: 106 | keyname: GITHUB_KEY 107 | 108 | - wercker/add-to-known_hosts@1.4.0: 109 | hostname: github.com 110 | fingerprint: 16:27:ac:a5:76:28:2d:36:63:1b:56:4d:eb:df:a6:48 111 | 112 | - script: 113 | name: set release id variable for version 114 | code: | 115 | go get github.com/xchapter7x/versioning 116 | export WERCKER_GITHUB_CREATE_RELEASE_ID=`versioning bump_patch`-`git rev-parse HEAD | cut -c1-6` 117 | 118 | - github-create-release: 119 | token: $GITHUB_TOKEN 120 | tag: $WERCKER_GITHUB_CREATE_RELEASE_ID 121 | title: $WERCKER_GITHUB_CREATE_RELEASE_ID 122 | draft: true 123 | 124 | - github-upload-asset: 125 | token: $GITHUB_TOKEN 126 | file: goblob-linux 127 | release_id: $WERCKER_GITHUB_CREATE_RELEASE_ID 128 | content-type: application/x-gzip 129 | - github-upload-asset: 130 | token: $GITHUB_TOKEN 131 | file: goblob-osx 132 | release_id: $WERCKER_GITHUB_CREATE_RELEASE_ID 133 | content-type: application/x-gzip 134 | - github-upload-asset: 135 | token: $GITHUB_TOKEN 136 | file: goblob.exe 137 | release_id: $WERCKER_GITHUB_CREATE_RELEASE_ID 138 | content-type: application/x-gzip 139 | --------------------------------------------------------------------------------