├── .editorconfig ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── codecov.yml ├── error.go ├── error_test.go ├── examples ├── logger │ └── example.go ├── simple │ └── example.go └── statistics │ └── example.go ├── go.mod ├── go.sum ├── logger.go ├── option.go ├── option_test.go ├── renovate.json ├── s3path.go ├── s3path_test.go ├── s3sync.go ├── s3sync_test.go └── util_test.go /.editorconfig: -------------------------------------------------------------------------------- 1 | root=true 2 | [*] 3 | trim_trailing_whitespace=true 4 | insert_final_newline=true 5 | [*.go] 6 | indent_style=tab 7 | indent_size=8 8 | [*.yml] 9 | indent_style=space 10 | indent_size=2 11 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - master 7 | 8 | env: 9 | # These dummy credentials are necessary for running tests against 10 | # localstack s3 service 11 | AWS_ACCESS_KEY_ID: foo 12 | AWS_SECRET_ACCESS_KEY: bar 13 | jobs: 14 | test: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | go_version: 20 | - '1.18' 21 | - '1.19' 22 | - '1.20' 23 | - '1.21' 24 | - '1.22' 25 | steps: 26 | - uses: actions/checkout@v4 27 | - uses: actions/setup-go@v5 28 | with: 29 | go-version: ${{ matrix.go_version }} 30 | - name: Go vet 31 | run: go vet ./... 32 | - name: Test 33 | run: | 34 | # Start mock s3 service 35 | make s3-bg 36 | while ! curl http://localhost:4572; do sleep 1; done 37 | sleep 5 38 | # Set up fixture S3 files 39 | make fixture 40 | make cover # run test with coverage 41 | env: 42 | AWS_REGION: ap-northeast-1 # for awscli 43 | - uses: codecov/codecov-action@v4 44 | with: 45 | file: ./cover.out 46 | token: ${{ secrets.CODECOV_TOKEN }} 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # coverage output 2 | cover.out 3 | report.html 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 SEQSENSE, Inc. 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | LOCALSTACK_VERSION = 3.8.0 15 | 16 | .PHONY: test 17 | test: 18 | go test . -v 19 | 20 | .PHONY: cover 21 | cover: 22 | go test -race -coverprofile=cover.out . 23 | go tool cover -html=cover.out -o report.html 24 | 25 | .PHONY: s3 26 | s3: 27 | docker run -p 4572:4566 -e SERVICES=s3 localstack/localstack:$(LOCALSTACK_VERSION) 28 | 29 | .PHONY: s3-bg 30 | s3-bg: 31 | docker run -d -p 4572:4566 -e SERVICES=s3 localstack/localstack:$(LOCALSTACK_VERSION) 32 | 33 | .PHONY: fixture 34 | fixture: 35 | aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket 36 | aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket 37 | aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket/foo/ 38 | aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket/bar/baz/ 39 | aws s3 --endpoint-url http://localhost:4572 mb s3://s3-source 40 | aws s3 --endpoint-url http://localhost:4572 cp README.md s3://s3-source 41 | aws s3 --endpoint-url http://localhost:4572 cp README.md s3://s3-source/foo/ 42 | aws s3 --endpoint-url http://localhost:4572 cp README.md s3://s3-source/bar/baz/ 43 | aws s3 --endpoint-url http://localhost:4572 mb s3://s3-destination 44 | aws s3 --endpoint-url http://localhost:4572 mb s3://s3-destination2 45 | aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-escaped 46 | aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-upload 47 | aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket-upload/dest_only_file 48 | aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-upload-file 49 | aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-delete 50 | aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket-delete/dest_only_file 51 | aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-delete-file 52 | aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket-delete-file 53 | aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket-delete-file/dest_only_file 54 | aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-dryrun 55 | aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket-dryrun/dest_only_file 56 | aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-directory 57 | aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-mime 58 | aws s3api --endpoint-url http://localhost:4572 put-object --bucket example-bucket-directory --key test/ 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # s3sync 2 | 3 | ![CI](https://github.com/seqsense/s3sync/workflows/CI/badge.svg) 4 | [![codecov](https://codecov.io/gh/seqsense/s3sync/branch/master/graph/badge.svg)](https://codecov.io/gh/seqsense/s3sync) 5 | 6 | > Golang utility for syncing between s3 and local 7 | 8 | # Usage 9 | 10 | Use `New` to create a manager, and `Sync` function syncs between s3 and local filesystem. 11 | 12 | ```go 13 | import ( 14 | "github.com/aws/aws-sdk-go/aws" 15 | "github.com/aws/aws-sdk-go/aws/session" 16 | "github.com/seqsense/s3sync" 17 | ) 18 | 19 | func main() { 20 | // Creates an AWS session 21 | sess, _ := session.NewSession(&aws.Config{ 22 | Region: aws.String("us-east-1"), 23 | }) 24 | 25 | syncManager := s3sync.New(sess) 26 | 27 | // Sync from s3 to local 28 | syncManager.Sync("s3://yourbucket/path/to/dir", "local/path/to/dir") 29 | 30 | // Sync from local to s3 31 | syncManager.Sync("local/path/to/dir", "s3://yourbucket/path/to/dir") 32 | 33 | // Sync from s3 to s3 34 | syncManager.Sync("s3://yourbucket/path/to/dir", "s3://anotherbucket/path/to/dir") 35 | } 36 | ``` 37 | 38 | ## Sets the custom logger 39 | 40 | You can set your custom logger. 41 | 42 | ```go 43 | import "github.com/seqsense/s3sync" 44 | 45 | ... 46 | s3sync.SetLogger(&CustomLogger{}) 47 | ... 48 | ``` 49 | 50 | The logger needs to implement `Log` and `Logf` methods. See the godoc for details. 51 | 52 | ## Sets up the parallelism 53 | 54 | You can configure the number of parallel jobs for sync. Default is 16. 55 | 56 | ``` 57 | s3sync.new(sess, s3sync.WithParallel(16)) // This is the same as default. 58 | s3sync.new(sess, s3sync.WithParallel(1)) // You can sync one by one. 59 | ``` 60 | 61 | # License 62 | 63 | Apache 2.0 License. See [LICENSE](https://github.com/seqsense/s3sync/blob/master/LICENSE). 64 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | # Allow decreasing 2% of total coverage to avoid noise. 6 | threshold: 2% 7 | patch: 8 | default: 9 | target: 50% 10 | only_pulls: true 11 | -------------------------------------------------------------------------------- /error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 SEQSENSE, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package s3sync 15 | 16 | import ( 17 | "strings" 18 | "sync" 19 | ) 20 | 21 | type multiErr struct { 22 | mu sync.Mutex 23 | err []error 24 | } 25 | 26 | func (e *multiErr) Append(err error) { 27 | e.mu.Lock() 28 | e.err = append(e.err, err) 29 | e.mu.Unlock() 30 | } 31 | 32 | func (e *multiErr) Len() int { 33 | e.mu.Lock() 34 | defer e.mu.Unlock() 35 | return len(e.err) 36 | } 37 | 38 | func (e *multiErr) ErrOrNil() error { 39 | if e.Len() > 0 { 40 | return e 41 | } 42 | return nil 43 | } 44 | 45 | func (e *multiErr) Error() string { 46 | var errMsgs []string 47 | for _, err := range e.err { 48 | errMsgs = append(errMsgs, err.Error()) 49 | } 50 | return strings.Join(errMsgs, "\n") 51 | } 52 | -------------------------------------------------------------------------------- /error_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 SEQSENSE, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package s3sync 15 | 16 | import ( 17 | "errors" 18 | "testing" 19 | ) 20 | 21 | func TestMultiErr(t *testing.T) { 22 | t.Run("Nil", func(t *testing.T) { 23 | err := &multiErr{} 24 | if err.Error() != "" { 25 | t.Error("Empty multiErr should return empty string") 26 | } 27 | if err.ErrOrNil() != nil { 28 | t.Error("Empty multiErr should return nil error") 29 | } 30 | }) 31 | t.Run("MultipleErrors", func(t *testing.T) { 32 | err := &multiErr{} 33 | err.Append(errors.New("error1")) 34 | err.Append(errors.New("error2")) 35 | if err.Error() != "error1\nerror2" { 36 | t.Error("Empty multiErr should return joined error message") 37 | } 38 | if err.ErrOrNil() != err { 39 | t.Error("Empty multiErr should return self pointer") 40 | } 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /examples/logger/example.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SEQSENSE, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | package main 14 | 15 | import ( 16 | "fmt" 17 | "os" 18 | 19 | "github.com/aws/aws-sdk-go/aws" 20 | "github.com/aws/aws-sdk-go/aws/session" 21 | "github.com/seqsense/s3sync" 22 | ) 23 | 24 | type Logger struct { 25 | } 26 | 27 | func (l *Logger) Log(v ...interface{}) { 28 | fmt.Println(v...) 29 | } 30 | 31 | func (l *Logger) Logf(format string, v ...interface{}) { 32 | fmt.Printf(format, v...) 33 | } 34 | 35 | // Usage: go run ./examples/simple s3://example-bucket/path/to/source path/to/dest 36 | func main() { 37 | sess, err := session.NewSession(&aws.Config{ 38 | Region: aws.String("ap-northeast-1"), 39 | }) 40 | if err != nil { 41 | panic(err) 42 | } 43 | 44 | fmt.Printf("from=%s\n", os.Args[1]) 45 | fmt.Printf("to=%s\n", os.Args[2]) 46 | 47 | s3sync.SetLogger(&Logger{}) 48 | 49 | err = s3sync.New(sess).Sync(os.Args[1], os.Args[2]) 50 | if err != nil { 51 | panic(err) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /examples/simple/example.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SEQSENSE, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | package main 14 | 15 | import ( 16 | "fmt" 17 | "os" 18 | 19 | "github.com/aws/aws-sdk-go/aws" 20 | "github.com/aws/aws-sdk-go/aws/session" 21 | "github.com/seqsense/s3sync" 22 | ) 23 | 24 | // Usage: go run ./examples/simple s3://example-bucket/path/to/source path/to/dest 25 | func main() { 26 | sess, err := session.NewSession(&aws.Config{ 27 | Region: aws.String("ap-northeast-1"), 28 | }) 29 | if err != nil { 30 | panic(err) 31 | } 32 | 33 | fmt.Printf("from=%s\n", os.Args[1]) 34 | fmt.Printf("to=%s\n", os.Args[2]) 35 | 36 | err = s3sync.New(sess).Sync(os.Args[1], os.Args[2]) 37 | if err != nil { 38 | panic(err) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /examples/statistics/example.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SEQSENSE, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | package main 14 | 15 | import ( 16 | "fmt" 17 | "os" 18 | "time" 19 | 20 | "github.com/aws/aws-sdk-go/aws" 21 | "github.com/aws/aws-sdk-go/aws/session" 22 | "github.com/seqsense/s3sync" 23 | ) 24 | 25 | // Usage: go run ./examples/simple s3://example-bucket/path/to/source path/to/dest 26 | func main() { 27 | sess, err := session.NewSession(&aws.Config{ 28 | Region: aws.String("ap-northeast-1"), 29 | }) 30 | if err != nil { 31 | panic(err) 32 | } 33 | 34 | fmt.Printf("from=%s\n", os.Args[1]) 35 | fmt.Printf("to=%s\n", os.Args[2]) 36 | 37 | startSync := time.Now() 38 | manager := s3sync.New(sess) 39 | err = manager.Sync(os.Args[1], os.Args[2]) 40 | syncTime := (time.Now().UnixNano() - startSync.UnixNano()) / (int64(time.Millisecond) / int64(time.Nanosecond)) 41 | if err != nil { 42 | panic(err) 43 | } 44 | s := manager.GetStatistics() 45 | fmt.Printf("Sync results:\nBytes written: %d\nFiles uploaded: %d\nTime spent: %d millisecond(s)\nFiles deleted: %d\n", s.Bytes, s.Files, syncTime, s.DeletedFiles) 46 | } 47 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/seqsense/s3sync 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go v1.55.5 7 | github.com/gabriel-vasile/mimetype v1.4.5 8 | ) 9 | 10 | require ( 11 | github.com/jmespath/go-jmespath v0.4.0 // indirect 12 | golang.org/x/net v0.27.0 // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= 2 | github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= 3 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= 6 | github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= 7 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 8 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 9 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 10 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 11 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 12 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 13 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 14 | golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= 15 | golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= 16 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 17 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 18 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 19 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SEQSENSE, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | package s3sync 14 | 15 | import "log" 16 | 17 | // LoggerIF is the logger interface which this library requires. 18 | type LoggerIF interface { 19 | // Log inserts a log entry. Arguments are handled in the manner 20 | // of fmt.Print. 21 | Log(v ...interface{}) 22 | // Log inserts a log entry. Arguments are handled in the manner 23 | // of fmt.Printf. 24 | Logf(format string, v ...interface{}) 25 | } 26 | 27 | // Logger is the logger instance. 28 | var logger LoggerIF 29 | 30 | // SetLogger sets the logger. 31 | func SetLogger(l LoggerIF) { 32 | logger = l 33 | } 34 | 35 | func println(v ...interface{}) { 36 | if logger == nil { 37 | log.Println(v...) 38 | return 39 | } 40 | logger.Log(v...) 41 | } 42 | -------------------------------------------------------------------------------- /option.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SEQSENSE, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package s3sync 15 | 16 | import "github.com/aws/aws-sdk-go/service/s3/s3manager" 17 | 18 | const ( 19 | // Default number of parallel file sync jobs. 20 | DefaultParallel = 16 21 | ) 22 | 23 | // Option is a functional option type of Manager. 24 | type Option func(*Manager) 25 | 26 | // WithParallel sets maximum number of parallel file sync jobs. 27 | func WithParallel(n int) Option { 28 | return func(m *Manager) { 29 | m.nJobs = n 30 | } 31 | } 32 | 33 | // WithDelete enables to delete files unexisting on source directory. 34 | func WithDelete() Option { 35 | return func(m *Manager) { 36 | m.del = true 37 | } 38 | } 39 | 40 | // WithACL sets Access Control List string for uploading. 41 | func WithACL(acl string) Option { 42 | return func(m *Manager) { 43 | acl := acl 44 | m.acl = &acl 45 | } 46 | } 47 | 48 | // WithDryRun enables dry-run mode. 49 | func WithDryRun() Option { 50 | return func(m *Manager) { 51 | m.dryrun = true 52 | } 53 | } 54 | 55 | // WithoutGuessMimeType disables guessing MIME type from contents. 56 | func WithoutGuessMimeType() Option { 57 | return func(m *Manager) { 58 | m.guessMime = false 59 | } 60 | } 61 | 62 | // WithContentType overwrites uploading MIME type. 63 | func WithContentType(mime string) Option { 64 | return func(m *Manager) { 65 | m.contentType = &mime 66 | } 67 | } 68 | 69 | // WithDownloaderOptions sets underlying s3manager's options. 70 | func WithDownloaderOptions(opts ...func(*s3manager.Downloader)) Option { 71 | return func(m *Manager) { 72 | m.downloaderOpts = opts 73 | } 74 | } 75 | 76 | // WithUploaderOptions sets underlying s3manager's options. 77 | func WithUploaderOptions(opts ...func(*s3manager.Uploader)) Option { 78 | return func(m *Manager) { 79 | m.uploaderOpts = opts 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /option_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SEQSENSE, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package s3sync 15 | 16 | import ( 17 | "testing" 18 | 19 | "github.com/aws/aws-sdk-go/aws" 20 | "github.com/aws/aws-sdk-go/aws/credentials" 21 | "github.com/aws/aws-sdk-go/aws/session" 22 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 23 | ) 24 | 25 | func TestWithParallel(t *testing.T) { 26 | sess := session.New(&aws.Config{ 27 | Credentials: credentials.AnonymousCredentials, 28 | Region: aws.String("dummy"), 29 | }) 30 | 31 | m := New(sess, WithParallel(2)) 32 | if m.nJobs != 2 { 33 | t.Fatal("Manager.nJobs must be configured by WithParallel option") 34 | } 35 | } 36 | 37 | func TestWithACL(t *testing.T) { 38 | sess := session.New(&aws.Config{ 39 | Credentials: credentials.AnonymousCredentials, 40 | Region: aws.String("dummy"), 41 | }) 42 | 43 | t.Run("Nil", func(t *testing.T) { 44 | m := New(sess) 45 | if m.acl != nil { 46 | t.Fatal("Manager.acl must be nil if initialized with WithACL") 47 | } 48 | }) 49 | t.Run("WithACL", func(t *testing.T) { 50 | m := New(sess, WithACL("test")) 51 | if *m.acl != "test" { 52 | t.Fatal("Manager.acl must be configured by WithParallel option") 53 | } 54 | }) 55 | } 56 | 57 | func TestUploaderDownloaderOptions(t *testing.T) { 58 | sess := session.New(&aws.Config{ 59 | Credentials: credentials.AnonymousCredentials, 60 | Region: aws.String("dummy"), 61 | }) 62 | 63 | t.Run("Uploader", func(t *testing.T) { 64 | m := New(sess, WithUploaderOptions( 65 | func(u *s3manager.Uploader) {}, 66 | )) 67 | if len(m.uploaderOpts) != 1 { 68 | t.Fatal("Manager.uploaderOpts must have a option") 69 | } 70 | }) 71 | t.Run("Downloader", func(t *testing.T) { 72 | m := New(sess, WithDownloaderOptions( 73 | func(d *s3manager.Downloader) {}, 74 | )) 75 | if len(m.downloaderOpts) != 1 { 76 | t.Fatal("Manager.downloaderOpts must have a option") 77 | } 78 | }) 79 | } 80 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base", 4 | "schedule:weekends" 5 | ], 6 | "regexManagers": [ 7 | { 8 | "fileMatch": [ 9 | "^Makefile$" 10 | ], 11 | "matchStrings": [ 12 | "LOCALSTACK_VERSION\\s*=\\s*(?.*)" 13 | ], 14 | "datasourceTemplate": "docker", 15 | "depNameTemplate": "localstack/localstack", 16 | "versioningTemplate": "docker" 17 | } 18 | ], 19 | "postUpdateOptions": [ 20 | "gomodTidy" 21 | ], 22 | "packageRules": [ 23 | { 24 | "packagePatterns": [ 25 | "^golang.org/x/" 26 | ], 27 | "schedule": [ 28 | "on the first day of the month" 29 | ] 30 | }, 31 | { 32 | "packagePatterns": [ 33 | "^github.com/aws/aws-sdk-go" 34 | ], 35 | "separateMinorPatch": true 36 | }, 37 | { 38 | "packagePatterns": [ 39 | "^github.com/aws/aws-sdk-go" 40 | ], 41 | "matchUpdateTypes": ["patch"], 42 | "automerge": true 43 | } 44 | ], 45 | "timezone": "Asia/Tokyo" 46 | } 47 | -------------------------------------------------------------------------------- /s3path.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SEQSENSE, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package s3sync 15 | 16 | import ( 17 | "errors" 18 | "net/url" 19 | "path/filepath" 20 | "strings" 21 | ) 22 | 23 | var errNoBucketName = errors.New("s3 url is missing bucket name") 24 | 25 | type s3Path struct { 26 | bucket string 27 | bucketPrefix string 28 | } 29 | 30 | func urlToS3Path(url *url.URL) (*s3Path, error) { 31 | if url.Host == "" { 32 | return nil, errNoBucketName 33 | } 34 | 35 | path := url.RawPath 36 | if path == "" { 37 | // If the path doesn't contain any special characters, RawPath would be empty 38 | path = url.Path 39 | } 40 | 41 | return &s3Path{ 42 | bucket: url.Host, 43 | // Using filepath.ToSlash for change backslash to slash on Windows 44 | bucketPrefix: strings.TrimPrefix(filepath.ToSlash(path), "/"), 45 | }, nil 46 | } 47 | 48 | func (p *s3Path) String() string { 49 | return "s3://" + p.bucket + "/" + p.bucketPrefix 50 | } 51 | -------------------------------------------------------------------------------- /s3path_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SEQSENSE, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package s3sync 15 | 16 | import ( 17 | "fmt" 18 | "net/url" 19 | "testing" 20 | ) 21 | 22 | func assertS3Path(t *testing.T, expectedBucket, expectedPrefix string, p *s3Path) { 23 | t.Helper() 24 | if p.bucket != expectedBucket || p.bucketPrefix != expectedPrefix { 25 | t.Fatalf( 26 | `Expected bucket="%s" prefix="%s", got bucket="%s" prefix="%s"`, 27 | expectedBucket, expectedPrefix, p.bucket, p.bucketPrefix, 28 | ) 29 | } 30 | } 31 | 32 | func TestURLToS3Path(t *testing.T) { 33 | t.Run("NoBucketName", func(t *testing.T) { 34 | _, err := urlToS3Path(&url.URL{ 35 | Host: "", 36 | Path: "test", 37 | }) 38 | if err != errNoBucketName { 39 | t.Fatalf("Expected error %v, got %v", errNoBucketName, err) 40 | } 41 | }) 42 | t.Run("Normal", func(t *testing.T) { 43 | p, err := urlToS3Path(&url.URL{ 44 | Host: "bucket", 45 | Path: "test", 46 | }) 47 | if err != nil { 48 | t.Fatalf("Unexpected error: %v", err) 49 | } 50 | assertS3Path(t, "bucket", "test", p) 51 | }) 52 | t.Run("UrlEscapedPath", func(t *testing.T) { 53 | urlHost := "bucket" 54 | urlPath := fmt.Sprintf("space /%s", url.QueryEscape("test/it")) 55 | srUrl, err := url.Parse(fmt.Sprintf("s3://%s/%s", urlHost, urlPath)) 56 | 57 | if err != nil { 58 | t.Fatalf("Unexpected error: %v", err) 59 | } 60 | 61 | p, err := urlToS3Path(srUrl) 62 | 63 | if err != nil { 64 | t.Fatalf("Unexpected error: %v", err) 65 | } 66 | assertS3Path(t, urlHost, urlPath, p) 67 | }) 68 | } 69 | 70 | func TestS3Path_String(t *testing.T) { 71 | p := &s3Path{ 72 | bucket: "bucket", 73 | bucketPrefix: "test", 74 | } 75 | const expected = "s3://bucket/test" 76 | if s := p.String(); s != expected { 77 | t.Fatalf("Expected %s, got %s", expected, s) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /s3sync.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SEQSENSE, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | package s3sync 14 | 15 | import ( 16 | "context" 17 | "errors" 18 | "net/url" 19 | "os" 20 | "path/filepath" 21 | "strings" 22 | "sync" 23 | "time" 24 | 25 | "github.com/aws/aws-sdk-go/aws" 26 | "github.com/aws/aws-sdk-go/aws/session" 27 | "github.com/aws/aws-sdk-go/service/s3" 28 | "github.com/aws/aws-sdk-go/service/s3/s3iface" 29 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 30 | "github.com/gabriel-vasile/mimetype" 31 | ) 32 | 33 | // Manager manages the sync operation. 34 | type Manager struct { 35 | s3 s3iface.S3API 36 | nJobs int 37 | del bool 38 | dryrun bool 39 | acl *string 40 | guessMime bool 41 | contentType *string 42 | downloaderOpts []func(*s3manager.Downloader) 43 | uploaderOpts []func(*s3manager.Uploader) 44 | statistics SyncStatistics 45 | } 46 | 47 | // SyncStatistics captures the sync statistics. 48 | type SyncStatistics struct { 49 | Bytes int64 50 | Files int64 51 | DeletedFiles int64 52 | mutex sync.RWMutex 53 | } 54 | 55 | type operation int 56 | 57 | const ( 58 | opUpdate operation = iota 59 | opDelete 60 | ) 61 | 62 | type fileInfo struct { 63 | name string 64 | err error 65 | path string 66 | size int64 67 | lastModified time.Time 68 | singleFile bool 69 | existsInSource bool 70 | } 71 | 72 | type fileOp struct { 73 | *fileInfo 74 | op operation 75 | } 76 | 77 | // New returns a new Manager. 78 | func New(sess *session.Session, options ...Option) *Manager { 79 | m := &Manager{ 80 | s3: s3.New(sess), 81 | nJobs: DefaultParallel, 82 | guessMime: true, 83 | } 84 | for _, o := range options { 85 | o(m) 86 | } 87 | return m 88 | } 89 | 90 | // Sync syncs the files between s3 and local disks. 91 | func (m *Manager) Sync(source, dest string) error { 92 | return m.SyncWithContext(context.Background(), source, dest) 93 | } 94 | 95 | // SyncWithContext syncs the files between s3 and local disks. 96 | // The context will be used for operation cancellation. 97 | func (m *Manager) SyncWithContext(ctx context.Context, source, dest string) error { 98 | sourceURL, err := url.Parse(source) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | destURL, err := url.Parse(dest) 104 | if err != nil { 105 | return err 106 | } 107 | 108 | ctx, cancel := context.WithCancel(ctx) 109 | defer cancel() 110 | 111 | chJob := make(chan func()) 112 | var wg sync.WaitGroup 113 | for i := 0; i < m.nJobs; i++ { 114 | wg.Add(1) 115 | go func() { 116 | defer wg.Done() 117 | for job := range chJob { 118 | job() 119 | } 120 | }() 121 | } 122 | defer func() { 123 | close(chJob) 124 | wg.Wait() 125 | }() 126 | 127 | if isS3URL(sourceURL) { 128 | sourceS3Path, err := urlToS3Path(sourceURL) 129 | if err != nil { 130 | return err 131 | } 132 | if isS3URL(destURL) { 133 | destS3Path, err := urlToS3Path(destURL) 134 | if err != nil { 135 | return err 136 | } 137 | return m.syncS3ToS3(ctx, chJob, sourceS3Path, destS3Path) 138 | } 139 | return m.syncS3ToLocal(ctx, chJob, sourceS3Path, dest) 140 | } 141 | 142 | if isS3URL(destURL) { 143 | destS3Path, err := urlToS3Path(destURL) 144 | if err != nil { 145 | return err 146 | } 147 | return m.syncLocalToS3(ctx, chJob, source, destS3Path) 148 | } 149 | 150 | return errors.New("local to local sync is not supported") 151 | } 152 | 153 | // GetStatistics returns the structure that contains the sync statistics 154 | func (m *Manager) GetStatistics() SyncStatistics { 155 | m.statistics.mutex.Lock() 156 | defer m.statistics.mutex.Unlock() 157 | return SyncStatistics{Bytes: m.statistics.Bytes, Files: m.statistics.Files, DeletedFiles: m.statistics.DeletedFiles} 158 | } 159 | 160 | func isS3URL(url *url.URL) bool { 161 | return url.Scheme == "s3" 162 | } 163 | 164 | func (m *Manager) syncS3ToS3(ctx context.Context, chJob chan func(), sourcePath *s3Path, destPath *s3Path) error { 165 | wg := &sync.WaitGroup{} 166 | errs := &multiErr{} 167 | for source := range filterFilesForSync( 168 | m.listS3Files(ctx, sourcePath), m.listS3Files(ctx, destPath), m.del, 169 | ) { 170 | wg.Add(1) 171 | source := source 172 | chJob <- func() { 173 | defer wg.Done() 174 | if source.err != nil { 175 | errs.Append(source.err) 176 | return 177 | } 178 | switch source.op { 179 | case opUpdate: 180 | if err := m.copyS3ToS3(ctx, source.fileInfo, sourcePath, destPath); err != nil { 181 | errs.Append(err) 182 | } 183 | } 184 | } 185 | } 186 | wg.Wait() 187 | 188 | return errs.ErrOrNil() 189 | 190 | } 191 | 192 | func (m *Manager) syncLocalToS3(ctx context.Context, chJob chan func(), sourcePath string, destPath *s3Path) error { 193 | wg := &sync.WaitGroup{} 194 | errs := &multiErr{} 195 | for source := range filterFilesForSync( 196 | listLocalFiles(ctx, sourcePath), m.listS3Files(ctx, destPath), m.del, 197 | ) { 198 | wg.Add(1) 199 | source := source 200 | chJob <- func() { 201 | defer wg.Done() 202 | if source.err != nil { 203 | errs.Append(source.err) 204 | return 205 | } 206 | switch source.op { 207 | case opUpdate: 208 | if err := m.upload(source.fileInfo, sourcePath, destPath); err != nil { 209 | errs.Append(err) 210 | } 211 | case opDelete: 212 | if err := m.deleteRemote(source.fileInfo, destPath); err != nil { 213 | errs.Append(err) 214 | } 215 | } 216 | } 217 | } 218 | wg.Wait() 219 | 220 | return errs.ErrOrNil() 221 | } 222 | 223 | // syncS3ToLocal syncs the given s3 path to the given local path. 224 | func (m *Manager) syncS3ToLocal(ctx context.Context, chJob chan func(), sourcePath *s3Path, destPath string) error { 225 | wg := &sync.WaitGroup{} 226 | errs := &multiErr{} 227 | for source := range filterFilesForSync( 228 | m.listS3Files(ctx, sourcePath), listLocalFiles(ctx, destPath), m.del, 229 | ) { 230 | wg.Add(1) 231 | source := source 232 | chJob <- func() { 233 | defer wg.Done() 234 | if source.err != nil { 235 | errs.Append(source.err) 236 | return 237 | } 238 | switch source.op { 239 | case opUpdate: 240 | if err := m.download(source.fileInfo, sourcePath, destPath); err != nil { 241 | errs.Append(err) 242 | } 243 | case opDelete: 244 | if err := m.deleteLocal(source.fileInfo, destPath); err != nil { 245 | errs.Append(err) 246 | } 247 | } 248 | } 249 | } 250 | wg.Wait() 251 | 252 | return errs.ErrOrNil() 253 | } 254 | 255 | func (m *Manager) copyS3ToS3(ctx context.Context, file *fileInfo, sourcePath *s3Path, destPath *s3Path) error { 256 | copySource := filepath.ToSlash(filepath.Join(sourcePath.bucket, sourcePath.bucketPrefix, file.name)) 257 | destinationKey := filepath.ToSlash(filepath.Join(destPath.bucketPrefix, file.name)) 258 | println("Copying from", copySource, "to key", destinationKey, "in bucket", destPath.bucket) 259 | if m.dryrun { 260 | return nil 261 | } 262 | 263 | _, err := m.s3.CopyObject(&s3.CopyObjectInput{ 264 | Bucket: aws.String(destPath.bucket), 265 | CopySource: aws.String(copySource), 266 | Key: aws.String(destinationKey), 267 | ACL: m.acl, 268 | }) 269 | 270 | if err != nil { 271 | return err 272 | } 273 | 274 | m.updateFileTransferStatistics(file.size) 275 | return nil 276 | } 277 | 278 | func (m *Manager) download(file *fileInfo, sourcePath *s3Path, destPath string) error { 279 | var targetFilename string 280 | if !strings.HasSuffix(destPath, "/") && file.singleFile { 281 | // Destination path is not a directory and source is a single file. 282 | targetFilename = destPath 283 | } else { 284 | targetFilename = filepath.Join(destPath, file.name) 285 | } 286 | targetDir := filepath.Dir(targetFilename) 287 | 288 | println("Downloading", file.name, "to", targetFilename) 289 | if m.dryrun { 290 | return nil 291 | } 292 | 293 | if err := os.MkdirAll(targetDir, 0755); err != nil { 294 | return err 295 | } 296 | 297 | writer, err := os.Create(targetFilename) 298 | if err != nil { 299 | return err 300 | } 301 | 302 | defer writer.Close() 303 | 304 | var sourceFile string 305 | if file.singleFile { 306 | sourceFile = file.name 307 | } else { 308 | // Using filepath.ToSlash for change backslash to slash on Windows 309 | sourceFile = filepath.ToSlash(filepath.Join(sourcePath.bucketPrefix, file.name)) 310 | } 311 | 312 | c := s3manager.NewDownloaderWithClient(m.s3, m.downloaderOpts...) 313 | written, err := c.Download(writer, &s3.GetObjectInput{ 314 | Bucket: aws.String(sourcePath.bucket), 315 | Key: aws.String(sourceFile), 316 | }) 317 | if err != nil { 318 | return err 319 | } 320 | m.updateFileTransferStatistics(written) 321 | err = os.Chtimes(targetFilename, file.lastModified, file.lastModified) 322 | if err != nil { 323 | return err 324 | } 325 | 326 | return nil 327 | } 328 | 329 | func (m *Manager) deleteLocal(file *fileInfo, destPath string) error { 330 | var targetFilename string 331 | if !strings.HasSuffix(destPath, "/") && file.singleFile { 332 | // Destination path is not a directory and source is a single file. 333 | targetFilename = destPath 334 | } else { 335 | targetFilename = filepath.Join(destPath, file.name) 336 | } 337 | 338 | println("Deleting", targetFilename) 339 | if m.dryrun { 340 | return nil 341 | } 342 | err := os.Remove(targetFilename) 343 | if err != nil { 344 | return err 345 | } 346 | m.incrementDeletedFiles() 347 | return nil 348 | } 349 | 350 | func (m *Manager) upload(file *fileInfo, sourcePath string, destPath *s3Path) error { 351 | var sourceFilename string 352 | if file.singleFile { 353 | sourceFilename = sourcePath 354 | } else { 355 | sourceFilename = filepath.Join(sourcePath, file.name) 356 | } 357 | 358 | destFile := *destPath 359 | if strings.HasSuffix(destPath.bucketPrefix, "/") || destPath.bucketPrefix == "" || !file.singleFile { 360 | // If source is a single file and destination is not a directory, use destination URL as is. 361 | // Using filepath.ToSlash for change backslash to slash on Windows 362 | destFile.bucketPrefix = filepath.ToSlash(filepath.Join(destPath.bucketPrefix, file.name)) 363 | } 364 | 365 | println("Uploading", file.name, "to", destFile.String()) 366 | if m.dryrun { 367 | return nil 368 | } 369 | 370 | var contentType *string 371 | switch { 372 | case m.contentType != nil: 373 | contentType = m.contentType 374 | case m.guessMime: 375 | mime, err := mimetype.DetectFile(sourceFilename) 376 | if err != nil { 377 | return err 378 | } 379 | s := mime.String() 380 | contentType = &s 381 | } 382 | 383 | reader, err := os.Open(sourceFilename) 384 | if err != nil { 385 | return err 386 | } 387 | 388 | defer reader.Close() 389 | 390 | _, err = s3manager.NewUploaderWithClient( 391 | m.s3, 392 | m.uploaderOpts..., 393 | ).Upload(&s3manager.UploadInput{ 394 | Bucket: aws.String(destFile.bucket), 395 | Key: aws.String(destFile.bucketPrefix), 396 | ACL: m.acl, 397 | Body: reader, 398 | ContentType: contentType, 399 | }) 400 | if err != nil { 401 | return err 402 | } 403 | m.updateFileTransferStatistics(file.size) 404 | return nil 405 | } 406 | 407 | func (m *Manager) deleteRemote(file *fileInfo, destPath *s3Path) error { 408 | destFile := *destPath 409 | if strings.HasSuffix(destPath.bucketPrefix, "/") || destPath.bucketPrefix == "" || !file.singleFile { 410 | // If source is a single file and destination is not a directory, use destination URL as is. 411 | // Using filepath.ToSlash for change backslash to slash on Windows 412 | destFile.bucketPrefix = filepath.ToSlash(filepath.Join(destPath.bucketPrefix, file.name)) 413 | } 414 | 415 | println("Deleting", destFile.String()) 416 | if m.dryrun { 417 | return nil 418 | } 419 | 420 | _, err := m.s3.DeleteObject(&s3.DeleteObjectInput{ 421 | Bucket: aws.String(destFile.bucket), 422 | Key: aws.String(destFile.bucketPrefix), 423 | }) 424 | if err != nil { 425 | return err 426 | } 427 | m.incrementDeletedFiles() 428 | return nil 429 | } 430 | 431 | // listS3Files return a channel which receives the file infos under the given s3Path. 432 | func (m *Manager) listS3Files(ctx context.Context, path *s3Path) chan *fileInfo { 433 | c := make(chan *fileInfo, 50000) // TODO: revisit this buffer size later 434 | 435 | go func() { 436 | defer close(c) 437 | var token *string 438 | for { 439 | if token = m.listS3FileWithToken(ctx, c, path, token); token == nil { 440 | break 441 | } 442 | } 443 | }() 444 | 445 | return c 446 | } 447 | 448 | // listS3FileWithToken lists (send to the result channel) the s3 files from the given continuation token. 449 | func (m *Manager) listS3FileWithToken(ctx context.Context, c chan *fileInfo, path *s3Path, token *string) *string { 450 | list, err := m.s3.ListObjectsV2(&s3.ListObjectsV2Input{ 451 | Bucket: &path.bucket, 452 | Prefix: &path.bucketPrefix, 453 | ContinuationToken: token, 454 | }) 455 | if err != nil { 456 | sendErrorInfoToChannel(ctx, c, err) 457 | return nil 458 | } 459 | 460 | for _, object := range list.Contents { 461 | if strings.HasSuffix(*object.Key, "/") { 462 | // Skip directory like object 463 | continue 464 | } 465 | name, err := filepath.Rel(path.bucketPrefix, *object.Key) 466 | if err != nil { 467 | sendErrorInfoToChannel(ctx, c, err) 468 | continue 469 | } 470 | var fi *fileInfo 471 | if name == "." { 472 | // Single file was specified 473 | fi = &fileInfo{ 474 | name: filepath.Base(*object.Key), 475 | path: filepath.Dir(*object.Key), 476 | size: *object.Size, 477 | lastModified: *object.LastModified, 478 | singleFile: true, 479 | } 480 | } else { 481 | fi = &fileInfo{ 482 | name: name, 483 | path: *object.Key, 484 | size: *object.Size, 485 | lastModified: *object.LastModified, 486 | } 487 | } 488 | select { 489 | case c <- fi: 490 | case <-ctx.Done(): 491 | return nil 492 | } 493 | } 494 | 495 | return list.NextContinuationToken 496 | } 497 | 498 | // updateSyncStatistics updates the statistics of the amount of bytes transferred for one file 499 | func (m *Manager) updateFileTransferStatistics(written int64) { 500 | m.statistics.mutex.Lock() 501 | defer m.statistics.mutex.Unlock() 502 | m.statistics.Files++ 503 | m.statistics.Bytes += written 504 | } 505 | 506 | // incrementDeletedFiles increments the counter used to capture the number of remote files deleted during the synchronization process 507 | func (m *Manager) incrementDeletedFiles() { 508 | m.statistics.mutex.Lock() 509 | defer m.statistics.mutex.Unlock() 510 | m.statistics.DeletedFiles++ 511 | } 512 | 513 | // listLocalFiles returns a channel which receives the infos of the files under the given basePath. 514 | // basePath have to be absolute path. 515 | func listLocalFiles(ctx context.Context, basePath string) chan *fileInfo { 516 | c := make(chan *fileInfo) 517 | 518 | basePath = filepath.ToSlash(basePath) 519 | 520 | go func() { 521 | defer close(c) 522 | 523 | stat, err := os.Stat(basePath) 524 | if os.IsNotExist(err) { 525 | // The path doesn't exist. 526 | // Returns and closes the channel without sending any. 527 | return 528 | } else if err != nil { 529 | sendErrorInfoToChannel(ctx, c, err) 530 | return 531 | } 532 | 533 | if !stat.IsDir() { 534 | sendFileInfoToChannel(ctx, c, filepath.Dir(basePath), basePath, stat, true) 535 | return 536 | } 537 | 538 | sendFileInfoToChannel(ctx, c, basePath, basePath, stat, false) 539 | 540 | err = filepath.Walk(basePath, func(path string, stat os.FileInfo, err error) error { 541 | if err != nil { 542 | return err 543 | } 544 | sendFileInfoToChannel(ctx, c, basePath, path, stat, false) 545 | return ctx.Err() 546 | }) 547 | 548 | if err != nil { 549 | sendErrorInfoToChannel(ctx, c, err) 550 | } 551 | 552 | }() 553 | return c 554 | } 555 | 556 | func sendFileInfoToChannel(ctx context.Context, c chan *fileInfo, basePath, path string, stat os.FileInfo, singleFile bool) { 557 | if stat == nil || stat.IsDir() { 558 | return 559 | } 560 | relPath, _ := filepath.Rel(basePath, path) 561 | fi := &fileInfo{ 562 | name: relPath, 563 | path: path, 564 | size: stat.Size(), 565 | lastModified: stat.ModTime(), 566 | singleFile: singleFile, 567 | } 568 | select { 569 | case c <- fi: 570 | case <-ctx.Done(): 571 | } 572 | } 573 | 574 | func sendErrorInfoToChannel(ctx context.Context, c chan *fileInfo, err error) { 575 | fi := &fileInfo{ 576 | err: err, 577 | } 578 | select { 579 | case c <- fi: 580 | case <-ctx.Done(): 581 | } 582 | } 583 | 584 | // filterFilesForSync filters the source files from the given destination files, and returns 585 | // another channel which includes the files necessary to be synced. 586 | func filterFilesForSync(sourceFileChan, destFileChan chan *fileInfo, del bool) chan *fileOp { 587 | c := make(chan *fileOp) 588 | 589 | destFiles, err := fileInfoChanToMap(destFileChan) 590 | 591 | go func() { 592 | defer close(c) 593 | if err != nil { 594 | c <- &fileOp{fileInfo: &fileInfo{err: err}} 595 | return 596 | } 597 | for sourceInfo := range sourceFileChan { 598 | destInfo, ok := destFiles[sourceInfo.name] 599 | // source is necessary to sync if 600 | // 1. The dest doesn't exist 601 | // 2. The dest doesn't have the same size as the source 602 | // 3. The dest is older than the source 603 | if !ok || sourceInfo.size != destInfo.size || sourceInfo.lastModified.After(destInfo.lastModified) { 604 | c <- &fileOp{fileInfo: sourceInfo} 605 | } 606 | if ok { 607 | destInfo.existsInSource = true 608 | } 609 | } 610 | if del { 611 | for _, destInfo := range destFiles { 612 | if !destInfo.existsInSource { 613 | // The source doesn't exist 614 | c <- &fileOp{fileInfo: destInfo, op: opDelete} 615 | } 616 | } 617 | } 618 | }() 619 | 620 | return c 621 | } 622 | 623 | // fileInfoChanToMap accumulates the fileInfos from the given channel and returns a map. 624 | // It retruns an error if the channel contains an error. 625 | func fileInfoChanToMap(files chan *fileInfo) (map[string]*fileInfo, error) { 626 | result := make(map[string]*fileInfo) 627 | 628 | for file := range files { 629 | if file.err != nil { 630 | return nil, file.err 631 | } 632 | result[file.name] = file 633 | } 634 | return result, nil 635 | } 636 | -------------------------------------------------------------------------------- /s3sync_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SEQSENSE, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | package s3sync 14 | 15 | import ( 16 | "bytes" 17 | "compress/gzip" 18 | "context" 19 | "fmt" 20 | "io/ioutil" 21 | "net/url" 22 | "os" 23 | "path/filepath" 24 | "reflect" 25 | "sort" 26 | "sync/atomic" 27 | "testing" 28 | "time" 29 | ) 30 | 31 | const dummyFilename = "README.md" 32 | 33 | func TestS3syncNotImplemented(t *testing.T) { 34 | m := New(getSession()) 35 | 36 | if err := m.Sync("foo", "bar"); err == nil { 37 | t.Fatal("local to local sync is not supported") 38 | } 39 | } 40 | 41 | func TestS3sync(t *testing.T) { 42 | data, err := ioutil.ReadFile(dummyFilename) 43 | if err != nil { 44 | t.Fatal("Failed to read", dummyFilename) 45 | } 46 | 47 | dummyFileSize := len(data) 48 | 49 | t.Run("Download", func(t *testing.T) { 50 | temp, err := ioutil.TempDir("", "s3synctest") 51 | defer os.RemoveAll(temp) 52 | 53 | if err != nil { 54 | t.Fatal("Failed to create temp dir") 55 | } 56 | 57 | destOnlyFilename := filepath.Join(temp, "dest_only_file") 58 | const destOnlyFileSize = 10 59 | if err := ioutil.WriteFile(destOnlyFilename, make([]byte, destOnlyFileSize), 0644); err != nil { 60 | t.Fatal("Failed to write", err) 61 | } 62 | 63 | tBeforeSync := time.Now() 64 | 65 | // The dummy s3 bucket has following files. 66 | // 67 | // s3://example-bucket/ 68 | // ├── README.md 69 | // ├── bar 70 | // │   └── baz 71 | // │   └── README.md 72 | // └── foo 73 | // └── README.md 74 | if err := New(getSession()).Sync("s3://example-bucket", temp); err != nil { 75 | t.Fatal("Sync should be successful", err) 76 | } 77 | 78 | fileHasSize(t, destOnlyFilename, destOnlyFileSize) 79 | 80 | for _, filename := range []string{ 81 | filepath.Join(temp, dummyFilename), 82 | filepath.Join(temp, "foo", dummyFilename), 83 | filepath.Join(temp, "bar/baz", dummyFilename), 84 | } { 85 | fileHasSize(t, filename, dummyFileSize) 86 | // Files must be made before test 87 | fileModTimeBefore(t, filename, tBeforeSync) 88 | } 89 | }) 90 | t.Run("DownloadSkipDirectory", func(t *testing.T) { 91 | temp, err := ioutil.TempDir("", "s3synctest") 92 | defer os.RemoveAll(temp) 93 | 94 | if err != nil { 95 | t.Fatal("Failed to create temp dir") 96 | } 97 | 98 | if err := New(getSession()).Sync("s3://example-bucket-directory", temp); err != nil { 99 | t.Fatal("Sync should be successful", err) 100 | } 101 | }) 102 | t.Run("DownloadSingleFile", func(t *testing.T) { 103 | temp, err := ioutil.TempDir("", "s3synctest") 104 | defer os.RemoveAll(temp) 105 | 106 | if err != nil { 107 | t.Fatal("Failed to create temp dir") 108 | } 109 | 110 | destOnlyFilename := filepath.Join(temp, "dest_only_file") 111 | const destOnlyFileSize = 10 112 | if err := ioutil.WriteFile(destOnlyFilename, make([]byte, destOnlyFileSize), 0644); err != nil { 113 | t.Fatal("Failed to write", err) 114 | } 115 | 116 | // Download to ./README.md 117 | if err := New(getSession()).Sync("s3://example-bucket/README.md", temp+"/"); err != nil { 118 | t.Fatal("Sync should be successful", err) 119 | } 120 | // Download to ./foo/README.md 121 | if err := New(getSession()).Sync("s3://example-bucket/README.md", filepath.Join(temp, "foo")+"/"); err != nil { 122 | t.Fatal("Sync should be successful", err) 123 | } 124 | // Download to ./test.md 125 | if err := New(getSession()).Sync("s3://example-bucket/README.md", filepath.Join(temp, "test.md")); err != nil { 126 | t.Fatal("Sync should be successful", err) 127 | } 128 | 129 | fileHasSize(t, destOnlyFilename, destOnlyFileSize) 130 | fileHasSize(t, filepath.Join(temp, dummyFilename), dummyFileSize) 131 | fileHasSize(t, filepath.Join(temp, "foo", dummyFilename), dummyFileSize) 132 | fileHasSize(t, filepath.Join(temp, "test.md"), dummyFileSize) 133 | }) 134 | 135 | t.Run("S3ToS3Copy", func(t *testing.T) { 136 | if err := New(getSession()).Sync("s3://s3-source", "s3://s3-destination"); err != nil { 137 | t.Fatal("Sync should be successful", err) 138 | } 139 | 140 | objs := listObjectsSorted(t, "s3-destination") 141 | if n := len(objs); n != 3 { 142 | t.Fatalf("Number of the files should be 3 (result: %v)", objs) 143 | } 144 | for _, obj := range objs { 145 | if obj.size != dummyFileSize { 146 | t.Errorf("Object size should be %d, actual %d", dummyFileSize, obj.size) 147 | } 148 | } 149 | if objs[0].path != "README.md" || 150 | objs[1].path != "bar/baz/README.md" || 151 | objs[2].path != "foo/README.md" { 152 | t.Error("Unexpected keys", objs) 153 | } 154 | }) 155 | 156 | t.Run("S3ToS3CopyWithPrefix", func(t *testing.T) { 157 | if err := New(getSession()).Sync("s3://s3-source/bar", "s3://s3-destination2/hoge"); err != nil { 158 | t.Fatal("Sync should be successful", err) 159 | } 160 | 161 | objs := listObjectsSorted(t, "s3-destination2") 162 | if n := len(objs); n != 1 { 163 | t.Fatalf("Number of the files should be 1 (result: %v)", objs) 164 | } 165 | for _, obj := range objs { 166 | if obj.size != dummyFileSize { 167 | t.Errorf("Object size should be %d, actual %d", dummyFileSize, obj.size) 168 | } 169 | } 170 | if objs[0].path != "hoge/baz/README.md" { 171 | t.Error("Unexpected keys", objs) 172 | } 173 | }) 174 | 175 | t.Run("Upload", func(t *testing.T) { 176 | temp, err := ioutil.TempDir("", "s3synctest") 177 | defer os.RemoveAll(temp) 178 | 179 | if err != nil { 180 | t.Fatal("Failed to create temp dir") 181 | } 182 | 183 | for _, dir := range []string{ 184 | filepath.Join(temp, "foo"), filepath.Join(temp, "bar", "baz"), 185 | } { 186 | if err := os.MkdirAll(dir, 0755); err != nil { 187 | t.Fatal("Failed to mkdir", err) 188 | } 189 | } 190 | 191 | for _, file := range []string{ 192 | filepath.Join(temp, dummyFilename), 193 | filepath.Join(temp, "foo", dummyFilename), 194 | filepath.Join(temp, "bar", "baz", dummyFilename), 195 | } { 196 | if err := ioutil.WriteFile(file, make([]byte, dummyFileSize), 0644); err != nil { 197 | t.Fatal("Failed to write", err) 198 | } 199 | } 200 | 201 | if err := New(getSession()).Sync(temp, "s3://example-bucket-upload"); err != nil { 202 | t.Fatal("Sync should be successful", err) 203 | } 204 | 205 | objs := listObjectsSorted(t, "example-bucket-upload") 206 | if n := len(objs); n != 4 { 207 | t.Fatalf("Number of the files should be 4 (result: %v)", objs) 208 | } 209 | for _, obj := range objs { 210 | if obj.size != dummyFileSize { 211 | t.Errorf("Object size should be %d, actual %d", dummyFileSize, obj.size) 212 | } 213 | } 214 | if objs[0].path != "README.md" || 215 | objs[1].path != "bar/baz/README.md" || 216 | objs[2].path != "dest_only_file" || 217 | objs[3].path != "foo/README.md" { 218 | t.Error("Unexpected keys", objs) 219 | } 220 | }) 221 | 222 | t.Run("UploadEscaped", func(t *testing.T) { 223 | temp, err := ioutil.TempDir("", "s3synctest") 224 | defer os.RemoveAll(temp) 225 | 226 | escapedFileName := fmt.Sprintf("SPACE %s", url.QueryEscape("READ%2FME.md")) 227 | 228 | if err != nil { 229 | t.Fatal("Failed to create temp dir") 230 | } 231 | 232 | for _, dir := range []string{ 233 | filepath.Join(temp, "foo%2Fescape space"), 234 | filepath.Join(temp, "bar", "baz%2Fescape space"), 235 | filepath.Join(temp, "pre%2Ffix space/foo%2Fescape space"), 236 | filepath.Join(temp, "pre%2Ffix space/bar", "baz%2Fescape space"), 237 | } { 238 | if err := os.MkdirAll(dir, 0755); err != nil { 239 | t.Fatal("Failed to mkdir", err) 240 | } 241 | } 242 | 243 | for _, file := range []string{ 244 | filepath.Join(temp, escapedFileName), 245 | filepath.Join(temp, "foo%2Fescape space", escapedFileName), 246 | filepath.Join(temp, "bar", "baz%2Fescape space", escapedFileName), 247 | } { 248 | if err := ioutil.WriteFile(file, make([]byte, dummyFileSize), 0644); err != nil { 249 | t.Fatal("Failed to write", err) 250 | } 251 | } 252 | 253 | if err := New(getSession()).Sync(temp, "s3://example-bucket-escaped"); err != nil { 254 | t.Fatal("Sync should be successful", err) 255 | } 256 | 257 | if err := New(getSession()).Sync(temp, "s3://example-bucket-escaped/pre%2Ffix space"); err != nil { 258 | t.Fatal("Sync should be successful", err) 259 | } 260 | 261 | objs := listObjectsSorted(t, "example-bucket-escaped") 262 | if n := len(objs); n != 6 { 263 | t.Fatalf("Number of the files should be 6 (result: %v)", objs) 264 | } 265 | 266 | objPaths := []string{} 267 | for _, obj := range objs { 268 | objPaths = append(objPaths, obj.path) 269 | if obj.size != dummyFileSize { 270 | t.Errorf("Object size should be %d, actual %d", dummyFileSize, obj.size) 271 | } 272 | } 273 | 274 | expectedKeys := []string{ 275 | "SPACE READ%252FME.md", 276 | "bar/baz%2Fescape space/SPACE READ%252FME.md", 277 | "foo%2Fescape space/SPACE READ%252FME.md", 278 | "pre%2Ffix space/SPACE READ%252FME.md", 279 | "pre%2Ffix space/bar/baz%2Fescape space/SPACE READ%252FME.md", 280 | "pre%2Ffix space/foo%2Fescape space/SPACE READ%252FME.md", 281 | } 282 | if !reflect.DeepEqual(objPaths, expectedKeys) { 283 | t.Errorf("Keys don't match\nexpected %v\nactual %v", objPaths, expectedKeys) 284 | } 285 | }) 286 | t.Run("UploadSingleFile", func(t *testing.T) { 287 | temp, err := ioutil.TempDir("", "s3synctest") 288 | defer os.RemoveAll(temp) 289 | 290 | if err != nil { 291 | t.Fatal("Failed to create temp dir") 292 | } 293 | 294 | filePath := filepath.Join(temp, dummyFilename) 295 | if err := ioutil.WriteFile(filePath, make([]byte, dummyFileSize), 0644); err != nil { 296 | t.Fatal("Failed to write", err) 297 | } 298 | 299 | if err := os.MkdirAll(filepath.Join(temp, "foo"), 0755); err != nil { 300 | t.Fatal("Failed to mkdir", err) 301 | } 302 | filePath2 := filepath.Join(temp, "foo", "test2.md") 303 | if err := ioutil.WriteFile(filePath2, make([]byte, dummyFileSize), 0644); err != nil { 304 | t.Fatal("Failed to write", err) 305 | } 306 | 307 | // Copy README.md to s3://example-bucket-upload-file/README.md 308 | if err := New(getSession()).Sync(filePath, "s3://example-bucket-upload-file"); err != nil { 309 | t.Fatal("Sync should be successful", err) 310 | } 311 | 312 | // Copy README.md to s3://example-bucket-upload-file/foo/README.md 313 | if err := New(getSession()).Sync(filePath, "s3://example-bucket-upload-file/foo/"); err != nil { 314 | t.Fatal("Sync should be successful", err) 315 | } 316 | 317 | // Copy README.md to s3://example-bucket-upload-file/foo/test.md 318 | if err := New(getSession()).Sync(filePath, "s3://example-bucket-upload-file/foo/test.md"); err != nil { 319 | t.Fatal("Sync should be successful", err) 320 | } 321 | 322 | // Copy foo/README.md to s3://example-bucket-upload-file/foo/bar/test.md 323 | if err := New(getSession()).Sync(filePath2, "s3://example-bucket-upload-file/foo/bar/test2.md"); err != nil { 324 | t.Fatal("Sync should be successful", err) 325 | } 326 | 327 | objs := listObjectsSorted(t, "example-bucket-upload-file") 328 | if n := len(objs); n != 4 { 329 | t.Fatalf("Number of the files should be 4 (result: %v)", objs) 330 | } 331 | for _, obj := range objs { 332 | if obj.size != dummyFileSize { 333 | t.Errorf("Object size should be %d, actual %d", dummyFileSize, obj.size) 334 | } 335 | } 336 | if objs[0].path != "README.md" || 337 | objs[1].path != "foo/README.md" || 338 | objs[2].path != "foo/bar/test2.md" || 339 | objs[3].path != "foo/test.md" { 340 | t.Error("Unexpected keys", objs) 341 | } 342 | }) 343 | } 344 | 345 | func TestDelete(t *testing.T) { 346 | data, err := ioutil.ReadFile(dummyFilename) 347 | if err != nil { 348 | t.Fatal("Failed to read", dummyFilename) 349 | } 350 | 351 | dummyFileSize := len(data) 352 | 353 | t.Run("DeleteLocal", func(t *testing.T) { 354 | temp, err := ioutil.TempDir("", "s3synctest") 355 | defer os.RemoveAll(temp) 356 | 357 | if err != nil { 358 | t.Fatal("Failed to create temp dir") 359 | } 360 | 361 | destOnlyFilename := filepath.Join(temp, "dest_only_file") 362 | const destOnlyFileSize = 10 363 | if err := ioutil.WriteFile(destOnlyFilename, make([]byte, destOnlyFileSize), 0644); err != nil { 364 | t.Fatal("Failed to write", err) 365 | } 366 | 367 | m := New(getSession(), WithDelete()) 368 | if err := m.Sync( 369 | "s3://example-bucket", temp, 370 | ); err != nil { 371 | t.Fatal("Sync should be successful", err) 372 | } 373 | 374 | if _, err := os.Stat(destOnlyFilename); !os.IsNotExist(err) { 375 | t.Error("Destination-only-file should be removed by sync") 376 | } 377 | 378 | fileHasSize(t, filepath.Join(temp, dummyFilename), dummyFileSize) 379 | fileHasSize(t, filepath.Join(temp, "foo", dummyFilename), dummyFileSize) 380 | fileHasSize(t, filepath.Join(temp, "bar/baz", dummyFilename), dummyFileSize) 381 | 382 | stats := m.GetStatistics() 383 | if stats.Files != 3 { 384 | t.Errorf("Expected files uploaded: %d, but found %d", 3, stats.Files) 385 | } 386 | if stats.Bytes != int64(stats.Files)*int64(dummyFileSize) { 387 | t.Errorf("Expected bytes uploaded: %d, but found %d", int64(stats.Files)*int64(dummyFileSize), stats.Bytes) 388 | } 389 | if stats.DeletedFiles != 1 { 390 | t.Errorf("Expected deleted files: %d, but found %d", 1, stats.DeletedFiles) 391 | } 392 | }) 393 | t.Run("DeleteLocalSingleFile", func(t *testing.T) { 394 | temp, err := ioutil.TempDir("", "s3synctest") 395 | defer os.RemoveAll(temp) 396 | 397 | if err != nil { 398 | t.Fatal("Failed to create temp dir") 399 | } 400 | 401 | destOnlyFilename := filepath.Join(temp, "dest_only_file") 402 | const destOnlyFileSize = 10 403 | if err := ioutil.WriteFile(destOnlyFilename, make([]byte, destOnlyFileSize), 0644); err != nil { 404 | t.Fatal("Failed to write", err) 405 | } 406 | 407 | m := New(getSession(), WithDelete()) 408 | if err := m.Sync( 409 | "s3://example-bucket/dest_only_file", destOnlyFilename, 410 | ); err != nil { 411 | t.Fatal("Sync should be successful", err) 412 | } 413 | 414 | if _, err := os.Stat(destOnlyFilename); !os.IsNotExist(err) { 415 | t.Error("Destination-only-file should be removed by sync") 416 | } 417 | stats := m.GetStatistics() 418 | if stats.Files != 0 { 419 | t.Errorf("Expected files uploaded: %d, but found %d", 0, stats.Files) 420 | } 421 | if stats.Bytes != int64(stats.Files)*int64(dummyFileSize) { 422 | t.Errorf("Expected bytes uploaded: %d, but found %d", int64(stats.Files)*int64(dummyFileSize), stats.Bytes) 423 | } 424 | if stats.DeletedFiles != 1 { 425 | t.Errorf("Expected deleted files: %d, but found %d", 1, stats.DeletedFiles) 426 | } 427 | }) 428 | t.Run("DeleteRemote", func(t *testing.T) { 429 | temp, err := ioutil.TempDir("", "s3synctest") 430 | defer os.RemoveAll(temp) 431 | 432 | if err != nil { 433 | t.Fatal("Failed to create temp dir") 434 | } 435 | 436 | for _, dir := range []string{ 437 | filepath.Join(temp, "foo"), filepath.Join(temp, "bar", "baz"), 438 | } { 439 | if err := os.MkdirAll(dir, 0755); err != nil { 440 | t.Fatal("Failed to mkdir", err) 441 | } 442 | } 443 | 444 | for _, file := range []string{ 445 | filepath.Join(temp, dummyFilename), 446 | filepath.Join(temp, "foo", dummyFilename), 447 | filepath.Join(temp, "bar", "baz", dummyFilename), 448 | } { 449 | if err := ioutil.WriteFile(file, make([]byte, dummyFileSize), 0644); err != nil { 450 | t.Fatal("Failed to write", err) 451 | } 452 | } 453 | m := New(getSession(), WithDelete()) 454 | if err := m.Sync(temp, "s3://example-bucket-delete"); err != nil { 455 | t.Fatal("Sync should be successful", err) 456 | } 457 | 458 | objs := listObjectsSorted(t, "example-bucket-delete") 459 | if n := len(objs); n != 3 { 460 | t.Fatalf("Number of the files should be 3 (result: %v)", objs) 461 | } 462 | for _, obj := range objs { 463 | if obj.size != dummyFileSize { 464 | t.Errorf("Object size should be %d, actual %d", dummyFileSize, obj.size) 465 | } 466 | } 467 | if objs[0].path != "README.md" || 468 | objs[1].path != "bar/baz/README.md" || 469 | objs[2].path != "foo/README.md" { 470 | t.Error("Unexpected keys", objs) 471 | } 472 | stats := m.GetStatistics() 473 | if stats.Files != 3 { 474 | t.Errorf("Expected files uploaded: %d, but found %d", 3, stats.Files) 475 | } 476 | if stats.Bytes != int64(stats.Files)*int64(dummyFileSize) { 477 | t.Errorf("Expected bytes uploaded: %d, but found %d", int64(stats.Files)*int64(dummyFileSize), stats.Bytes) 478 | } 479 | if stats.DeletedFiles != 1 { 480 | t.Errorf("Expected deleted files: %d, but found %d", 1, stats.DeletedFiles) 481 | } 482 | }) 483 | t.Run("DeleteRemoteSingleFile", func(t *testing.T) { 484 | temp, err := ioutil.TempDir("", "s3synctest") 485 | defer os.RemoveAll(temp) 486 | 487 | if err != nil { 488 | t.Fatal("Failed to create temp dir") 489 | } 490 | m := New(getSession(), WithDelete()) 491 | if err := m.Sync(filepath.Join(temp, "dest_only_file"), "s3://example-bucket-delete-file/dest_only_file"); err != nil { 492 | t.Fatal("Sync should be successful", err) 493 | } 494 | 495 | objs := listObjectsSorted(t, "example-bucket-delete-file") 496 | if n := len(objs); n != 1 { 497 | t.Fatalf("Number of the files should be 1 (result: %v)", objs) 498 | } 499 | if objs[0].size != dummyFileSize { 500 | t.Errorf("Object size should be %d, actual %d", dummyFileSize, objs[0].size) 501 | } 502 | if objs[0].path != "README.md" { 503 | t.Error("Unexpected keys", objs) 504 | } 505 | stats := m.GetStatistics() 506 | if stats.Files != 0 { 507 | t.Errorf("Expected files uploaded: %d, but found %d", 0, stats.Files) 508 | } 509 | if stats.Bytes != int64(stats.Files)*int64(dummyFileSize) { 510 | t.Errorf("Expected bytes uploaded: %d, but found %d", int64(stats.Files)*int64(dummyFileSize), stats.Bytes) 511 | } 512 | if stats.DeletedFiles != 1 { 513 | t.Errorf("Expected deleted files: %d, but found %d", 1, stats.DeletedFiles) 514 | } 515 | 516 | }) 517 | } 518 | 519 | func TestDryRun(t *testing.T) { 520 | data, err := ioutil.ReadFile(dummyFilename) 521 | if err != nil { 522 | t.Fatal("Failed to read", dummyFilename) 523 | } 524 | 525 | dummyFileSize := len(data) 526 | 527 | t.Run("Download", func(t *testing.T) { 528 | temp, err := ioutil.TempDir("", "s3synctest") 529 | defer os.RemoveAll(temp) 530 | 531 | if err != nil { 532 | t.Fatal("Failed to create temp dir") 533 | } 534 | 535 | destOnlyFilename := filepath.Join(temp, "dest_only_file") 536 | const destOnlyFileSize = 10 537 | if err := ioutil.WriteFile(destOnlyFilename, make([]byte, destOnlyFileSize), 0644); err != nil { 538 | t.Fatal("Failed to write", err) 539 | } 540 | 541 | m := New(getSession(), WithDelete(), WithDryRun()) 542 | if err := m.Sync( 543 | "s3://example-bucket", temp, 544 | ); err != nil { 545 | t.Fatal("Sync should be successful", err) 546 | } 547 | 548 | fileHasSize(t, destOnlyFilename, destOnlyFileSize) 549 | 550 | if _, err := os.Stat(filepath.Join(temp, dummyFilename)); !os.IsNotExist(err) { 551 | t.Error("File must not be downloaded on dry-run") 552 | } 553 | if _, err := os.Stat(filepath.Join(temp, "foo", dummyFilename)); !os.IsNotExist(err) { 554 | t.Error("File must not be downloaded on dry-run") 555 | } 556 | if _, err := os.Stat(filepath.Join(temp, "bar/baz", dummyFilename)); !os.IsNotExist(err) { 557 | t.Error("File must not be downloaded on dry-run") 558 | } 559 | stats := m.GetStatistics() 560 | if !reflect.DeepEqual(&stats, &SyncStatistics{}) { 561 | t.Error("Statistics must not change on a dry-run") 562 | } 563 | 564 | }) 565 | t.Run("Upload", func(t *testing.T) { 566 | temp, err := ioutil.TempDir("", "s3synctest") 567 | defer os.RemoveAll(temp) 568 | 569 | if err != nil { 570 | t.Fatal("Failed to create temp dir") 571 | } 572 | 573 | for _, dir := range []string{ 574 | filepath.Join(temp, "foo"), filepath.Join(temp, "bar", "baz"), 575 | } { 576 | if err := os.MkdirAll(dir, 0755); err != nil { 577 | t.Fatal("Failed to mkdir", err) 578 | } 579 | } 580 | 581 | for _, file := range []string{ 582 | filepath.Join(temp, dummyFilename), 583 | filepath.Join(temp, "foo", dummyFilename), 584 | filepath.Join(temp, "bar", "baz", dummyFilename), 585 | } { 586 | if err := ioutil.WriteFile(file, make([]byte, dummyFileSize), 0644); err != nil { 587 | t.Fatal("Failed to write", err) 588 | } 589 | } 590 | 591 | m := New(getSession(), WithDelete(), WithDryRun()) 592 | if err := m.Sync(temp, "s3://example-bucket-dryrun"); err != nil { 593 | t.Fatal("Sync should be successful", err) 594 | } 595 | 596 | objs := listObjectsSorted(t, "example-bucket-dryrun") 597 | if n := len(objs); n != 1 { 598 | t.Fatalf("Number of the files should be 1 (result: %v)", objs) 599 | } 600 | if n := objs[0].size; n != dummyFileSize { 601 | t.Errorf("Object size should be %d, actual %d", dummyFileSize, n) 602 | } 603 | if objs[0].path != "dest_only_file" { 604 | t.Error("Unexpected key", objs[0].path) 605 | } 606 | stats := m.GetStatistics() 607 | if !reflect.DeepEqual(&stats, &SyncStatistics{}) { 608 | t.Error("Statistics must not change on a dry-run") 609 | } 610 | }) 611 | } 612 | 613 | func TestPartialS3sync(t *testing.T) { 614 | data, err := ioutil.ReadFile(dummyFilename) 615 | if err != nil { 616 | t.Fatal("Failed to read", dummyFilename) 617 | } 618 | 619 | expectedFileSize := len(data) 620 | 621 | temp, err := ioutil.TempDir("", "s3synctest") 622 | defer os.RemoveAll(temp) 623 | 624 | if err != nil { 625 | t.Fatal("Failed to create temp dir") 626 | } 627 | 628 | var syncCount uint32 629 | SetLogger(createLoggerWithLogFunc(func(v ...interface{}) { 630 | atomic.AddUint32(&syncCount, 1) // This function is called once per one download 631 | })) 632 | 633 | assertFileSize := func(t *testing.T) { 634 | fileHasSize(t, filepath.Join(temp, dummyFilename), expectedFileSize) 635 | fileHasSize(t, filepath.Join(temp, "foo", dummyFilename), expectedFileSize) 636 | fileHasSize(t, filepath.Join(temp, "bar/baz", dummyFilename), expectedFileSize) 637 | } 638 | 639 | t.Run("DestinationEmpty", func(t *testing.T) { 640 | atomic.StoreUint32(&syncCount, 0) 641 | m := New(getSession()) 642 | if err := m.Sync("s3://example-bucket", temp); err != nil { 643 | t.Fatal("Sync should be successful", err) 644 | } 645 | 646 | if atomic.LoadUint32(&syncCount) != 3 { 647 | t.Fatal("3 files should be synced") 648 | } 649 | assertFileSize(t) 650 | 651 | stats := m.GetStatistics() 652 | if stats.Files != 3 { 653 | t.Errorf("Expected files uploaded: %d, but found %d", 3, stats.Files) 654 | } 655 | if stats.Bytes != int64(stats.Files)*int64(expectedFileSize) { 656 | t.Errorf("Expected bytes uploaded: %d, but found %d", int64(stats.Files)*int64(expectedFileSize), stats.Bytes) 657 | } 658 | if stats.DeletedFiles != 0 { 659 | t.Errorf("Expected deleted files: %d, but found %d", 0, stats.DeletedFiles) 660 | } 661 | 662 | }) 663 | 664 | t.Run("DestinationLackOneFile", func(t *testing.T) { 665 | atomic.StoreUint32(&syncCount, 0) 666 | 667 | os.RemoveAll(filepath.Join(temp, "foo")) 668 | 669 | m := New(getSession()) 670 | if m.Sync("s3://example-bucket", temp) != nil { 671 | t.Fatal("Sync should be successful") 672 | } 673 | 674 | if n := atomic.LoadUint32(&syncCount); n != 1 { 675 | t.Fatalf("Only 1 file should be synced, %d files synced", n) 676 | } 677 | 678 | assertFileSize(t) 679 | stats := m.GetStatistics() 680 | if stats.Files != 1 { 681 | t.Errorf("Expected files uploaded: %d, but found %d", 1, stats.Files) 682 | } 683 | if stats.Bytes != int64(stats.Files)*int64(expectedFileSize) { 684 | t.Errorf("Expected bytes uploaded: %d, but found %d", int64(stats.Files)*int64(expectedFileSize), stats.Bytes) 685 | } 686 | if stats.DeletedFiles != 0 { 687 | t.Errorf("Expected deleted files: %d, but found %d", 0, stats.DeletedFiles) 688 | } 689 | 690 | }) 691 | 692 | t.Run("DestinationOneOldFile", func(t *testing.T) { 693 | atomic.StoreUint32(&syncCount, 0) 694 | 695 | oldTime := time.Date(1980, time.January, 1, 0, 0, 0, 0, time.UTC) 696 | filename := filepath.Join(temp, "README.md") 697 | os.Chtimes(filename, oldTime, oldTime) 698 | 699 | m := New(getSession()) 700 | if m.Sync("s3://example-bucket", temp) != nil { 701 | t.Fatal("Sync should be successful") 702 | } 703 | 704 | if n := atomic.LoadUint32(&syncCount); n != 1 { 705 | t.Fatalf("Only 1 file should be synced, %d files synced", n) 706 | } 707 | 708 | stat, err := os.Stat(filename) 709 | if err != nil { 710 | t.Fatal(err) 711 | } 712 | if !stat.ModTime().After(oldTime) { 713 | t.Errorf("File modification time must be updated, expected: ModTime>%v, actual: %v<=%v", oldTime, stat.ModTime(), oldTime) 714 | } 715 | 716 | assertFileSize(t) 717 | stats := m.GetStatistics() 718 | if stats.Files != 1 { 719 | t.Errorf("Expected files uploaded: %d, but found %d", 1, stats.Files) 720 | } 721 | if stats.Bytes != int64(stats.Files)*int64(expectedFileSize) { 722 | t.Errorf("Expected bytes uploaded: %d, but found %d", int64(stats.Files)*int64(expectedFileSize), stats.Bytes) 723 | } 724 | if stats.DeletedFiles != 0 { 725 | t.Errorf("Expected deleted files: %d, but found %d", 0, stats.DeletedFiles) 726 | } 727 | }) 728 | } 729 | 730 | func TestListLocalFiles(t *testing.T) { 731 | temp, err := ioutil.TempDir("", "s3synctest") 732 | defer os.RemoveAll(temp) 733 | 734 | if err != nil { 735 | t.Fatal("Failed to create temp dir") 736 | } 737 | 738 | for _, dir := range []string{ 739 | filepath.Join(temp, "empty"), 740 | filepath.Join(temp, "foo"), 741 | filepath.Join(temp, "bar", "baz"), 742 | } { 743 | if err := os.MkdirAll(dir, 0755); err != nil { 744 | t.Fatal("Failed to mkdir", err) 745 | } 746 | } 747 | 748 | for _, file := range []string{ 749 | filepath.Join(temp, "test1"), 750 | filepath.Join(temp, "foo", "test2"), 751 | filepath.Join(temp, "bar", "baz", "test3"), 752 | } { 753 | if err := ioutil.WriteFile(file, make([]byte, 10), 0644); err != nil { 754 | t.Fatal("Failed to write", err) 755 | } 756 | } 757 | 758 | collectFilePaths := func(ch chan *fileInfo) []string { 759 | list := []string{} 760 | for f := range ch { 761 | list = append(list, f.path) 762 | } 763 | sort.Strings(list) 764 | return list 765 | } 766 | 767 | t.Run("Root", func(t *testing.T) { 768 | paths := collectFilePaths(listLocalFiles(context.Background(), temp)) 769 | expected := []string{ 770 | filepath.Join(temp, "bar", "baz", "test3"), 771 | filepath.Join(temp, "foo", "test2"), 772 | filepath.Join(temp, "test1"), 773 | } 774 | if !reflect.DeepEqual(expected, paths) { 775 | t.Errorf("Local file list is expected to be %v, got %v", expected, paths) 776 | } 777 | }) 778 | 779 | t.Run("EmptyDir", func(t *testing.T) { 780 | paths := collectFilePaths(listLocalFiles(context.Background(), filepath.Join(temp, "empty"))) 781 | expected := []string{} 782 | if !reflect.DeepEqual(expected, paths) { 783 | t.Errorf("Local file list is expected to be %v, got %v", expected, paths) 784 | } 785 | }) 786 | 787 | t.Run("File", func(t *testing.T) { 788 | paths := collectFilePaths(listLocalFiles(context.Background(), filepath.Join(temp, "test1"))) 789 | expected := []string{ 790 | filepath.Join(temp, "test1"), 791 | } 792 | if !reflect.DeepEqual(expected, paths) { 793 | t.Errorf("Local file list is expected to be %v, got %v", expected, paths) 794 | } 795 | }) 796 | 797 | t.Run("Dir", func(t *testing.T) { 798 | paths := collectFilePaths(listLocalFiles(context.Background(), filepath.Join(temp, "foo"))) 799 | expected := []string{ 800 | filepath.Join(temp, "foo", "test2"), 801 | } 802 | if !reflect.DeepEqual(expected, paths) { 803 | t.Errorf("Local file list is expected to be %v, got %v", expected, paths) 804 | } 805 | }) 806 | 807 | t.Run("Dir2", func(t *testing.T) { 808 | paths := collectFilePaths(listLocalFiles(context.Background(), filepath.Join(temp, "bar"))) 809 | expected := []string{ 810 | filepath.Join(temp, "bar", "baz", "test3"), 811 | } 812 | if !reflect.DeepEqual(expected, paths) { 813 | t.Errorf("Local file list is expected to be %v, got %v", expected, paths) 814 | } 815 | }) 816 | } 817 | 818 | func TestS3sync_GuessMime(t *testing.T) { 819 | data, err := ioutil.ReadFile(dummyFilename) 820 | if err != nil { 821 | t.Fatal("Failed to read", dummyFilename) 822 | } 823 | 824 | temp, err := ioutil.TempDir("", "s3synctest") 825 | defer os.RemoveAll(temp) 826 | 827 | if err != nil { 828 | t.Fatal("Failed to create temp dir", err) 829 | } 830 | 831 | var buf bytes.Buffer 832 | zw := gzip.NewWriter(&buf) 833 | if _, err := zw.Write(data); err != nil { 834 | t.Fatal(err) 835 | } 836 | 837 | if err := ioutil.WriteFile(filepath.Join(temp, dummyFilename), buf.Bytes(), 0644); err != nil { 838 | t.Fatal("Failed to write", err) 839 | } 840 | 841 | testCases := map[string]struct { 842 | options []Option 843 | expected string 844 | }{ 845 | "Guess": { 846 | options: []Option{}, 847 | expected: "application/gzip", 848 | }, 849 | "NoGuess": { 850 | options: []Option{WithoutGuessMimeType()}, 851 | expected: "binary/octet-stream", 852 | }, 853 | "Overwrite": { 854 | options: []Option{WithContentType("test/dummy")}, 855 | expected: "test/dummy", 856 | }, 857 | } 858 | for name, tt := range testCases { 859 | tt := tt 860 | t.Run(name, func(t *testing.T) { 861 | deleteObject(t, "example-bucket-mime", dummyFilename) 862 | 863 | if err := New(getSession(), tt.options...).Sync(temp, "s3://example-bucket-mime"); err != nil { 864 | t.Fatal("Sync should be successful", err) 865 | } 866 | 867 | objs := listObjectsSorted(t, "example-bucket-mime") 868 | if n := len(objs); n != 1 { 869 | t.Fatalf("Number of the files should be 1 (result: %v)", objs) 870 | } 871 | if objs[0].contentType != tt.expected { 872 | t.Errorf("Object ContentType should be %s, actual %s", tt.expected, objs[0].contentType) 873 | } 874 | }) 875 | } 876 | } 877 | 878 | type dummyLogger struct { 879 | log func(...interface{}) 880 | } 881 | 882 | func (d *dummyLogger) Log(v ...interface{}) { 883 | d.log(v...) 884 | } 885 | func (d *dummyLogger) Logf(format string, v ...interface{}) { 886 | } 887 | 888 | func createLoggerWithLogFunc(log func(v ...interface{})) LoggerIF { 889 | return &dummyLogger{log: log} 890 | } 891 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 SEQSENSE, Inc. 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | package s3sync 14 | 15 | import ( 16 | "io/ioutil" 17 | "os" 18 | "sort" 19 | "testing" 20 | "time" 21 | 22 | "github.com/aws/aws-sdk-go/aws" 23 | "github.com/aws/aws-sdk-go/aws/session" 24 | "github.com/aws/aws-sdk-go/service/s3" 25 | ) 26 | 27 | const awsRegion = "ap-northeast-1" 28 | 29 | func getSession() *session.Session { 30 | sess, _ := session.NewSession(&aws.Config{ 31 | Region: aws.String(awsRegion), 32 | S3ForcePathStyle: aws.Bool(true), 33 | Endpoint: aws.String("http://localhost:4572"), 34 | }) 35 | return sess 36 | } 37 | 38 | type s3Object struct { 39 | path string 40 | size int 41 | contentType string 42 | } 43 | 44 | type s3ObjectList []s3Object 45 | 46 | func (l s3ObjectList) Len() int { 47 | return len(l) 48 | } 49 | func (l s3ObjectList) Less(i, j int) bool { 50 | return l[i].path < l[j].path 51 | } 52 | func (l s3ObjectList) Swap(i, j int) { 53 | l[i], l[j] = l[j], l[i] 54 | } 55 | 56 | func deleteObject(t *testing.T, bucket, key string) { 57 | svc := s3.New(session.New(&aws.Config{ 58 | Region: aws.String(awsRegion), 59 | Endpoint: aws.String("http://localhost:4572"), 60 | S3ForcePathStyle: aws.Bool(true), 61 | })) 62 | 63 | _, err := svc.DeleteObject(&s3.DeleteObjectInput{ 64 | Bucket: &bucket, 65 | Key: &key, 66 | }) 67 | if err != nil { 68 | t.Fatal("DeleteObject failed", err) 69 | } 70 | } 71 | 72 | func listObjectsSorted(t *testing.T, bucket string) []s3Object { 73 | svc := s3.New(session.New(&aws.Config{ 74 | Region: aws.String(awsRegion), 75 | Endpoint: aws.String("http://localhost:4572"), 76 | S3ForcePathStyle: aws.Bool(true), 77 | })) 78 | 79 | result, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{ 80 | Bucket: &bucket, 81 | MaxKeys: aws.Int64(100), 82 | }) 83 | if err != nil { 84 | t.Fatal("ListObjects failed", err) 85 | } 86 | var objs []s3Object 87 | for _, obj := range result.Contents { 88 | o, err := svc.GetObject(&s3.GetObjectInput{ 89 | Bucket: &bucket, 90 | Key: obj.Key, 91 | }) 92 | if err != nil { 93 | t.Fatal("GetObject failed", err) 94 | } 95 | objs = append(objs, s3Object{ 96 | path: *obj.Key, 97 | size: int(*obj.Size), 98 | contentType: *o.ContentType, 99 | }) 100 | } 101 | sort.Sort(s3ObjectList(objs)) 102 | return objs 103 | } 104 | 105 | func fileHasSize(t *testing.T, filename string, expectedSize int) { 106 | data, err := ioutil.ReadFile(filename) 107 | if err != nil { 108 | t.Error(filename, "is not synced") 109 | return 110 | } 111 | if n := len(data); n != expectedSize { 112 | t.Errorf("%s is not synced (file size is expected to be %d, actual %d)", filename, expectedSize, n) 113 | } 114 | } 115 | 116 | func fileModTimeBefore(t *testing.T, filename string, t0 time.Time) { 117 | info, err := os.Stat(filename) 118 | if err != nil { 119 | t.Error("Failed to get stat:", err) 120 | return 121 | } 122 | if t1 := info.ModTime(); !t1.Before(t0) { 123 | t.Errorf("File modification time %v is later than %v", t1, t0) 124 | } 125 | } 126 | --------------------------------------------------------------------------------