├── .gitignore ├── LICENSE ├── README.md ├── cli.go ├── config.go ├── config.toml ├── docs ├── api.md ├── config.md ├── diagram │ ├── client.drawio │ ├── client.jpg │ ├── demux.drawio │ ├── demux.jpg │ └── sender&receiver.drawio ├── reports │ └── v0.1.0实现.pdf ├── rsync_subprotocol.md └── v0.2.x_Zh-CN.md ├── go.mod ├── go.sum ├── main.go ├── rsync ├── attribs.go ├── chksum.go ├── client.go ├── compress.go ├── connection.go ├── const.go ├── demux.go ├── demux_old.go ├── exclude.go ├── flist.go ├── fs.go ├── receiver.go ├── sender.go ├── ssh.go └── utils.go └── storage ├── cache ├── finfo.pb.go ├── finfo.proto ├── pbgen.sh ├── redis.go └── utils.go ├── local.go └── minio.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | .idea/ 18 | 19 | # boltdb 20 | *.db -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RSYNC-OS 2 | ## A port of rsync written in Go, with object storage support 3 | 4 | # Introduction 5 | rsync-os is a Golang implementation of Rsync built from scratch. Its goal is to be a modern rsync-compatible tool with features like: 6 | * Supports file storage and object storage 7 | * Uses a subset of rsync wire protocol to avoid blocks efficient transfer (For sender and receiver) 8 | * Rsync APIs 9 | * Safe 10 | 11 | ![client](https://raw.githubusercontent.com/kaiakz/rsync-os/master/docs/client.jpg) 12 | 13 | ## Usage 14 | ### Use minio as backend 15 | 1. install & run minio, you need to configure the `config.toml`. 16 | 2. `go build` 17 | 3. `./rsync-os rsync://[USER@]HOST[:PORT]/SRC minio`, for example, `./rsync-os rsync://mirrors.tuna.tsinghua.edu.cn/ubuntu minio` 18 | 19 | ## Roadmap 20 | ### Client 21 | #### Rsync wire protocol 27: 22 | - [x] Connect to rsync daemon by rsync://: Handshake, Fetches the file list, Requests & download files 23 | - [ ] Handles error 24 | #### Storage backend 25 | - [x] Minio: supports regular files, supports folder & symlink 26 | - [x] Minio: caches file list in boltdb 27 | - [ ] Local 28 | #### Other 29 | - [x] CLI 30 | 31 | ## Detailed Information 32 | #### openrsync has a really good [documentation](https://github.com/kristapsdz/openrsync/blob/master/README.md) to describe how rsync algorithm works. 33 | 34 | #### Why do this? 35 | Just as [rsyn](https://github.com/sourcefrog/rsyn#why-do-this) said, "The rsync C code is quite convoluted, with many interacting options and parameters stored in global variables affecting many different parts of the control flow, including how structures are encoded and decoded." I would like to provide a rsync written in clean and understandable Golang code. 36 | 37 | rsync has [a bad performance](https://github.com/tuna/rsync/blob/master/README-huai.md). Inspired by rsync-huai, rsync-os stores the file list in database to avoid recursively generating the list. 38 | 39 | Modernized rsync: rsync-os supports both file storage and object storage. 40 | 41 | #### What's the difference between rsync and rsync-os 42 | rsync-os is the express edition of rsync, with object storage support. It uses a subset of rsync wire protocol(without rolling block checksum). 43 | 44 | #### rsync-os and rclone are completely different 45 | rclone does not support rsync wire protocol although it is called "rsync for object storage". With rclone you can't transfer files between rsync and object storage. 46 | 47 | #### Why we don't need rolling block checksum for regular file? 48 | In the rsync algorithm, rsync requires random access of files to do the block exchange. But object storage does not support that. 49 | 50 | rsync-os simplifies the rsync algorithm to avoid random reading and writing, since rsync-os don't need to do a rolling checksum scanning the file. 51 | 52 | As a client, when a file has different size or modified time compared to the remote file, rsync-os just pretend 'the file does not exist here', then send a reply to download the entire file from the server and finally replace it. 53 | 54 | As a server, TBC 55 | 56 | #### HandShake 57 | rysnc-os supports rsync protocol 27. 58 | Now it sends the arguments "--server--sender-l-p-r-t" to the remote rsyncd by default. 59 | 60 | #### The File List 61 | According to the arguments rsync-os sent, the file list should contain path, size, modified time & mode. 62 | 63 | #### Request the file 64 | rsync-os always saves the file list in its database(local file list). rsync2os doesn't compare each file with the file list from remote server(remote file list), but the latest local file list. If the file in the local file list has different size, modified time, or doesn't exist, rsync2os will download the whole file(without [block exchange](https://github.com/kristapsdz/openrsync#block-exchange)). To to do that, rsync2os sends the empty block checksum with the file's index of the remote file list. 65 | 66 | #### Download the file 67 | The rsync server sends the entire file as a stream of bytes. 68 | 69 | #### Multiplex & De-Multiplex 70 | Most rsync transmissions are wrapped in a multiplexing envelope protocol. The code written in C to multiplex & de-multiplex is obscure. 71 | Unlike rsync, rsync-os reimplements this part: It just does multiplexing & de-multiplexing in a goroutine. 72 | ![de-multiplex](https://raw.githubusercontent.com/kaiakz/rsync-os/master/docs/demux.jpg) 73 | 74 | ### Limitations 75 | * Do not support block exchange for regular files. If a file was modified, just downloads the whole file. 76 | * rsync-os can only act as client/receiver now. 77 | 78 | # Reference 79 | * [rsync](https://rsync.samba.org/) 80 | * [openrsync](https://github.com/openbsd/src/tree/master/usr.bin/rsync), a BSD-liscesed rsync 81 | * [rsync-huai](https://github.com/tuna/rsync), a modified version rsync by Tsinghua University TUNA Association 82 | * [yajsync](https://github.com/perlundq/yajsync), a port of rsync written in Java 83 | * [rsyn](https://github.com/sourcefrog/rsyn), wire-compaible rsync in Rust 84 | * [acrosync-library](https://github.com/gilbertchen/acrosync-library) 85 | * [repositoryd](https://github.com/APNIC-net/repositoryd), An rsync-compatible RPKI repository daemon. 86 | * https://rsync.samba.org/resources.html 87 | * https://github.com/boundary/wireshark/blob/master/epan/dissectors/packet-rsync.c 88 | * https://tools.ietf.org/html/rfc5781 -------------------------------------------------------------------------------- /cli.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | type Cli struct { 4 | Src string 5 | Dest string 6 | // Flags -aDglnoprtvx 7 | } 8 | 9 | /* 10 | Usage: rsync [OPTION]... SRC [SRC]... DEST 11 | or rsync [OPTION]... SRC [SRC]... [USER@]HOST:DEST 12 | or rsync [OPTION]... SRC [SRC]... [USER@]HOST::DEST 13 | or rsync [OPTION]... SRC [SRC]... rsync://[USER@]HOST[:PORT]/DEST 14 | or rsync [OPTION]... [USER@]HOST:SRC [DEST] 15 | or rsync [OPTION]... [USER@]HOST::SRC [DEST] 16 | or rsync [OPTION]... rsync://[USER@]HOST[:PORT]/SRC [DEST] 17 | */ 18 | //func ParseArgs() *Cli { 19 | //cli := &Cli{} 20 | 21 | //syncCommand := flag.NewFlagSet("sync", flag.ExitOnError) 22 | //recoverCommand := flag.NewFlagSet("recover", flag.ExitOnError) 23 | //lsCommand := flag.NewFlagSet("ls", flag.ExitOnError) 24 | //testCommand := flag.NewFlagSet("test", flag.ExitOnError) 25 | 26 | //} -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/spf13/viper" 5 | "io/ioutil" 6 | "log" 7 | ) 8 | 9 | // Create a default config file if not found config.toml 10 | func loadConfigIfExists() { 11 | viper.SetConfigName("config") // name of config file (without extension) 12 | viper.SetConfigType("toml") // REQUIRED if the config file does not have the extension in the name 13 | //viper.AddConfigPath("/etc/rsync-os/") // path to look for the config file in 14 | //viper.AddConfigPath("$HOME/.rsync-os") // call multiple times to add many search paths 15 | viper.AddConfigPath(".") // optionally look for config in the working directory 16 | 17 | err := viper.ReadInConfig() // Find and read the config file 18 | if err != nil { // Handle errors reading the config file 19 | if _, ok := err.(viper.ConfigFileNotFoundError); ok { 20 | // Config File not found 21 | 22 | createSampleConfig() 23 | log.Fatalln("Config does not exist, a sample of config was created") 24 | } else { 25 | // Found but got errors 26 | log.Fatalln(err) 27 | } 28 | } 29 | } 30 | 31 | func createSampleConfig() { 32 | confSample := []byte( 33 | `title = "configuration of rsync-os" 34 | 35 | # [object storage's name] 36 | [minio] 37 | endpoint = "127.0.0.1:9000" 38 | keyAccess = "minioadmin" 39 | keySecret = "minioadmin" 40 | [minio.boltdb] 41 | path = "test.db" 42 | `) 43 | 44 | if ioutil.WriteFile("config.toml", confSample, 0666) != nil { 45 | log.Fatalln("Can't create a sample of config") 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /config.toml: -------------------------------------------------------------------------------- 1 | title = "configuration of rsync-os" 2 | 3 | # [object storage's name] 4 | [minio] 5 | endpoint = "127.0.0.1:9000" 6 | keyAccess = "minioadmin" 7 | keySecret = "minioadmin" 8 | [minio.boltdb] 9 | path = "test.db" 10 | -------------------------------------------------------------------------------- /docs/api.md: -------------------------------------------------------------------------------- 1 | # rsync API 2 | **STATUS: UNSTABLE** With the API, you can create an rsync receiver/sender and customize how it works by your arguments and callbacks. 3 | 4 | ### Preknowledge 5 | https://rsync.samba.org/how-rsync-works.html 6 | 7 | ## v0.2.0 8 | ### Client 9 | #### receiver: 10 | First of all, you need to implement the rsync.FS interface to specify a storage backend used for the task. 11 | ```go 12 | type FS interface { 13 | Put(fileName string, content io.Reader, fileSize int64, metadata FileMetadata) (written int64, err error) 14 | Get(fileName string, metadata FileMetadata) (File, error) 15 | Delete(fileName string, mode FileMode) error 16 | List() (FileList, error) 17 | Stats() (seekable bool) 18 | } 19 | ``` 20 | Then, decide what arguments you would like to use: 21 | ```go 22 | type Attribs struct { 23 | sender bool // --sender 24 | server bool // --server 25 | recursive bool // -r 26 | dryRun bool // -n 27 | hasTimes bool // -t 28 | hasPerms bool // -p 29 | hasLinks bool // -l 30 | hasGID bool // -g 31 | hasUID bool // -u 32 | } 33 | ``` 34 | 35 | ```go 36 | // via socket 37 | func SocketClient(storage FS, address string, module string, path string, options map[string]string) (SendReceiver, error) 38 | 39 | // via ssh 40 | func SshClient(storage FS, address string, module string, path string, options map[string]string) (SendReceiver, error) 41 | ``` 42 | Call `Run()` for the receiver to start a syncing task. 43 | -------------------------------------------------------------------------------- /docs/config.md: -------------------------------------------------------------------------------- 1 | # Configration Format 2 | **NOTE** The format is unstable, it may be changed in the future version. 3 | The `config.toml` is the runtime configuration for rsync-os (both daemon/server and client). Similiar to `rsyncd.conf`, it can control authentication, access, logging, available modules and which storage backend rsync-os uses(local file storage, s3 object storage). 4 | 5 | A storage backend begins with the name of the baackend in the square brackets and continues until the next square brackets. 6 | 7 | * To configure a s3 object storage: 8 | ```toml 9 | [minio] 10 | endpoint = "192.168.11.192:9000" # Require 11 | keyAccess = "minioadmin" # Require 12 | keySecret = "minioadmin" # Require 13 | mods = [] # Unlike rsync, all module will specify in the storage backend. 14 | ```` 15 | 16 | -------------------------------------------------------------------------------- /docs/diagram/client.drawio: -------------------------------------------------------------------------------- 1 | 5Vhbb9owFP41SNsDU0hCWh5XoO2u0kQ7HicTm8TFsTPH3PbrZ8fOxcRj6za2jkkI4nNOHM73nZvTC8bZ7oaDPH3HICI934O7XjDp+f4g9P2e+nhwryUXwUALEo6hMWoEM/wFGaFnpGsMUWEZCsaIwLktjBmlKBaWDHDOtrbZkhH7qTlIUEcwiwHpSucYilRLL/2LRn6LcJJWTx5EI63JQGVsPClSANm2JQqmvWDMGRP6KtuNEVHgVbjMX+3n5O0qunn9ofgM7q/e3L3/2NebXT/mltoFjqj46a1j/Gn+aTSa7Fej+zf8Yf56Cry+IXcDyNrgZXwV+wpABCWeZsm4SFnCKCDTRnrF2ZpCpB7jyVVj85axXAoHUviAhNib4ABrwaQoFRkx2h/0z+BQsDWP0RG7wIQZ4AkSR5w3vCoHW7Fi0LtBLEOC76UBRwQIvLEDCpi4TGq7Bnt5YeB/BMtBh4pbQKEMuhXqcNIgruDbpligWQ5KULYyjW10l4wKA/1AuneVEFAUhqxCcLaqE0NZ11HuPZqYDeIC7Y5CWWkjg/3eLhbbJkMHVdqlrewMvV8H3xkK1R84r0SIuongtLv4W4ng5iL8T7hweh88qaIUOaiIiFB1IgfU4iT6vFadsCw3veCl1A38fNdTaXytFi/88joCWV6iGwShAhyRDRI4Bh2Nsi6/ZU0RfUBwQvW2sSQE8VodM8K41vBk8cwrnyK99VpXz+3NBAe0WDKe6dsoo8g2wDJ0Ki887USpJDJiEO9L32NMk+pungFi3w9RzLikh1HHE7aMQ3uL9iMgLnIC9lqBKcGtO5eEAeHYkcnKK3Xbvi7+jn+1APEqKVOif4CXH17WOPnhqLkeKtQqUuVVon7HLFMZofxM1beMQVQI9dcwKde4XGEVGjGIlY0OlwWvtqgkMix0DFXiwx6XsmyxLr7f36yOpaLvGmSYqM5yWwdXtwuagJqYaHK2woqjSdSs7sry0Q9P2R2HQ7s7Vut2ewwc7XH0G9rjsQbRKgOzFc7PbiwJLg6Av/yDc4nbhe6Afp9DmXR1Bn478c6MnEFoz4yBi5zIQc7wZENjd2T/twaVY6ei7w6N0ZMaGrvz+zUScdpJkCVnmUIB8Y2aIc4tRw7OVT9av06XIq4B8p9PEccLhqPV+4nkSLdcTdiWypkSWt2k0APgOm80KmUkpnLSXDyol3SKPzngJufYZYYv7CHAmUThqJtE0cmSaOhIInNaootC/Ux3urzV8uos5c1ApogtFMwlr5nAWZc3dWw5GKtLAsb6tDApzxuSMUzIgagzSCuw5cxNXhpFhiEs09gVDXZqQ1CkdfCcimL/8mDKi7oEjxxFMng8v3LZvB8uda237MH0Kw== -------------------------------------------------------------------------------- /docs/diagram/client.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaiakz/rsync-os/64e84daeabb1fa4d2c7cf766c196306adfba6cb2/docs/diagram/client.jpg -------------------------------------------------------------------------------- /docs/diagram/demux.drawio: -------------------------------------------------------------------------------- 1 | 5ZjbctowEIafxpfM+IAhXNaQkqahMw0lXCu2YqvIWiqvOeTpK2EZY+whh07SlM5wgX4tMvv92hXC8obpZizJMplARLnl2tHG8kaW6zpd17X0y462hdLv9QohliwyQZUwZY/UiLZRcxbRrBaIABzZsi6GIAQNsaYRKWFdD3sAXn/qksS0IUxDwpvqnEWYFOqF26/0K8ripHyy0xsUMykpg00mWUIiWB9I3qXlDSUAFu/SzZByDa/kMv+ynfObRW98/T37RWbB1x/f7jrFYp9f8pF9CpIKfPXSPgbJoiNugtkMHu9Gs9vw6rrjFUuvCM8NL5MrbkuANFI8zRAkJhCDIPyyUgMJuYiofoytRlXMDcBSiY4Sf1LErdkcJEdQUoIpN7PPzM9wyCCXIT0RZ5JCImOKJ5I3ueoED/aKoTemkFKUWxUgKSfIVvUNRcy+jPdxFXv1xuB/gctNK26zrQh1wlThlQ1jKuya4TphSKdLsiOzVrVcR/wAAg1/R+UYxJxkmXEsQwmLfXXo6P1Wt1/szopKpJuTPM1s19SVaSz7OltXZeqUWnJQohf2nzvQuh+cdgc6kJ0de9/5aPC7/3gnOtVhnuxEzt/qRKe+9YEVIzrJN8OEiLMrBLfnf7BC8Bv0x6AoIxO0QV8liXXEBcYhcJBKESB0WTwwzo8kwlks1DBUPNXZ4gUaGVM/mz6ZiZRF0a6m2jyt19mbeTOoe+M5TW96Lda4b2VNv2HNFMKFquynfXk+b0kz9kjud0tpuktgAneZ+IHlj/Raqm1lpozeEL8zODoj+k38/nviv2jg1y1J6AvLOfLv2oMn+b/r9h80+E9yfZHjdKN6gboiEiTnaYVzfEp4LaeE/Z5elK2wdkin/4kd3nFnarOj7dB+hR1qWF3xd3MHf5R4l78B -------------------------------------------------------------------------------- /docs/diagram/demux.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaiakz/rsync-os/64e84daeabb1fa4d2c7cf766c196306adfba6cb2/docs/diagram/demux.jpg -------------------------------------------------------------------------------- /docs/diagram/sender&receiver.drawio: -------------------------------------------------------------------------------- 1 | 7VhdU6MwFP01fVyHj9LSR9tad2fsrqPO6D7tRLiWTAPBEJT66zeBAKXBtlbR/bAPbXISLuGcc3NDe/YkzE4ZioM59YH0LMPPeva0Z1kj1xTfElgVgGO5BbBg2C8gswYu8RMo0FBoin1IGhM5pYTjuAl6NIrA4w0MMUYfm9PuKGneNUYL0IBLDxEdvcY+DwrUdYwa/wp4EZR3Ng01EqJysgKSAPn0cQ2yT3r2hFHKi1aYTYBI7kpeiutmz4xWC2MQ8X0usKN0PMvYyFudGXH04/rKQqdfVJQHRFL1wBfgYXgAphbNVyUTYv2xbKYhOcN3QHAkeuMYGA6Bi/n2lCj4vMbGQheOBCbHzbxPCIoTfJuHNQTCwEtZgh/gApJC/hylaeSDr3oVd3mHM7qs1JBBdSrK5wLGIVuDFDWnQMUC2UpMKUdLmZRPXdV9rEWvpA3WBLdLECmjLarQtRaioeR4gTSWJo0mSUxxxPP7OuOeM92QgzIe0AWNEFkX5JXEbnXR3mz3nQbZwzayda4r8M25djSuL0Gw9L8lQWXmPyYJBn9PEjgvZXsjCVrJbkmCzrgealzPMBG1Ny+uXD68ZVAmyQdYajLI58aibh4TvIgEdEs5p6GgTyTSsSzEUowYxNDYR0mQk28Ww6rqu7KXYX6jlJDtn7J9NHJeYf+EpsyD3R7jiC1gm8DKeuA3Tgy6vAwI4iKVm2eRFrXUpefSwLUtrGHTF5Y9aIYoFqqu2tC8WsbhNhi1HAnuU0jkEpUhDpD+llBv2aWK1p4qDt5HRdNtbqVa2hYPpKmo26G/I1DHdnBbdwWBnOGEd2mF1q1AtafZemelOv/sFmAOOtN8+Avfo/7sLl3ZprlMh1dPLGt5K/gmymUmL4vULvAaA+xfBswDy8CGQC2meHPN7L5zZKx9zGbWDkZHo/XPXnru3lW2h31mj3krm/Q1m4zz1LaMSQDeMknDT3vsaQ+3G3tsD9uxPfTKIV+q3u8U8TEqNwSw7QOr/qaSWqCuK4ChifcdhWLXn8w5Ln6pL386zvBD3wM+QPtNycxhM/kG+x3jdU85L4vbcVab+uFgPhWVYDafOp9meNYMo47MsCPuwWYQ3fqv6WJ6/f++ffIb -------------------------------------------------------------------------------- /docs/reports/v0.1.0实现.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaiakz/rsync-os/64e84daeabb1fa4d2c7cf766c196306adfba6cb2/docs/reports/v0.1.0实现.pdf -------------------------------------------------------------------------------- /docs/rsync_subprotocol.md: -------------------------------------------------------------------------------- 1 | # Rsync subprotocol 2 | A subset of rsync protocol without block checksum, designed for the compatibility of any version of standard rsync receiver/sender (client/server). 3 | 4 | ## Introdution 5 | In the rsync algorithm, rsync requires random access of files to do the block exchange. It will cause a high CPU and disk usage. `rsync subprotocol` simplifies the block exchange to avoid that. It can **work with sequential files**. 6 | 7 | ### **NOTICE** Although `rsync subprotocol` don't need to do rolling checksum, a whole-file MD4/MD5 checksum is still required to validate the transfer. We can compute MD5 checksums during receiving/sending files. 8 | 9 | ## Receiver Side 10 | In the `rsync algorithm`, after the receiver reads the file list, it interates through each file in the list and handles three situations when processes each file: 11 | 1. same as remote file: skip 12 | 2. exists but was modified (has different size or last modification time): the receiver examines each file in blocks of a fixed size. Each block will be hashed twice (Adler-32 and MD4/MD5), and sent to the sender. 13 | 3. does not exist or empty: can't hash blocks of file, so just send a zero block to require the entire file. 14 | 15 | But in `rsync subprotocol`, receiver do not support block checksum, so it won't send any hashes to the sender, but zero blocks instead. For a modified file, receiver always pretends `the file does not exist` to get the entire file from sender. **Just like you specify the argument `--whole-file` in standard rsync.** 16 | 17 | 18 | ## Sender Side 19 | In the `rsync algorithm`, when receiver find a modfied file, it will send the file's block hashes to sender. Once accepted, the sender will also calculate the hashes of each file's blocks: 20 | 1. If a block is matched, just send the index of the block to receiver so receiver will copy relative data from the file locally. 21 | 2. If a block is not matched, it means the block from receiver was modified. The block of the file will be sent as a stream of bytes, and receiver will copy it to file directly. 22 | 23 | In `rsync subprotocol`, sender will ignore any block that comes from receiver so there are no matched blocks. For each file required by receiver, sender always sends the entire file. -------------------------------------------------------------------------------- /docs/v0.2.x_Zh-CN.md: -------------------------------------------------------------------------------- 1 | # v0.2.x 2 | 在开始之前首先明确，主动建立连接的一方是client，被动连接的一方是server。sender负责发送文件，receiver负责接收文件，rsync算法之中sender和receiver是配对的，而client和server会分别成为二者之一。 3 | 4 | # 实现协议v27,参考的rsync版本v2.6.0 5 | 6 | ## 目标 7 | v0.2.x的目标是实现一套高可定制的rsync API，用户基于这套API可以自由创建以及定制rsync的sender以及receiver。在API约定下，支持包括连接方式（Socket，SSH等），存储后端（Posix，S3等）以及回调函数等定制。 8 | 9 | # 调用流程： 10 | 1. 用户首先需要初始化好连接Conn以及存储FS，提供Attribs（可以看作rsync的命令行参数），Receiver支持设置回调函数（下载或过滤特定文件）， 11 | 2. 完成握手 12 | 3. 创建Receiver或者Sender，最终调用其Sync方法即可开始rsync协议的自动同步。 13 | 14 | ## 详解： 15 | ```go 16 | type Conn struct { 17 | writer io.WriteCloser // Write only 18 | reader io.ReadCloser // Read only 19 | bytespool []byte // Anti memory-wasted, default size: 8 bytes 20 | } 21 | ``` 22 | 23 | 表示与远程的连接，可以是Socket的TCP连接，SSH连接，标准输出输入等（只要远程支持，理论上还可以rsync over UDP）。Conn的内部读写通道是分开的，这是为了应对rsync在Socket连接时读写不对称问题，比如握手完成后server向client发送的数据是multiple的，而client向server发送却不是。 24 | 25 | Conn提供了rsync协议数据类型的读写支持：byte，short，int，varint变长整数（int64）。 26 | 27 | ```go 28 | type MuxReader struct { 29 | in io.ReadCloser 30 | remain uint32 // Default value: 0 31 | header []byte // Size: 4 bytes 32 | } 33 | ``` 34 | 35 | 36 | MuxReader负责处理multiple消息，剥离消息头，并返回真实的数据。可以通过调用NewMuxReader(reader io.ReadCloser) *MuxReader 完成MuxReader的初始化。 37 | 38 | ```go 39 | type Attribs struct { 40 | Sender bool // --sender 41 | Server bool // --server 42 | Recursive bool // -r 43 | DryRun bool // -n 44 | HasModTime bool // -t 45 | HasPerms bool // -p 46 | HasLinks bool // -l 47 | HasGID bool // -g 48 | HasUID bool // -u 49 | compress bool // -z 未支持 50 | } 51 | ``` 52 | 53 | 类似与rsync的命令行参数，可以控制Receiver/Sender的内部执行流程。 54 | 55 | ```go 56 | type FS interface { 57 | Put(fileName string, content io.Reader, fileSize int64, metadata FileMetadata) (written int64, err error) 58 | Get(fileName string, metadata FileMetadata) (File, error) 59 | Delete(fileName string, mode FileMode) error 60 | List() (FileList, error) 61 | Stats() (seekable bool) 62 | } 63 | ``` 64 | FS是一套存储后端的类型无关的接口，无论是传统的文件存储，或者是对象存储，只要实现这一套接口均可以在rsync同步过程之中作为存储后端使用，不同的系统的差别主要在Seekable，若设置返回false则Sender/Receiver会运行精简后的rsync传输协议（不进行block checksum），否则运行传统的rsync传输协议。 65 | 66 | 67 | **未完成** Callback 这部分接口由用户实现，主要是Rsync的Request以及Delete两个过程回调（Receiver）。 68 | 1. Request：下载文件，回调函数会传入远程的文件列表，用户经过筛选后返回一个int数组（对应文件列表的index）来指示下载的文件。 69 | 2. Delete：删除文件。回调函数会传入即将删除的文件列表，用户决定删除的文件。 70 | 71 | 函数默认会提供这两个过程的函数，对应原版rsync的对应功能，用户调用即可。 72 | 73 | # 工作进度&笔记 74 | ## Sender 75 | 正在实现v27版本的Sender部分，预计实现后可以与rsync的Sender进行文件同步，但不会进行block checksum： 76 | 1. 忽略所有Receiver发送的checksum 77 | 2. 只发送实际的文件内容 78 | 79 | ## 压缩 （对应原版rsync的token.c） 80 | 原版Rsync内部使用了zlib对数据进行压缩，但它zlib相关window bit设置成了-15也就是仅对数据进行deflate或者inflate处例，不会添加额外的包头。 81 | 计划使用go自带的flate包处理这些数据。 82 | 83 | **难题** 原版Rsync使用状态机处理数据，尤其发送压缩的数据时： 84 | 不会发送压缩数据的后4bytes给对方 85 | ```c 86 | /* 87 | * We have to trim off the last 4 88 | * bytes of output when flushing 89 | * (they are just 0, 0, ff, ff). 90 | */ 91 | ``` 92 | 暂时想不出有什么比较好的解决方法。 93 | 94 | 95 | ## Exclusion 96 | 原版rsync v2.6.0是把每个路径以字符串数组存起来的（最新版有重写），发送的时候会去掉文件夹路径结尾的`/` 97 | 98 | # 参考项目 99 | [repositoryd](https://github.com/APNIC-net/repositoryd), An rsync-compatible RPKI repository daemon. -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module rsync-os 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/golang/protobuf v1.4.2 7 | github.com/kaiakz/ubuffer v0.0.0-20200803053910-dd1083087166 8 | github.com/minio/minio-go/v6 v6.0.57 9 | github.com/pkg/errors v0.9.1 10 | github.com/spf13/viper v1.7.1 11 | go.etcd.io/bbolt v1.3.5 12 | golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de 13 | google.golang.org/protobuf v1.23.0 14 | ) 15 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 4 | cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= 5 | cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= 6 | cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= 7 | cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= 8 | cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= 9 | cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= 10 | cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= 11 | cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= 12 | cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= 13 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 14 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 15 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 16 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 17 | github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= 18 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 19 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 20 | github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= 21 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= 22 | github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 23 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 24 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 25 | github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= 26 | github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= 27 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 28 | github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= 29 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 30 | github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= 31 | github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 32 | github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 33 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 34 | github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 35 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 36 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 37 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 38 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 39 | github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= 40 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 41 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 42 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 43 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 44 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 45 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 46 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 47 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 48 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 49 | github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= 50 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 51 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 52 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 53 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 54 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 55 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 56 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 57 | github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 58 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 59 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 60 | github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= 61 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 62 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 63 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 64 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 65 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 66 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 67 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 68 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 69 | github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0= 70 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 71 | github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= 72 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 73 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 74 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 75 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 76 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 77 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 78 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 79 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 80 | github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= 81 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 82 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 83 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 84 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 85 | github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 86 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 87 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 88 | github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= 89 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= 90 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 91 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 92 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= 93 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 94 | github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= 95 | github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= 96 | github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= 97 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 98 | github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 99 | github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 100 | github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= 101 | github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= 102 | github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= 103 | github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= 104 | github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= 105 | github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 106 | github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 107 | github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= 108 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 109 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 110 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 111 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 112 | github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= 113 | github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= 114 | github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= 115 | github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= 116 | github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 117 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 118 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= 119 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 120 | github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= 121 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 122 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 123 | github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= 124 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 125 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 126 | github.com/kaiakz/ubuffer v0.0.0-20200803053910-dd1083087166 h1:IAukUBAVLUWBcexOYgkTD/EjMkfnNos7g7LFpyIdHJI= 127 | github.com/kaiakz/ubuffer v0.0.0-20200803053910-dd1083087166/go.mod h1:T4xUEny5PVedYIbkMAKYEBjMyDsOvvP0qK4s324AKA8= 128 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 129 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 130 | github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= 131 | github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 132 | github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= 133 | github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= 134 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 135 | github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= 136 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 137 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 138 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 139 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 140 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 141 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 142 | github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= 143 | github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 144 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 145 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 146 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 147 | github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= 148 | github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= 149 | github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= 150 | github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/9Tw= 151 | github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM= 152 | github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= 153 | github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= 154 | github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= 155 | github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 156 | github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= 157 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 158 | github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= 159 | github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= 160 | github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= 161 | github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 162 | github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= 163 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 164 | github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8= 165 | github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 166 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 167 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 168 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 169 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 170 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= 171 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 172 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 173 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 174 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 175 | github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= 176 | github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 177 | github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= 178 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 179 | github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw= 180 | github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= 181 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 182 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 183 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 184 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 185 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 186 | github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= 187 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 188 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 189 | github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= 190 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 191 | github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= 192 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 193 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 194 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 195 | github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 196 | github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 197 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 198 | github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 199 | github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= 200 | github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= 201 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 202 | github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 203 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= 204 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 205 | github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= 206 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= 207 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 208 | github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 209 | github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= 210 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 211 | github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= 212 | github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 213 | github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= 214 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 215 | github.com/spf13/afero v1.3.3 h1:p5gZEKLYoL7wh8VrJesMaYeNxdEd1v3cb4irOk9zB54= 216 | github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= 217 | github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= 218 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 219 | github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= 220 | github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 221 | github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= 222 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 223 | github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= 224 | github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= 225 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 226 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 227 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 228 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 229 | github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= 230 | github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= 231 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 232 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 233 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 234 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 235 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 236 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 237 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 238 | github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= 239 | github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= 240 | github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 241 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 242 | go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= 243 | go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= 244 | go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= 245 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 246 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 247 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 248 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 249 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 250 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 251 | golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 252 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 253 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 254 | golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 255 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 256 | golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 257 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 258 | golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig= 259 | golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 260 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 261 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 262 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 263 | golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= 264 | golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= 265 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 266 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 267 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 268 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 269 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 270 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 271 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 272 | golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 273 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 274 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 275 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 276 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 277 | golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= 278 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 279 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 280 | golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 281 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 282 | golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 283 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 284 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 285 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 286 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 287 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 288 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 289 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 290 | golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= 291 | golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 292 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 293 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= 294 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 295 | golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= 296 | golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 297 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 298 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 299 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 300 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 301 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 302 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 303 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 304 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 305 | golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 306 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 307 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 308 | golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 309 | golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 310 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 311 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 312 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 313 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 314 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= 315 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 316 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 317 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 318 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 319 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 320 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 321 | golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= 322 | golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 323 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 324 | golang.org/x/sys v0.0.0-20200802091954-4b90ce9b60b3 h1:qDJKu1y/1SjhWac4BQZjLljqvqiWUhjmDMnonmVGDAU= 325 | golang.org/x/sys v0.0.0-20200802091954-4b90ce9b60b3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 326 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 327 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 328 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 329 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 330 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 331 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= 332 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 333 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 334 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 335 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 336 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 337 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 338 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 339 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 340 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 341 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 342 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 343 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 344 | golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 345 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 346 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 347 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 348 | golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 349 | golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 350 | golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 351 | golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 352 | golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 353 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 354 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 355 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 356 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 357 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= 358 | google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 359 | google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 360 | google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 361 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 362 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 363 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 364 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 365 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 366 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 367 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 368 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 369 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 370 | google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 371 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 372 | google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= 373 | google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 374 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 375 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 376 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 377 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 378 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 379 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 380 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 381 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 382 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 383 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 384 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 385 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 386 | google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= 387 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 388 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 389 | google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= 390 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 391 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 392 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 393 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 394 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 395 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 396 | gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= 397 | gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 398 | gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= 399 | gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 400 | gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= 401 | gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 402 | gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= 403 | gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 404 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 405 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 406 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 407 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 408 | gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= 409 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 410 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 411 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 412 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 413 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 414 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 415 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 416 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Send @RSYNCD x.x\n 2 | // Send modname\n 3 | // Send arugment with mod list\0 filter list write(0) \n 4 | // handshake 5 | // batch seed 6 | // Recv file list 7 | // 8 | 9 | package main 10 | 11 | import ( 12 | "flag" 13 | "fmt" 14 | "github.com/spf13/viper" 15 | "log" 16 | "rsync-os/rsync" 17 | "rsync-os/storage" 18 | "time" 19 | ) 20 | 21 | func ClientS3(src string, dest string) { 22 | addr, module, path, err := rsync.SplitURI(src) 23 | 24 | if err != nil { 25 | log.Println("Invaild URI") 26 | return 27 | } 28 | 29 | log.Println(module, path) 30 | 31 | ppath := rsync.TrimPrepath(path) 32 | 33 | if viper.GetStringMapString(dest) == nil { 34 | log.Println("Lack of ", dest) 35 | return 36 | } 37 | 38 | // Config 39 | dbconf := viper.GetStringMapString(dest + ".boltdb") 40 | minioConf := viper.GetStringMapString(dest) 41 | 42 | 43 | stor, _ := storage.NewMinio(module, ppath, dbconf["path"], minioConf["endpoint"], minioConf["keyaccess"], minioConf["keysecret"], false) 44 | defer stor.Close() 45 | 46 | client, err := rsync.SocketClient(stor, addr, module, ppath, nil) 47 | if err != nil { 48 | panic("rsync client fails to initialize") 49 | } 50 | if err := client.Sync(); err != nil { 51 | panic(err) 52 | } 53 | 54 | } 55 | 56 | func main() { 57 | loadConfigIfExists() 58 | flag.Parse() 59 | args := flag.Args() 60 | if len(args) < 2 { 61 | fmt.Println("Usage: rsync-os [OPTION]... rsync://[USER@]HOST[:PORT]/SRC") 62 | return 63 | } 64 | startTime := time.Now() 65 | ClientS3(args[0], args[1]) 66 | log.Println("Duration:", time.Since(startTime)) 67 | } 68 | -------------------------------------------------------------------------------- /rsync/attribs.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | type Attribs struct { 4 | Sender bool // --sender 5 | Server bool // --server 6 | Recursive bool // -r 7 | DryRun bool // -n 8 | HasModTime bool // -t 9 | HasPerms bool // -p 10 | HasLinks bool // -l 11 | HasGID bool // -g 12 | HasUID bool // -u 13 | //compress bool // -z 14 | } 15 | 16 | func (a *Attribs) Marshal() []byte { 17 | //"--server\n--sender\n-l\n-p\n-r\n-t\n.\n" 18 | args := make([]byte, 0, 64) 19 | if a.Server { 20 | args = append(args, []byte("--server\n")...) 21 | } 22 | if a.Sender { 23 | args = append(args, []byte("--sender\n")...) 24 | } 25 | if a.Recursive { 26 | args = append(args, []byte("-r\n")...) 27 | } 28 | if a.HasModTime { 29 | args = append(args, []byte("-t\n")...) 30 | } 31 | if a.HasLinks { 32 | args = append(args, []byte("-l\n")...) 33 | } 34 | if a.HasPerms { 35 | args = append(args, []byte("-p\n")...) 36 | } 37 | if a.HasGID { 38 | args = append(args, []byte("-g\n")...) 39 | } 40 | if a.HasUID { 41 | args = append(args, []byte("-u\n")...) 42 | } 43 | return args 44 | } 45 | -------------------------------------------------------------------------------- /rsync/chksum.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | type SumStruct struct { 4 | fileLen uint64 // totol file length 5 | count uint64 // how many chunks 6 | remainder uint64 // fileLen % blockLen 7 | blockLen uint64 // block length 8 | sum2Len uint64 // sum2 length 9 | sumList []SumChunk // chunks 10 | } 11 | 12 | type SumChunk struct { 13 | fileOffset int64 14 | chunkLen uint 15 | sum1 uint32 16 | sum2 []byte 17 | } 18 | 19 | 20 | -------------------------------------------------------------------------------- /rsync/client.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "log" 7 | "net" 8 | "strings" 9 | ) 10 | 11 | /* As a Client, we need to: 12 | 1. connect to server by socket or ssh 13 | 2. handshake: version, args, ioerror 14 | PS: client always sends exclusions/filter list 15 | 3. construct a Receiver or a Sender, then excute it. 16 | */ 17 | 18 | // TODO: passes more arguments: cmd 19 | // Connect to rsync daemon 20 | func SocketClient(storage FS, address string, module string, path string, options map[string]string) (SendReceiver, error) { 21 | skt, err := net.Dial("tcp", address) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | conn := &Conn{ 27 | writer: skt, 28 | reader: skt, 29 | bytespool: make([]byte, 8), 30 | } 31 | 32 | /* HandShake by socket */ 33 | // send my version 34 | _, err = conn.Write([]byte(RSYNC_VERSION)) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | // receive server's protocol version and seed 40 | versionStr, _ := readLine(conn) 41 | 42 | // recv(version) 43 | var remoteProtocol, remoteProtocolSub int 44 | _, err = fmt.Sscanf(versionStr, "@RSYNCD: %d.%d", remoteProtocol, remoteProtocolSub) 45 | if err != nil { 46 | // FIXME: (panic)type not a pointer: int 47 | //panic(err) 48 | } 49 | log.Println(versionStr) 50 | 51 | buf := new(bytes.Buffer) 52 | 53 | // send mod name 54 | buf.WriteString(module) 55 | buf.WriteByte('\n') 56 | _, err = conn.Write(buf.Bytes()) 57 | if err != nil { 58 | return nil, err 59 | } 60 | buf.Reset() 61 | 62 | // Wait for '@RSYNCD: OK' 63 | for { 64 | res, err := readLine(conn) 65 | if err != nil { 66 | return nil, err 67 | } 68 | log.Print(res) 69 | if strings.Contains(res, RSYNCD_OK) { 70 | break 71 | } 72 | } 73 | 74 | // Send arguments 75 | buf.Write([]byte(SAMPLE_ARGS)) 76 | buf.Write([]byte(module)) 77 | buf.Write([]byte(path)) 78 | buf.Write([]byte("\n\n")) 79 | _, err = conn.Write(buf.Bytes()) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | // read int32 as seed 85 | seed, err := conn.ReadInt() 86 | if err != nil { 87 | return nil, err 88 | } 89 | log.Println("SEED", seed) 90 | 91 | // HandShake OK 92 | log.Println("Handshake completed") 93 | 94 | // Begin to demux 95 | conn.reader = NewMuxReader(conn.reader) 96 | 97 | // As a client, we need to send filter list 98 | err = conn.WriteInt(EXCLUSION_END) 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | // TODO: Sender 104 | 105 | return &Receiver{ 106 | conn: conn, 107 | module: module, 108 | path: path, 109 | seed: seed, 110 | storage: storage, 111 | }, nil 112 | } 113 | 114 | // Connect to sshd, and start a rsync server on remote 115 | func SshClient(storage FS, address string, module string, path string, options map[string]string) (SendReceiver, error) { 116 | // TODO: build args 117 | 118 | ssh, err := NewSSH(address, "", "", "rsync --server --sender -l -p -r -t") 119 | if err != nil { 120 | return nil, err 121 | } 122 | conn := &Conn{ 123 | writer: ssh, 124 | reader: ssh, 125 | bytespool: make([]byte, 8), 126 | } 127 | 128 | // Handshake 129 | lver, err := conn.ReadInt() 130 | if err != nil { 131 | return nil, err 132 | } 133 | 134 | rver, err := conn.ReadInt() 135 | if err != nil { 136 | return nil, err 137 | } 138 | 139 | seed, err := conn.ReadInt() 140 | if err != nil { 141 | return nil, err 142 | } 143 | 144 | // TODO: Sender 145 | 146 | return &Receiver{ 147 | conn: conn, 148 | module: module, 149 | path: path, 150 | seed: seed, 151 | lVer: lver, 152 | rVer: rver, 153 | storage: storage, 154 | }, nil 155 | } 156 | -------------------------------------------------------------------------------- /rsync/compress.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | import ( 4 | "compress/flate" 5 | "io" 6 | ) 7 | 8 | /* 9 | rsync uses zlib to do compression, the windowsBits is -15: raw data 10 | */ 11 | 12 | const ( 13 | END_FLAG = 0 14 | TOKEN_LONG = 0x20 15 | TOKENRUN_LONG = 0x21 16 | DEFLATED_DATA = 0x40 17 | TOKEN_REL = 0x80 18 | TOKENRUN_REL = 0xc0 19 | 20 | ) 21 | 22 | // RFC 1951: https://tools.ietf.org/html/rfc1951 23 | type flatedtokenReader struct { 24 | in Conn 25 | flatedwraper *flatedWraper 26 | decompressor io.ReadCloser 27 | savedflag byte 28 | flag byte 29 | remains uint32 30 | } 31 | 32 | 33 | 34 | func NewflatedtokenReader(reader Conn) *flatedtokenReader { 35 | w := &flatedWraper{ 36 | raw: &reader, 37 | end: [4]byte{0, 0, 0xff, 0xff}, 38 | } 39 | return &flatedtokenReader{ 40 | in: reader, 41 | flatedwraper: w, 42 | decompressor: flate.NewReader(w), 43 | savedflag: -1, 44 | flag: 0, 45 | remains: 0, 46 | } 47 | } 48 | 49 | // Update flag & len of remain data 50 | func (f *flatedtokenReader) readFlag() error { 51 | if f.savedflag != 0 { 52 | f.flag = f.savedflag & 0xff 53 | f.savedflag = 0 54 | } else { 55 | var err error 56 | if f.flag, err = f.in.ReadByte(); err != nil { 57 | return err 58 | } 59 | } 60 | if (f.flag & 0xc0) == DEFLATED_DATA { 61 | l, err := f.in.ReadByte() 62 | if err != nil { 63 | return err 64 | } 65 | f.remains = uint32(f.flag & 0x3f) << 8 + uint32(l) 66 | } 67 | return nil 68 | } 69 | 70 | func (f *flatedtokenReader) Read(p []byte) (n int, err error) { 71 | n, err = f.decompressor.Read(p) 72 | f.remains -= uint32(n) 73 | return 74 | } 75 | 76 | func (f *flatedtokenReader) Close() error { 77 | return f.decompressor.Close() 78 | } 79 | 80 | // Hack only: rsync need to append 4 bytes(0, 0, ff, ff) at the end. 81 | type flatedWraper struct { 82 | raw io.Reader 83 | end [4]byte 84 | } 85 | 86 | func (f *flatedWraper) Read(p []byte) (n int, err error) { 87 | // Just append 4 bytes to the end of stream 88 | return f.raw.Read(p) 89 | } -------------------------------------------------------------------------------- /rsync/connection.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "io" 7 | ) 8 | 9 | type SendReceiver interface { 10 | Sync() error 11 | } 12 | 13 | // io.ReadWriteCloser 14 | // This struct has two main attributes, both of them can be used for a plain socket or an SSH 15 | type Conn struct { 16 | writer io.WriteCloser // Write only 17 | reader io.ReadCloser // Read only 18 | bytespool []byte // Anti memory-wasted, default size: 8 bytes 19 | } 20 | 21 | func (conn *Conn) Write(p []byte) (n int, err error) { 22 | return conn.writer.Write(p) 23 | } 24 | 25 | func (conn *Conn) Read(p []byte) (n int, err error) { 26 | return conn.reader.Read(p) 27 | } 28 | 29 | /* Encoding: little endian */ 30 | // size of: int: 4, long: 8, varint: 4 or 8 31 | func (conn *Conn) ReadByte() (byte, error) { 32 | val := conn.bytespool[:1] 33 | _, err := io.ReadFull(conn, val) 34 | if err != nil { 35 | return 0, err 36 | } 37 | return conn.bytespool[0], nil 38 | } 39 | 40 | func (conn *Conn) ReadShort() (int16, error) { 41 | val := conn.bytespool[:2] 42 | _, err := io.ReadFull(conn, val) 43 | if err != nil { 44 | return 0, err 45 | } 46 | return int16(binary.LittleEndian.Uint16(val)), nil 47 | } 48 | 49 | func (conn *Conn) ReadInt() (int32, error) { 50 | val := conn.bytespool[:4] 51 | _, err := io.ReadFull(conn, val) 52 | if err != nil { 53 | return 0, err 54 | } 55 | return int32(binary.LittleEndian.Uint32(val)), nil 56 | } 57 | 58 | func (conn *Conn) ReadLong() (int64, error) { 59 | val := conn.bytespool[:8] 60 | _, err := io.ReadFull(conn, val) 61 | if err != nil { 62 | return 0, err 63 | } 64 | return int64(binary.LittleEndian.Uint64(val)), nil 65 | } 66 | 67 | func (conn *Conn) ReadVarint() (int64, error) { 68 | sval, err := conn.ReadInt(); 69 | if err != nil { 70 | return 0, err 71 | } 72 | if sval != -1 { 73 | return int64(sval), nil 74 | } 75 | return conn.ReadLong() 76 | } 77 | 78 | func (conn *Conn) WriteByte(data byte) error { 79 | return binary.Write(conn.writer, binary.LittleEndian, data) 80 | } 81 | 82 | func (conn *Conn) WriteShort(data int16) error { 83 | return binary.Write(conn.writer, binary.LittleEndian, data) 84 | } 85 | 86 | func (conn *Conn) WriteInt(data int32) error { 87 | return binary.Write(conn.writer, binary.LittleEndian, data) 88 | } 89 | 90 | func (conn *Conn) WriteLong(data int64) error { 91 | return binary.Write(conn.writer, binary.LittleEndian, data) 92 | } 93 | 94 | // TODO: If both writer and reader are based on a same Connection (socket, SSH), how to close them twice? 95 | func (conn *Conn) Close() error { 96 | _ = conn.writer.Close() 97 | _ = conn.reader.Close() 98 | return nil 99 | } 100 | 101 | func readLine(conn *Conn) (string, error) { 102 | // until \n, then add \0 103 | line := new(bytes.Buffer) 104 | for { 105 | c, err := conn.ReadByte() 106 | if err != nil { 107 | return "", err 108 | } 109 | 110 | if c == '\r' { 111 | continue 112 | } 113 | 114 | err = line.WriteByte(c) 115 | if err != nil { 116 | return "", err 117 | } 118 | 119 | if c == '\n' { 120 | line.WriteByte(0) 121 | break 122 | } 123 | 124 | if c == 0 { 125 | break 126 | } 127 | } 128 | return line.String(), nil 129 | } -------------------------------------------------------------------------------- /rsync/const.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | const ( 4 | RSYNC_VERSION = "@RSYNCD: 27.0\n" 5 | RSYNCD_OK = "@RSYNCD: OK" 6 | RSYNC_EXIT = "@RSYNCD: EXIT" 7 | 8 | K = 1 << 10 9 | M = 1 << 20 10 | G = 1 << 30 11 | 12 | MAXPATHLEN = 1024 13 | INDEX_END = int32(-1) 14 | EXCLUSION_END = int32(0) 15 | END1 = '\n' 16 | END2 = '\x00' 17 | PHASE_END = int32(-1) 18 | 19 | // ARGUMENTS 20 | ARG_SERVER = "--server" 21 | ARG_SENDER = "--sender" 22 | ARG_SYMLINK = "-l" 23 | ARG_RECURSIVE = "-r" 24 | ARG_PERMS = "-p" 25 | SAMPLE_ARGS = "--server\n--sender\n-l\n-p\n-r\n-t\n.\n" 26 | SAMPLE_LIST_ARGS = "--server\n--sender\n--list-only\n-l\n-p\n-r\n-t\n.\n" 27 | 28 | // For Multiplex(1 byte) 29 | MUX_BASE = 7 30 | MSG_DATA = 0 31 | MSG_ERROR_XFER = 1 32 | MSG_INFO = 2 33 | MSG_ERROR = 3 34 | MSG_WARNING = 4 35 | MSG_IO_ERROR = 22 36 | MSG_NOOP = 42 37 | MSG_SUCCESS = 100 38 | MSG_DELETED = 101 39 | MSG_NO_SEND = 102 40 | 41 | // For FILE LIST(1 byte) 42 | FLIST_END = 0x00 43 | FLIST_TOP_LEVEL = 0x01 /* needed for remote --delete */ 44 | FLIST_MODE_SAME = 0x02 /* mode is repeat */ 45 | FLIST_RDEV_SAME = 0x04 /* rdev is repeat */ 46 | FLIST_UID_SAME = 0x08 /* uid is repeat */ 47 | FLIST_GID_SAME = 0x10 /* gid is repeat */ 48 | FLIST_NAME_SAME = 0x20 /* name is repeat */ 49 | FLIST_NAME_LONG = 0x40 /* name >255 bytes */ 50 | FLIST_TIME_SAME = 0x80 /* time is repeat */ 51 | 52 | // File type 53 | S_IFMT = 0170000 /* Type of file */ 54 | S_IFREG = 0100000 /* Regular file. */ 55 | S_IFDIR = 0040000 /* Directory. */ 56 | S_IFLNK = 0120000 /* Symbolic link. */ 57 | S_IFCHR = 0020000 /* Character device. */ 58 | S_IFBLK = 0060000 /* Block device. */ 59 | S_IFIFO = 0010000 /* FIFO. */ 60 | S_IFSOCK = 0140000 /* Socket. */ 61 | ) 62 | -------------------------------------------------------------------------------- /rsync/demux.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "io" 7 | "log" 8 | ) 9 | 10 | //Multiplexing 11 | //Most rsync transmissions are wrapped in a multiplexing envelope protocol. It is 12 | //composed as follows: 13 | // 14 | //1. envelope header (4 bytes) 15 | //2. envelope payload (arbitrary length) 16 | // 17 | //The first byte of the envelope header consists of a tag. If the tag is 7, the pay‐ 18 | //load is normal data. Otherwise, the payload is out-of-band server messages. If the 19 | //tag is 1, it is an error on the sender's part and must trigger an exit. This limits 20 | //message payloads to 24 bit integer size, 0x00ffffff. 21 | // 22 | //The only data not using this envelope are the initial handshake between client and 23 | //server 24 | 25 | type MuxReader struct { 26 | in io.ReadCloser 27 | remain uint32 // Default value: 0 28 | header []byte // Size: 4 bytes 29 | } 30 | 31 | func NewMuxReader(reader io.ReadCloser) *MuxReader { 32 | return &MuxReader{ 33 | in: reader, 34 | remain: 0, 35 | header: make([]byte, 4), 36 | } 37 | } 38 | 39 | func (r *MuxReader) Read(p []byte) (n int, err error) { 40 | if r.remain == 0 { 41 | err := r.readHeader() 42 | if err != nil { 43 | return 0, err 44 | } 45 | } 46 | rlen := uint32(len(p)) 47 | if rlen > r.remain { // Min(len(p), remain) 48 | rlen = r.remain 49 | } 50 | n, err = r.in.Read(p[:rlen]) 51 | r.remain = r.remain - uint32(n) 52 | return 53 | } 54 | 55 | func (r *MuxReader) readHeader() error { 56 | for { 57 | // Read header 58 | if _, err := io.ReadFull(r.in, r.header); err != nil { 59 | return err 60 | } 61 | tag := r.header[3] // Little Endian 62 | size := (binary.LittleEndian.Uint32(r.header) & 0xffffff) // TODO: zero? 63 | 64 | log.Printf(" tag %d size %d\n", tag, size) 65 | 66 | if tag == (MUX_BASE + MSG_DATA) { // MUX_BASE + MSG_DATA 67 | r.remain = size 68 | return nil 69 | } else { // out-of-band data 70 | // otag := tag - 7 71 | msg := make([]byte, size) 72 | if _, err := io.ReadFull(r.in, msg); err != nil { 73 | return err 74 | } 75 | return errors.New(string(msg)) 76 | } 77 | } 78 | } 79 | 80 | func (r *MuxReader) Close() error { 81 | return r.in.Close() 82 | } 83 | 84 | -------------------------------------------------------------------------------- /rsync/demux_old.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | import ( 4 | "encoding/binary" 5 | "io" 6 | "log" 7 | ) 8 | 9 | //Multiplexing 10 | //Most rsync transmissions are wrapped in a multiplexing envelope protocol. It is 11 | //composed as follows: 12 | // 13 | //1. envelope header (4 bytes) 14 | //2. envelope payload (arbitrary length) 15 | // 16 | //The first byte of the envelope header consists of a tag. If the tag is 7, the pay‐ 17 | //load is normal data. Otherwise, the payload is out-of-band server messages. If the 18 | //tag is 1, it is an error on the sender's part and must trigger an exit. This limits 19 | //message payloads to 24 bit integer size, 0x00ffffff. 20 | // 21 | //The only data not using this envelope are the initial handshake between client and 22 | //server 23 | 24 | type MuxReaderV0 struct { 25 | in io.ReadCloser 26 | Data chan byte 27 | closeCh chan byte 28 | } 29 | 30 | func NewMuxReaderV0(reader io.ReadCloser) *MuxReaderV0 { 31 | mr := &MuxReaderV0{ 32 | in: reader, 33 | Data: make(chan byte, 16 * M), 34 | closeCh: make(chan byte), 35 | } 36 | // Demux in Goroutine 37 | go func() { 38 | 39 | header := make([]byte, 4) // Header size: 4 bytes 40 | var dsize uint32 = 1 << 16 // Default size: 65536 41 | bytespool := make([]byte, dsize) 42 | 43 | for { 44 | select { 45 | case <-mr.closeCh: // Close the channel, then exit the goroutine 46 | close(mr.Data) 47 | return 48 | default: 49 | // read the multipex data & put them to channel 50 | _, err := io.ReadFull(reader, header) 51 | if err != nil { 52 | panic("Multiplex: wire protocol error") 53 | } 54 | 55 | tag := header[3] // Little Endian 56 | size := (binary.LittleEndian.Uint32(header) & 0xffffff) // TODO: zero? 57 | 58 | log.Printf(" tag %d size %d\n", tag, size) 59 | 60 | if tag == (MUX_BASE + MSG_DATA) { // MUX_BASE + MSG_DATA 61 | if size > dsize { 62 | bytespool = make([]byte, size) 63 | dsize = size 64 | } 65 | 66 | body := bytespool[:size] 67 | _, err := io.ReadFull(reader, body) 68 | 69 | // FIXME: Never return EOF 70 | if err != nil { // The connection was closed by server 71 | panic(err) 72 | } 73 | 74 | for _, b := range body { 75 | mr.Data <- b 76 | } 77 | 78 | } else { // out-of-band data 79 | //otag := tag - 7 80 | msg := make([]byte, size) 81 | if _, err := io.ReadFull(reader, msg); err != nil { 82 | panic("Failed to read out-of-band data") 83 | } 84 | panic("out-of-band data: " + string(msg)) 85 | } 86 | } 87 | } 88 | }() 89 | return mr 90 | } 91 | 92 | // FIXME: Never return error 93 | func (r *MuxReaderV0) Read(p []byte) (n int, err error) { 94 | for i, _ := range p { 95 | p[i] = <- r.Data 96 | } 97 | return len(p), nil 98 | } 99 | 100 | func (r *MuxReaderV0) Close() error { 101 | r.closeCh <- 0 // close the channel Data & exit the demux goroutine 102 | return r.in.Close() 103 | } 104 | -------------------------------------------------------------------------------- /rsync/exclude.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | import ( 4 | "path/filepath" 5 | "strings" 6 | ) 7 | 8 | // Reference: rsync 2.6.0 9 | // --exclude & --exclude-from 10 | 11 | /* These default ignored items come from the CVS manual. 12 | * "RCS SCCS CVS CVS.adm RCSLOG cvslog.* tags TAGS" 13 | * " .make.state .nse_depinfo *~ #* .#* ,* _$* *$" 14 | * " *.old *.bak *.BAK *.orig *.rej .del-*" 15 | * " *.a *.olb *.o *.obj *.so *.exe" 16 | * " *.Z *.elc *.ln core" 17 | The rest we added to suit ourself. 18 | * " .svn/ .bzr/" 19 | */ 20 | 21 | // Filter List 22 | type Exclusion struct { 23 | patterns []string 24 | root string 25 | } 26 | 27 | func (e *Exclusion) Match(name string) (matched bool, err error) { 28 | matched = false 29 | for _, p := range e.patterns { 30 | if strings.HasPrefix(name, p) && name[len(p)] == '/' { 31 | return true, nil 32 | } 33 | if matched, err = filepath.Match(p, name); matched || err != nil {break} 34 | } 35 | return 36 | } 37 | 38 | func (e *Exclusion) Add(pattern string) { 39 | // Check the root, if not empty, join them 40 | e.patterns = append(e.patterns, filepath.Join(e.root, pattern)) 41 | } 42 | 43 | // This is only called by the client 44 | func (e *Exclusion) SendExlusion(conn Conn) error { 45 | // If list_only && !recurse, add '/*/*' 46 | 47 | 48 | // For each item, send its length first 49 | for _, p := range e.patterns { 50 | plen := int32(len(p)) 51 | // TODO: If a dir, append a '/' at the end 52 | if err := conn.WriteInt(plen); err != nil { 53 | return err 54 | } 55 | if _, err := conn.Write([]byte(p)); err != nil { 56 | return err 57 | } 58 | } 59 | 60 | if err := conn.WriteInt(EXCLUSION_END); err != nil { 61 | return err 62 | } 63 | return nil 64 | } 65 | 66 | -------------------------------------------------------------------------------- /rsync/flist.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | ) 7 | 8 | type FileInfo struct { 9 | Path []byte 10 | Size int64 11 | Mtime int32 12 | Mode FileMode 13 | } 14 | 15 | // For unix/linux 16 | type FileMode uint32 17 | 18 | // reference: os/types.go 19 | func NewFileMode(mode os.FileMode) FileMode { 20 | m := FileMode(mode & 0777) 21 | // TODO: supports symlink 22 | if mode.IsRegular() { 23 | return m | S_IFREG 24 | } 25 | 26 | types := map[uint32]FileMode { 27 | 0: S_IFDIR, 28 | 4: S_IFLNK, 29 | 5: S_IFBLK, 30 | 6: S_IFIFO, 31 | 7: S_IFSOCK, 32 | 10: S_IFCHR, 33 | } 34 | for i, t := range types { 35 | if m&(1< b. 169 | // Compare their paths by bytes.Compare 170 | // The result will be 0 if a==b, -1 if a < b, and +1 if a > b 171 | // If 1, B doesn't have 172 | // If 0, A & B have 173 | // If -1, A doesn't have 174 | switch bytes.Compare(L[i].Path, R[j].Path) { 175 | case 0: 176 | if L[i].Mtime != R[j].Mtime || L[i].Size != R[j].Size { 177 | newitems = append(newitems, j) 178 | } 179 | i++ 180 | j++ 181 | break 182 | case 1: 183 | newitems = append(newitems, j) 184 | j++ 185 | break 186 | case -1: 187 | olditems = append(olditems, i) 188 | i++ 189 | break 190 | } 191 | } 192 | 193 | // Handle remains 194 | for ; i < len(L); i++ { 195 | olditems = append(olditems, i) 196 | } 197 | for ; j < len(R); j++ { 198 | newitems = append(newitems, j) 199 | } 200 | 201 | return 202 | } 203 | -------------------------------------------------------------------------------- /rsync/fs.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | type FileMetadata struct { 8 | Mtime int32 9 | Mode FileMode 10 | } 11 | 12 | // File System: need to handle all type of files: regular, folder, symlink, etc 13 | type FS interface { 14 | Put(fileName string, content io.Reader, fileSize int64, metadata FileMetadata) (written int64, err error) 15 | //Get(fileName string, metadata FileMetadata) (File, error) 16 | Delete(fileName string, mode FileMode) error 17 | List() (FileList, error) 18 | //Stats() (seekable bool) 19 | } 20 | 21 | // Interface: Read, ReadAt, Seek, Close 22 | type File interface { 23 | io.Reader 24 | io.ReaderAt 25 | io.Seeker 26 | io.Closer 27 | } -------------------------------------------------------------------------------- /rsync/receiver.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "errors" 7 | "fmt" 8 | "github.com/kaiakz/ubuffer" 9 | "io" 10 | "log" 11 | "sort" 12 | "time" 13 | ) 14 | 15 | /* Receiver: 16 | 1. Receive File list 17 | 2. Request files by sending files' index 18 | 3. Receive Files, pass the files to storage 19 | */ 20 | type Receiver struct { 21 | conn *Conn 22 | module string 23 | path string 24 | seed int32 25 | lVer int32 26 | rVer int32 27 | storage FS 28 | } 29 | 30 | func (r *Receiver) BuildArgs() string { 31 | return "" 32 | } 33 | 34 | // DeMux was started here 35 | func (r *Receiver) StartMuxIn() { 36 | r.conn.reader = NewMuxReaderV0(r.conn.reader) 37 | } 38 | 39 | func (r *Receiver) SendExclusions() error { 40 | // Send exclusion 41 | return r.conn.WriteInt(EXCLUSION_END) 42 | } 43 | 44 | // Return a filelist from remote 45 | func (r *Receiver) RecvFileList() (FileList, map[string][]byte, error) { 46 | filelist := make(FileList, 0, 1 *M) 47 | symlinks := make(map[string][]byte) 48 | for { 49 | flags, err := r.conn.ReadByte() 50 | if err != nil { 51 | return filelist, symlinks, err 52 | } 53 | 54 | if flags == FLIST_END { 55 | break 56 | } 57 | //fmt.Printf("[%d]\n", flags) 58 | 59 | lastIndex := len(filelist) - 1 60 | var partial, pathlen uint32 = 0, 0 61 | 62 | /* 63 | * Read our filename. 64 | * If we have FLIST_NAME_SAME, we inherit some of the last 65 | * transmitted name. 66 | * If we have FLIST_NAME_LONG, then the string length is greater 67 | * than byte-size. 68 | */ 69 | if (flags & FLIST_NAME_SAME) != 0 { 70 | val, err := r.conn.ReadByte() 71 | if err != nil { 72 | return filelist, symlinks, err 73 | } 74 | partial = uint32(val) 75 | //fmt.Println("Partical", partial) 76 | } 77 | 78 | /* Get the (possibly-remaining) filename length. */ 79 | if (flags & FLIST_NAME_LONG) != 0 { 80 | val, err := r.conn.ReadInt() 81 | if err != nil { 82 | return filelist, symlinks, err 83 | } 84 | pathlen = uint32(val) // can't use for rsync 31 85 | 86 | } else { 87 | val, err := r.conn.ReadByte() 88 | if err != nil { 89 | return filelist, symlinks, err 90 | } 91 | pathlen = uint32(val) 92 | } 93 | //fmt.Println("PathLen", pathlen) 94 | 95 | /* Allocate our full filename length. */ 96 | /* FIXME: maximum pathname length. */ 97 | // TODO: if pathlen + partical == 0 98 | // malloc len error? 99 | 100 | p := make([]byte, pathlen) 101 | _, err = io.ReadFull(r.conn, p) 102 | if err != nil { 103 | return filelist, symlinks, err 104 | } 105 | 106 | path := make([]byte, 0, partial + pathlen) 107 | /* If so, use last */ 108 | if (flags & FLIST_NAME_SAME) != 0 { // FLIST_NAME_SAME 109 | last := filelist[lastIndex] 110 | path = append(path, last.Path[0:partial]...) 111 | } 112 | path = append(path, p...) 113 | //fmt.Println("Path ", string(path)) 114 | 115 | size, err := r.conn.ReadVarint() 116 | if err != nil { 117 | return filelist, symlinks, err 118 | } 119 | //fmt.Println("Size ", size) 120 | 121 | /* Read the modification time. */ 122 | var mtime int32 123 | if (flags & FLIST_TIME_SAME) == 0 { 124 | mtime, err = r.conn.ReadInt() 125 | if err != nil { 126 | return filelist, symlinks, err 127 | } 128 | } else { 129 | mtime = filelist[lastIndex].Mtime 130 | } 131 | //fmt.Println("MTIME ", mtime) 132 | 133 | /* Read the file mode. */ 134 | var mode FileMode 135 | if (flags & FLIST_MODE_SAME) == 0 { 136 | val, err := r.conn.ReadInt() 137 | if err != nil { 138 | return filelist, symlinks, err 139 | } 140 | mode = FileMode(val) 141 | } else { 142 | mode = filelist[lastIndex].Mode 143 | } 144 | //fmt.Println("Mode", uint32(mode)) 145 | 146 | // TODO: Sym link 147 | if ((mode & 32768) != 0) && ((mode & 8192) != 0) { 148 | sllen, err := r.conn.ReadInt() 149 | if err != nil { 150 | return filelist, symlinks, err 151 | } 152 | slink := make([]byte, sllen) 153 | _, err = io.ReadFull(r.conn, slink) 154 | symlinks[string(path)] = slink 155 | if err != nil { 156 | return filelist, symlinks, errors.New("Failed to read symlink") 157 | } 158 | //fmt.Println("Symbolic Len:", len, "Content:", slink) 159 | } 160 | 161 | fmt.Println("@", string(path), mode, size, mtime) 162 | 163 | filelist = append(filelist, FileInfo{ 164 | Path: path, 165 | Size: size, 166 | Mtime: mtime, 167 | Mode: mode, 168 | }) 169 | } 170 | 171 | // Sort the filelist lexicographically 172 | sort.Sort(filelist) 173 | 174 | return filelist, symlinks, nil 175 | } 176 | 177 | // Generator: handle files: if it's a regular file, send its index. Otherwise, put them to storage 178 | func (r *Receiver) Generator(remoteList FileList, downloadList []int, symlinks map[string][]byte) error { 179 | emptyBlocks := make([]byte, 16) // 4 + 4 + 4 + 4 bytes, all bytes set to 0 180 | content := new(bytes.Buffer) 181 | 182 | for _, v := range downloadList { 183 | if remoteList[v].Mode.IsREG() { 184 | if err := r.conn.WriteInt(int32(v)); err != nil { 185 | log.Println("Failed to send index") 186 | return err 187 | } 188 | //fmt.Println("Request: ", string(remoteList[v].Path), uint32(remoteList[v].Mode)) 189 | if _, err := r.conn.Write(emptyBlocks); err != nil { 190 | return err 191 | } 192 | } else { 193 | // TODO: Supports more file mode 194 | // EXPERIMENTAL 195 | // Handle folders & symbol links 196 | content.Reset() 197 | size := remoteList[v].Size 198 | if remoteList[v].Mode.IsLNK() { 199 | if _, err := content.Write(symlinks[string(remoteList[v].Path)]); err != nil { 200 | return err 201 | } 202 | size = int64(content.Len()) 203 | } 204 | 205 | if _, err := r.storage.Put(string(remoteList[v].Path), content, size, FileMetadata{ 206 | Mtime: remoteList[v].Mtime, 207 | Mode: remoteList[v].Mode, 208 | }); err != nil { 209 | return err 210 | } 211 | } 212 | } 213 | 214 | // Send -1 to finish, then start to download 215 | if err := r.conn.WriteInt(INDEX_END); err != nil { 216 | log.Println("Can't send INDEX_END") 217 | return err 218 | } 219 | log.Println("Request completed") 220 | 221 | startTime := time.Now() 222 | err := r.FileDownloader(remoteList[:]) 223 | log.Println("Downloaded duration:", time.Since(startTime)) 224 | return err 225 | } 226 | 227 | // TODO: It is better to update files in goroutine 228 | func (r *Receiver) FileDownloader(localList FileList) error { 229 | 230 | rmd4 := make([]byte, 16) 231 | 232 | for { 233 | index, err := r.conn.ReadInt() 234 | if err != nil { 235 | return err 236 | } 237 | if index == INDEX_END { // -1 means the end of transfer files 238 | return nil 239 | } 240 | //fmt.Println("INDEX:", index) 241 | 242 | count, err := r.conn.ReadInt() /* block count */ 243 | if err != nil { 244 | return err 245 | } 246 | 247 | blen, err := r.conn.ReadInt() /* block length */ 248 | if err != nil { 249 | return err 250 | } 251 | 252 | clen, err := r.conn.ReadInt() /* checksum length */ 253 | if err != nil { 254 | return err 255 | } 256 | 257 | remainder, err := r.conn.ReadInt() /* block remainder */ 258 | if err != nil { 259 | return err 260 | } 261 | 262 | path := localList[index].Path 263 | log.Println("Downloading:", string(path), count, blen, clen, remainder, localList[index].Size) 264 | 265 | // If the file is too big to store in memory, creates a temporary file in the directory 'tmp' 266 | buffer := ubuffer.NewBuffer(localList[index].Size) 267 | downloadeSize := 0 268 | bufwriter := bufio.NewWriter(buffer) 269 | 270 | // Create MD4 271 | //lmd4 := md4.New() 272 | //if err := binary.Write(lmd4, binary.LittleEndian, r.seed); err != nil { 273 | // log.Println("Failed to compute md4") 274 | //} 275 | 276 | for { 277 | token, err := r.conn.ReadInt() 278 | if err != nil { 279 | return err 280 | } 281 | log.Println("TOKEN", token) 282 | if token == 0 { 283 | break 284 | } else if token < 0 { 285 | return errors.New("Does not support block checksum") 286 | // Reference 287 | } else { 288 | ctx := make([]byte, token) // FIXME: memory leak? 289 | _, err = io.ReadFull(r.conn, ctx) 290 | if err != nil { 291 | return err 292 | } 293 | downloadeSize += int(token) 294 | log.Println("Downloaded:", downloadeSize, "byte") 295 | if _, err := bufwriter.Write(ctx); err != nil { 296 | return err 297 | } 298 | //if _, err := lmd4.Write(ctx); err != nil { 299 | // return err 300 | //} 301 | } 302 | } 303 | if bufwriter.Flush() != nil { 304 | return errors.New("Failed to flush buffer") 305 | } 306 | 307 | // Remote MD4 308 | // TODO: compare computed MD4 with remote MD4 309 | _, err = io.ReadFull(r.conn, rmd4) 310 | if err != nil { 311 | return err 312 | } 313 | // Compare two MD4 314 | //if bytes.Compare(rmd4, lmd4.Sum(nil)) != 0 { 315 | // log.Println("Checksum error") 316 | //} 317 | 318 | // Put file to object storage 319 | var n int64 320 | n, err = buffer.Seek(0, io.SeekStart) 321 | 322 | n, err = r.storage.Put(string(path), buffer, int64(downloadeSize), FileMetadata{ 323 | Mtime: localList[index].Mtime, 324 | Mode: localList[index].Mode, 325 | }) 326 | if err != nil { 327 | return err 328 | } 329 | 330 | if buffer.Finalize() != nil { 331 | return errors.New("Buffer can't be finalized") 332 | } 333 | 334 | log.Printf("Successfully uploaded %s of size %d\n", path, n) 335 | } 336 | } 337 | 338 | // Clean up local files 339 | func (r *Receiver) FileCleaner(localList FileList, deleteList []int) error { 340 | // Since file list was already sorted, we can iterate it in the reverse direction to traverse the file tree in post-order 341 | // Thus it always cleans sub-files firstly 342 | for i := len(deleteList) - 1; i >= 0; i-- { 343 | fname := string(localList[deleteList[i]].Path) 344 | err := r.storage.Delete(fname, localList[deleteList[i]].Mode) 345 | log.Println("Deleted:", fname) 346 | if err != nil { 347 | return err 348 | } 349 | } 350 | return nil 351 | } 352 | 353 | func (r *Receiver) FinalPhase() error { 354 | //go func() { 355 | // ioerror, err := r.conn.ReadInt() 356 | // log.Println(ioerror, err) 357 | //}() 358 | 359 | err := r.conn.WriteInt(INDEX_END) 360 | if err != nil { 361 | return err 362 | } 363 | return r.conn.WriteInt(INDEX_END) 364 | } 365 | 366 | func (r *Receiver) Sync() error { 367 | defer func() { 368 | log.Println("Task completed", r.conn.Close()) // TODO: How to handle errors from Close 369 | }() 370 | 371 | lfiles, err := r.storage.List() 372 | if err != nil { 373 | return err 374 | } 375 | //for _, v := range lfiles { 376 | // fmt.Println("Local File:", string(v.Path), v.Mode, v.Mtime) 377 | //} 378 | 379 | rfiles, symlinks, err := r.RecvFileList() 380 | if err != nil { 381 | return err 382 | } 383 | log.Println("Remote files count:", len(rfiles)) 384 | 385 | ioerr, err := r.conn.ReadInt() 386 | if err != nil { 387 | return nil 388 | } 389 | log.Println("IOERR", ioerr) 390 | 391 | newfiles, oldfiles := lfiles.Diff(rfiles) 392 | if len(newfiles) == 0 && len(oldfiles) == 0 { 393 | log.Println("There is nothing to do") 394 | } 395 | fmt.Print(newfiles, oldfiles) 396 | 397 | if err := r.Generator(rfiles[:], newfiles[:], symlinks); err != nil { 398 | return err 399 | } 400 | if err := r.FileCleaner(lfiles[:], oldfiles[:]); err != nil { 401 | return err 402 | } 403 | if err := r.FinalPhase(); err != nil { 404 | return err 405 | } 406 | return nil 407 | } 408 | 409 | -------------------------------------------------------------------------------- /rsync/sender.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | import ( 4 | "bytes" 5 | ) 6 | 7 | type Sender struct { 8 | conn *Conn 9 | module string 10 | path string 11 | seed int32 12 | lVer int32 13 | rVer int32 14 | storage FS 15 | } 16 | 17 | func (s *Sender) SendFileList() error { 18 | list, err := s.storage.List() 19 | if err != nil { 20 | return err 21 | } 22 | 23 | // Send list to receiver 24 | var last *FileInfo = nil 25 | for _, f := range list{ 26 | var flags byte = 0 27 | 28 | 29 | if bytes.Equal(f.Path, []byte(".")) { 30 | if f.Mode.IsDIR() { 31 | flags |= FLIST_TOP_LEVEL 32 | } 33 | } else { 34 | if f.Mode.IsDIR() { // TODO: recursive 35 | // flags |= Flags.NO_CONTENT_DIR | Flags.XFLAGS; 36 | } 37 | } 38 | 39 | lPathCount := 0 40 | if last != nil { 41 | lPathCount = longestMatch(last.Path, f.Path) 42 | if lPathCount > 255 { // Limit to 255 chars 43 | lPathCount = 255 44 | } 45 | if lPathCount > 0 { 46 | flags |= FLIST_NAME_SAME 47 | } 48 | if last.Mode == f.Mode { 49 | flags |= FLIST_MODE_SAME 50 | } 51 | if last.Mtime == f.Mtime { 52 | flags |= FLIST_TIME_SAME 53 | } 54 | // 55 | // 56 | // 57 | } 58 | 59 | rPathCount := int32(len(f.Path) - lPathCount) 60 | if rPathCount > 255 { 61 | flags |= FLIST_NAME_LONG 62 | } 63 | 64 | /* we must make sure we don't send a zero flags byte or the other 65 | end will terminate the flist transfer */ 66 | if flags == 0 && !f.Mode.IsDIR() { 67 | flags |= 1<<0 68 | } 69 | if flags == 0 { 70 | flags |= FLIST_NAME_LONG 71 | } 72 | /* Send flags */ 73 | if err != s.conn.WriteByte(flags) { 74 | return err 75 | } 76 | 77 | /* Send len of path, and bytes of path */ 78 | if flags& FLIST_NAME_SAME != 0 { 79 | if err = s.conn.WriteByte(flags); err != nil { 80 | return err 81 | } 82 | } 83 | 84 | if flags& FLIST_NAME_LONG != 0 { 85 | if err = s.conn.WriteInt(rPathCount); err != nil { 86 | return err 87 | } 88 | } else { 89 | if err = s.conn.WriteByte(byte(rPathCount)); err != nil { 90 | return err 91 | } 92 | } 93 | 94 | if _, err = s.conn.Write(f.Path[lPathCount:]); err != nil { 95 | return err 96 | } 97 | 98 | /* Send size of file */ 99 | if err = s.conn.WriteLong(f.Size); err != nil { 100 | return err 101 | } 102 | 103 | /* Send Mtime, GID, UID, RDEV if needed */ 104 | if flags& FLIST_TIME_SAME == 0 { 105 | if err = s.conn.WriteInt(f.Mtime); err != nil { 106 | return err 107 | } 108 | } 109 | if flags& FLIST_MODE_SAME == 0 { 110 | if err = s.conn.WriteInt(int32(f.Mode)); err != nil { 111 | return err 112 | } 113 | } 114 | // TODO: UID GID RDEV 115 | 116 | // TODO: Send symlink 117 | 118 | // TODO: if always_checksum? 119 | 120 | last = &f 121 | } 122 | return nil 123 | } 124 | 125 | func (s *Sender) Generator(fileList FileList) error { 126 | for { 127 | index, err := s.conn.ReadInt() 128 | if err != nil { 129 | return err 130 | } else if index == INDEX_END { 131 | break 132 | } 133 | 134 | // Receive block checksum from receiver 135 | count, err := s.conn.ReadInt() 136 | if err != nil { 137 | return nil 138 | } 139 | 140 | blen, err := s.conn.ReadInt() 141 | if err != nil { 142 | return nil 143 | } 144 | 145 | s2len, err := s.conn.ReadInt() 146 | if err != nil { 147 | return nil 148 | } else if s2len > 16 { 149 | // FIXME: check if sum2 length is valid 150 | } 151 | 152 | remainder, err := s.conn.ReadInt() 153 | if err != nil { 154 | return nil 155 | } 156 | 157 | sums := make([]SumChunk, 0, count) 158 | 159 | var ( 160 | i int32 = 0 161 | offset int64 = 0 162 | ) 163 | 164 | 165 | for ; i < count; i++ { 166 | sum1, err := s.conn.ReadInt() // sum1: 167 | if err != nil { 168 | return err 169 | } 170 | 171 | sum2 := make([]byte, 16) 172 | if _, err := s.conn.Read(sum2); err != nil { 173 | return err 174 | } 175 | 176 | chunk := new(SumChunk) 177 | chunk.sum1 = uint32(sum1) 178 | chunk.sum2 = sum2 179 | chunk.fileOffset = offset 180 | 181 | if i == count-1 && remainder != 0 { 182 | chunk.chunkLen = uint(remainder); 183 | } else { 184 | chunk.chunkLen = uint(blen) 185 | } 186 | offset += int64(chunk.chunkLen) 187 | sums = append(sums, ) 188 | } 189 | result := new(SumStruct) 190 | result.fileLen = uint64(offset) 191 | result.count = uint64(count) 192 | result.blockLen = uint64(blen) 193 | result.sum2Len = uint64(s2len) 194 | result.remainder = uint64(remainder) 195 | 196 | } 197 | if err := s.FileUploader(); err != nil { 198 | return err 199 | } 200 | return nil 201 | } 202 | 203 | func (s *Sender) FileUploader() error { 204 | panic("Not implemented yet"); 205 | return nil 206 | } 207 | 208 | func (s *Sender) FinalPhase() error { 209 | panic("Not implemented yet"); 210 | return nil 211 | } 212 | 213 | func (s *Sender) Sync() error { 214 | panic("Not implemented yet"); 215 | return nil 216 | } -------------------------------------------------------------------------------- /rsync/ssh.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | import ( 4 | "golang.org/x/crypto/ssh" 5 | ) 6 | 7 | type SSH struct { 8 | client *ssh.Client 9 | session *ssh.Session 10 | } 11 | 12 | func NewSSH(address string, username string, pwd string, cmd string) (*SSH, error) { 13 | config := &ssh.ClientConfig{ 14 | Config: ssh.Config{}, 15 | User: "", 16 | Auth: nil, 17 | HostKeyCallback: nil, 18 | BannerCallback: nil, 19 | ClientVersion: "", 20 | HostKeyAlgorithms: nil, 21 | Timeout: 0, 22 | } 23 | client, err := ssh.Dial("tcp", address, config) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | session, err := client.NewSession() 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | // Call remote rsync server 34 | if err := session.Run(cmd); err != nil { 35 | return nil, err 36 | } 37 | 38 | // Handshake over SSH 39 | 40 | 41 | return &SSH{ 42 | client: client, 43 | session: session, 44 | }, nil 45 | } 46 | 47 | 48 | func (s *SSH) Write(p []byte) (n int, err error) { 49 | return s.session.Stdout.Write(p) 50 | } 51 | 52 | func (s *SSH) Read(p []byte) (n int, err error) { 53 | return s.session.Stdin.Read(p) 54 | } 55 | 56 | func (s *SSH) Close() error { 57 | return s.client.Close() 58 | } 59 | 60 | -------------------------------------------------------------------------------- /rsync/utils.go: -------------------------------------------------------------------------------- 1 | package rsync 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "log" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | func SplitURIS(uri string) (string, int, string, string, error) { 11 | 12 | var host, module, path string 13 | first := uri 14 | var second string 15 | 16 | if strings.HasPrefix(uri, "rsync://") { 17 | /* rsync://host[:port]/module[/path] */ 18 | first = first[8:] 19 | i := strings.IndexByte(first, '/') 20 | if i == -1 { 21 | // No module name 22 | panic("No module name") 23 | } 24 | second = first[i+1:] //ignore '/' 25 | first = first[:i] 26 | } else { 27 | // Only for remote 28 | /* host::module[/path] */ 29 | panic("No implement yet") 30 | } 31 | 32 | port := 873 // Default port: 873 33 | 34 | // Parse port 35 | i := strings.IndexByte(first, ':') 36 | if i != -1 { 37 | var err error 38 | port, err = strconv.Atoi(string(first[i+1:])) 39 | if err != nil { 40 | // Wrong port 41 | panic("Wrong port") 42 | } 43 | first = first[:i] 44 | } 45 | host = first 46 | 47 | // Parse path 48 | i = strings.IndexByte(second, '/') 49 | if i != -1 { 50 | path = second[i:] 51 | second = second[:i] 52 | } 53 | module = second 54 | 55 | return host, port, module, path, nil 56 | 57 | } 58 | 59 | // For rsync 60 | func SplitURI(uri string) (string, string, string, error) { 61 | 62 | var address, module, path string 63 | var first = uri 64 | var second string 65 | 66 | if strings.HasPrefix(uri, "rsync://") { 67 | /* rsync://host[:port]/module[/path] */ 68 | first = first[8:] 69 | i := strings.IndexByte(first, '/') 70 | if i == -1 { 71 | // No module name 72 | return "", "", "", errors.New("No module name") 73 | } 74 | second = first[i+1:] //ignore '/' 75 | first = first[:i] 76 | } else { 77 | // Only for remote 78 | /* host::module[/path] */ 79 | log.Fatalln("No implement yet") 80 | } 81 | 82 | address = first 83 | // Parse port 84 | i := strings.IndexByte(first, ':') 85 | if i == -1 { 86 | address += ":873" // Default port: 873 87 | } 88 | 89 | // Parse path 90 | i = strings.IndexByte(second, '/') 91 | if i != -1 { 92 | path = second[i:] 93 | second = second[:i] 94 | } 95 | module = second 96 | 97 | return address, module, path, nil 98 | 99 | } 100 | 101 | // The path always has a trailing slash appended 102 | func TrimPrepath(prepath string) string { 103 | //pre-path shouldn't use "/" as prefix, and must have a "/" suffix 104 | //pre-path can be: "xx", "xx/", "/xx", "/xx/", "", "/" 105 | ppath := prepath 106 | if !strings.HasSuffix(ppath, "/") { 107 | ppath += "/" 108 | } 109 | if strings.HasPrefix(ppath, "/") { 110 | ppath = ppath[1:] 111 | } 112 | return ppath 113 | } 114 | 115 | func longestMatch(left []byte, right []byte) int { 116 | i := 0 117 | for ; i < len(left) && i < len(right) && i < 256; i++ { 118 | if left[i] != right[i] { 119 | break 120 | } 121 | } 122 | return i; 123 | } -------------------------------------------------------------------------------- /storage/cache/finfo.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.24.0 4 | // protoc v3.12.0 5 | // source: finfo.proto 6 | 7 | package cache 8 | 9 | import ( 10 | proto "github.com/golang/protobuf/proto" 11 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 12 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 13 | reflect "reflect" 14 | sync "sync" 15 | ) 16 | 17 | const ( 18 | // Verify that this generated code is sufficiently up-to-date. 19 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 20 | // Verify that runtime/protoimpl is sufficiently up-to-date. 21 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 22 | ) 23 | 24 | // This is a compile-time assertion that a sufficiently up-to-date version 25 | // of the legacy proto package is being used. 26 | const _ = proto.ProtoPackageIsVersion4 27 | 28 | type FInfo struct { 29 | state protoimpl.MessageState 30 | sizeCache protoimpl.SizeCache 31 | unknownFields protoimpl.UnknownFields 32 | 33 | Size int64 `protobuf:"varint,1,opt,name=Size,proto3" json:"Size,omitempty"` 34 | Mtime int32 `protobuf:"varint,2,opt,name=Mtime,proto3" json:"Mtime,omitempty"` 35 | Mode int32 `protobuf:"varint,3,opt,name=Mode,proto3" json:"Mode,omitempty"` // hash 16bytes 36 | } 37 | 38 | func (x *FInfo) Reset() { 39 | *x = FInfo{} 40 | if protoimpl.UnsafeEnabled { 41 | mi := &file_finfo_proto_msgTypes[0] 42 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 43 | ms.StoreMessageInfo(mi) 44 | } 45 | } 46 | 47 | func (x *FInfo) String() string { 48 | return protoimpl.X.MessageStringOf(x) 49 | } 50 | 51 | func (*FInfo) ProtoMessage() {} 52 | 53 | func (x *FInfo) ProtoReflect() protoreflect.Message { 54 | mi := &file_finfo_proto_msgTypes[0] 55 | if protoimpl.UnsafeEnabled && x != nil { 56 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 57 | if ms.LoadMessageInfo() == nil { 58 | ms.StoreMessageInfo(mi) 59 | } 60 | return ms 61 | } 62 | return mi.MessageOf(x) 63 | } 64 | 65 | // Deprecated: Use FInfo.ProtoReflect.Descriptor instead. 66 | func (*FInfo) Descriptor() ([]byte, []int) { 67 | return file_finfo_proto_rawDescGZIP(), []int{0} 68 | } 69 | 70 | func (x *FInfo) GetSize() int64 { 71 | if x != nil { 72 | return x.Size 73 | } 74 | return 0 75 | } 76 | 77 | func (x *FInfo) GetMtime() int32 { 78 | if x != nil { 79 | return x.Mtime 80 | } 81 | return 0 82 | } 83 | 84 | func (x *FInfo) GetMode() int32 { 85 | if x != nil { 86 | return x.Mode 87 | } 88 | return 0 89 | } 90 | 91 | var File_finfo_proto protoreflect.FileDescriptor 92 | 93 | var file_finfo_proto_rawDesc = []byte{ 94 | 0x0a, 0x0b, 0x66, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x05, 0x70, 95 | 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x45, 0x0a, 0x05, 0x46, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 96 | 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x53, 0x69, 0x7a, 97 | 0x65, 0x12, 0x14, 0x0a, 0x05, 0x4d, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 98 | 0x52, 0x05, 0x4d, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x18, 99 | 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x4d, 0x6f, 0x64, 0x65, 0x42, 0x21, 0x5a, 0x1f, 0x67, 100 | 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x61, 0x69, 0x61, 0x6b, 0x7a, 101 | 0x2f, 0x72, 0x73, 0x79, 0x6e, 0x63, 0x2d, 0x6f, 0x73, 0x2f, 0x66, 0x6c, 0x64, 0x62, 0x62, 0x06, 102 | 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 103 | } 104 | 105 | var ( 106 | file_finfo_proto_rawDescOnce sync.Once 107 | file_finfo_proto_rawDescData = file_finfo_proto_rawDesc 108 | ) 109 | 110 | func file_finfo_proto_rawDescGZIP() []byte { 111 | file_finfo_proto_rawDescOnce.Do(func() { 112 | file_finfo_proto_rawDescData = protoimpl.X.CompressGZIP(file_finfo_proto_rawDescData) 113 | }) 114 | return file_finfo_proto_rawDescData 115 | } 116 | 117 | var file_finfo_proto_msgTypes = make([]protoimpl.MessageInfo, 1) 118 | var file_finfo_proto_goTypes = []interface{}{ 119 | (*FInfo)(nil), // 0: proto.FInfo 120 | } 121 | var file_finfo_proto_depIdxs = []int32{ 122 | 0, // [0:0] is the sub-list for method output_type 123 | 0, // [0:0] is the sub-list for method input_type 124 | 0, // [0:0] is the sub-list for extension type_name 125 | 0, // [0:0] is the sub-list for extension extendee 126 | 0, // [0:0] is the sub-list for field type_name 127 | } 128 | 129 | func init() { file_finfo_proto_init() } 130 | func file_finfo_proto_init() { 131 | if File_finfo_proto != nil { 132 | return 133 | } 134 | if !protoimpl.UnsafeEnabled { 135 | file_finfo_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 136 | switch v := v.(*FInfo); i { 137 | case 0: 138 | return &v.state 139 | case 1: 140 | return &v.sizeCache 141 | case 2: 142 | return &v.unknownFields 143 | default: 144 | return nil 145 | } 146 | } 147 | } 148 | type x struct{} 149 | out := protoimpl.TypeBuilder{ 150 | File: protoimpl.DescBuilder{ 151 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 152 | RawDescriptor: file_finfo_proto_rawDesc, 153 | NumEnums: 0, 154 | NumMessages: 1, 155 | NumExtensions: 0, 156 | NumServices: 0, 157 | }, 158 | GoTypes: file_finfo_proto_goTypes, 159 | DependencyIndexes: file_finfo_proto_depIdxs, 160 | MessageInfos: file_finfo_proto_msgTypes, 161 | }.Build() 162 | File_finfo_proto = out.File 163 | file_finfo_proto_rawDesc = nil 164 | file_finfo_proto_goTypes = nil 165 | file_finfo_proto_depIdxs = nil 166 | } 167 | -------------------------------------------------------------------------------- /storage/cache/finfo.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package proto; 3 | 4 | option go_package = "github.com/kaiakz/rsync-os/fldb"; 5 | 6 | message FInfo { 7 | int64 Size = 1; 8 | int32 Mtime = 2; 9 | int32 Mode = 3; 10 | // hash 16bytes 11 | } -------------------------------------------------------------------------------- /storage/cache/pbgen.sh: -------------------------------------------------------------------------------- 1 | export PATH=$PATH:~/go/bin # Where protoc-gen-go is 2 | protoc -I=. --go_out=. --go_opt=paths=source_relative finfo.proto 3 | # protoc -I=fldb/ --go_out=fldb/ --go_opt=paths=source_relative fldb/finfo.proto -------------------------------------------------------------------------------- /storage/cache/redis.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | -------------------------------------------------------------------------------- /storage/cache/utils.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "github.com/golang/protobuf/proto" 5 | bolt "go.etcd.io/bbolt" 6 | "log" 7 | "rsync-os/rsync" 8 | "time" 9 | ) 10 | 11 | // Test 12 | func Snapshot(list rsync.FileList, module string, prepath string) { 13 | startTime := time.Now() 14 | db, err := bolt.Open("test.db", 0666, nil) 15 | defer db.Close() 16 | if err != nil { 17 | return 18 | } 19 | 20 | err = db.Update(func(tx *bolt.Tx) error { 21 | bucket := tx.Bucket([]byte(module)) 22 | // If bucket does not exist, create the bucket 23 | if bucket == nil { 24 | var err error 25 | bucket, err = tx.CreateBucket([]byte(module)) 26 | if err != nil { 27 | return err 28 | } 29 | } 30 | for _, info := range list { 31 | key := append([]byte(prepath), info.Path[:]...) 32 | value, err := proto.Marshal(&FInfo{ 33 | Size: info.Size, 34 | Mtime: info.Mtime, 35 | Mode: int32(info.Mode), // FIXME: convert uint32 to int32 36 | }) 37 | if err != nil { 38 | log.Println("Marshal failed", err) 39 | return err 40 | } 41 | err = bucket.Put(key, value) 42 | if err != nil { 43 | return err 44 | } 45 | } 46 | return nil 47 | }) 48 | if err != nil { 49 | log.Println("Update failed", err) 50 | } 51 | log.Println("Saved Duration", time.Since(startTime)) 52 | } -------------------------------------------------------------------------------- /storage/local.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | "rsync-os/rsync" 10 | ) 11 | 12 | type Local struct { 13 | workDir string // Module + Path 14 | } 15 | 16 | func NewLocal(module string, path string, topDir string) (*Local, error) { 17 | // First, creates a module folder under topDir 18 | workDir := filepath.Join(topDir, module, path) 19 | if err := os.MkdirAll(workDir, os.ModePerm); err != nil { 20 | return nil, err 21 | } 22 | return &Local{workDir: workDir}, nil 23 | } 24 | 25 | func (l *Local) Put(fileName string, content io.Reader, fileSize int64, metadata rsync.FileMetadata) (written int64, err error) { 26 | fpath := filepath.Join(l.workDir, fileName) 27 | // if the file is a folder, ignores content, just creates a folder under the workDir 28 | if metadata.Mode.IsDIR() { 29 | return 0, os.Mkdir(fpath, os.ModePerm) 30 | } 31 | 32 | if metadata.Mode.IsREG() { 33 | f, err := os.OpenFile(fpath, os.O_CREATE | os.O_EXCL | os.O_WRONLY, metadata.Mode.Convert()) 34 | if err != nil { 35 | return -1, err 36 | } 37 | defer f.Close() 38 | 39 | // Craete a buffer 40 | fb := bufio.NewWriter(f) 41 | defer fb.Flush() 42 | 43 | return io.Copy(fb, content) 44 | } 45 | 46 | return -2, errors.New("Do not support type " + fileName + metadata.Mode.String()) 47 | } 48 | 49 | func (l *Local) Delete(fileName string, mode rsync.FileMode) error { 50 | if mode.IsDIR() { 51 | return os.RemoveAll(fileName) 52 | } 53 | return os.Remove(fileName) 54 | } 55 | 56 | func (l *Local) List() (rsync.FileList, error) { 57 | filelist := make(rsync.FileList, 0, 1 << 16) 58 | 59 | if err := filepath.Walk(l.workDir, func(path string, info os.FileInfo, err error) error { 60 | if err != nil { 61 | return err 62 | } 63 | 64 | filelist = append(filelist, rsync.FileInfo{ 65 | Path: []byte(info.Name()), 66 | Size: info.Size(), 67 | Mtime: int32(info.ModTime().Unix()), // FIXME 68 | Mode: rsync.NewFileMode(info.Mode()), 69 | }) 70 | 71 | return nil 72 | }); err != nil { 73 | return filelist, err 74 | } 75 | return filelist, nil 76 | } 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /storage/minio.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "bytes" 5 | "github.com/golang/protobuf/proto" 6 | "github.com/minio/minio-go/v6" 7 | bolt "go.etcd.io/bbolt" 8 | "io" 9 | "log" 10 | "path" 11 | "path/filepath" 12 | "rsync-os/rsync" 13 | "rsync-os/storage/cache" 14 | "sort" 15 | "strconv" 16 | "strings" 17 | ) 18 | 19 | /* 20 | rsync-os will add addition information for each file that was uploaded to minio 21 | rsync-os stores the information of a folder in the metadata of an empty file called "..." 22 | rsync-os also uses a strange file to represent a soft link 23 | */ 24 | 25 | // S3 with cache 26 | type Minio struct { 27 | client *minio.Client 28 | bucketName string 29 | prefix string 30 | /* Cache */ 31 | cache *bolt.DB 32 | tx *bolt.Tx 33 | bucket *bolt.Bucket 34 | } 35 | 36 | const S3_DIR = ".dir.rsync-os" 37 | 38 | func NewMinio(bucket string, prefix string, cachePath string, endpoint string, accessKeyID string, secretAccessKey string, secure bool) (*Minio, error) { 39 | minioClient, err := minio.New(endpoint, accessKeyID, secretAccessKey, false) 40 | if err != nil { 41 | panic("Failed to init a minio client") 42 | } 43 | // Create a bucket for the module 44 | err = minioClient.MakeBucket(bucket, "us-east-1") 45 | if err != nil { 46 | // Check to see if we already own this bucket (which happens if you run this twice) 47 | exists, errBucketExists := minioClient.BucketExists(bucket) 48 | if errBucketExists == nil && exists { 49 | log.Printf("We already own %s\n", bucket) 50 | } else { 51 | log.Fatalln(err) 52 | } 53 | } else { 54 | log.Printf("Successfully created %s\n", bucket) 55 | } 56 | 57 | // Initialize cache 58 | db, err := bolt.Open(cachePath, 0666, nil) 59 | if err != nil { 60 | panic("Can't init cache: boltdb") 61 | } 62 | tx, err := db.Begin(true) 63 | if err != nil { 64 | panic(err) 65 | } 66 | 67 | // If bucket does not exist, create the bucket 68 | mod, err := tx.CreateBucketIfNotExists([]byte(bucket)) 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | return &Minio{ 74 | client: minioClient, 75 | bucketName: bucket, 76 | prefix: prefix, 77 | cache: db, 78 | tx: tx, 79 | bucket: mod, 80 | }, nil 81 | } 82 | 83 | // object can be a regualar file, folder or symlink 84 | func (m *Minio) Put(fileName string, content io.Reader, fileSize int64, metadata rsync.FileMetadata) (written int64, err error) { 85 | data := make(map[string]string) 86 | data["mtime"] = strconv.Itoa(int(metadata.Mtime)) 87 | data["mode"] = strconv.Itoa(int(metadata.Mode)) 88 | 89 | fpath := filepath.Join(m.prefix, fileName) 90 | fsize := fileSize 91 | fname := fpath 92 | /* EXPERIMENTAL */ 93 | // Folder 94 | if metadata.Mode.IsDIR() { 95 | fname = filepath.Join(m.prefix, fileName, S3_DIR) 96 | fsize = 0 97 | // FIXME: How to handle a file named ".rsync-os.dir"? 98 | } 99 | 100 | if metadata.Mode.IsLNK() { 101 | // Additional data of symbol link 102 | } 103 | 104 | written, err = m.client.PutObject(m.bucketName, fname, content, fsize, minio.PutObjectOptions{UserMetadata: data}) 105 | 106 | value, err := proto.Marshal(&cache.FInfo{ 107 | Size: fileSize, 108 | Mtime: metadata.Mtime, 109 | Mode: int32(metadata.Mode), // FIXME: convert uint32 to int32 110 | }) 111 | if err != nil { 112 | return -1, err 113 | } 114 | if err := m.bucket.Put([]byte(fpath), value); err != nil { 115 | return -1, err 116 | } 117 | 118 | return 119 | } 120 | 121 | func (m *Minio) Delete(fileName string, mode rsync.FileMode) (err error) { 122 | fpath := filepath.Join(m.prefix, fileName) 123 | // TODO: How to delete a folder 124 | if mode.IsDIR() { 125 | err = m.client.RemoveObject(m.bucketName, filepath.Join(fpath, S3_DIR)) 126 | } else { 127 | if err = m.client.RemoveObject(m.bucketName, fpath); err != nil { 128 | return 129 | } 130 | } 131 | log.Println(fileName) 132 | return m.bucket.Delete([]byte(fpath)) 133 | } 134 | 135 | // EXPERIMENTAL 136 | func (m *Minio) List() (rsync.FileList, error) { 137 | filelist := make(rsync.FileList, 0, 1 << 16) 138 | 139 | // We don't list all files directly 140 | 141 | info := &cache.FInfo{} 142 | 143 | // Add files in the work dir 144 | c := m.bucket.Cursor() 145 | prefix := []byte(m.prefix) 146 | k, v := c.Seek(prefix) 147 | hasdot := false 148 | for k != nil && bytes.HasPrefix(k, prefix) { 149 | p := k[len(prefix):] 150 | if bytes.Equal(p, []byte(".")) { 151 | hasdot = true 152 | } 153 | 154 | if err := proto.Unmarshal(v, info); err != nil { 155 | return filelist, err 156 | } 157 | filelist = append(filelist, rsync.FileInfo{ 158 | Path: p, // ignore prefix 159 | Size: info.Size, 160 | Mtime: info.Mtime, 161 | Mode: rsync.FileMode(info.Mode), 162 | }) 163 | k, v = c.Next() 164 | } 165 | 166 | // Add current dir as . 167 | if !hasdot { 168 | workdir := []byte(filepath.Clean(m.prefix)) // If a empty string, we get "." 169 | v := m.bucket.Get(workdir) 170 | if v == nil { 171 | return filelist, nil 172 | } 173 | if err := proto.Unmarshal(v, info); err != nil { 174 | return filelist, err 175 | } 176 | filelist = append(filelist, rsync.FileInfo{ 177 | Path: []byte("."), 178 | Size: info.Size, 179 | Mtime: info.Mtime, 180 | Mode: rsync.FileMode(info.Mode), 181 | }) 182 | } 183 | 184 | sort.Sort(filelist) 185 | 186 | return filelist, nil 187 | } 188 | 189 | func (m *Minio) ListObj() (rsync.FileList, error) { 190 | filelist := make(rsync.FileList, 0, 1 << 16) 191 | 192 | // Create a done channel to control 'ListObjects' go routine. 193 | doneCh := make(chan struct{}) 194 | 195 | // Indicate to our routine to exit cleanly upon return. 196 | defer close(doneCh) 197 | 198 | // FIXME: objectPrefix, recursive 199 | objectCh := m.client.ListObjectsV2(m.bucketName, "", true, doneCh) 200 | for object := range objectCh { 201 | if object.Err != nil { 202 | log.Println(object.Err) 203 | return filelist, object.Err 204 | } 205 | 206 | // FIXME: Handle folder 207 | objectName := object.Key 208 | if strings.Compare(path.Base(objectName), "...") == 0 { 209 | objectName = path.Dir(objectName) 210 | } 211 | 212 | mtime, err := strconv.Atoi(object.UserMetadata["mtime"]) 213 | if err != nil { 214 | panic("Can't get the mode from minio") 215 | } 216 | 217 | mode, err := strconv.Atoi(object.UserMetadata["mtime"]) 218 | if err != nil { 219 | panic("Can't get the mode from minio") 220 | } 221 | 222 | filelist = append(filelist, rsync.FileInfo{ 223 | Path: []byte(objectName), 224 | Size: object.Size, 225 | Mtime: int32(mtime), 226 | Mode: rsync.FileMode(mode), 227 | }) 228 | } 229 | return filelist, nil 230 | } 231 | 232 | func (m *Minio) Close() error { 233 | defer m.cache.Close() 234 | if err := m.tx.Commit(); err != nil { 235 | return err 236 | } 237 | return nil 238 | } 239 | --------------------------------------------------------------------------------