├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── chunker.go ├── chunker_fixed.go ├── chunker_gpt.go ├── chunker_mbr.go ├── chunker_ntfs.go ├── cmd └── fsdup │ └── main.go ├── export.go ├── go.mod ├── go.sum ├── import.go ├── index.go ├── log.go ├── manifest.go ├── map.go ├── meta_store.go ├── meta_store_file.go ├── meta_store_mysql.go ├── meta_store_remote.go ├── pb ├── manifest.pb.go ├── manifest.proto ├── service.pb.go └── service.proto ├── server.go ├── stat.go ├── store.go ├── store_ceph.go ├── store_dummy.go ├── store_file.go ├── store_gcloud.go ├── store_remote.go ├── store_swift.go ├── upload.go └── util.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | build 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 Philipp Heckel 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION=0.1.0-alpha 2 | 3 | help: 4 | @echo "Build:" 5 | @echo " make all - Build all deliverables" 6 | @echo " make cmd - Build the CLI tool" 7 | @echo " make clean - Clean build folder" 8 | 9 | all: clean proto cmd 10 | 11 | clean: 12 | @echo == Cleaning == 13 | rm -rf build 14 | @echo 15 | 16 | proto: 17 | @echo == Generating protobuf code == 18 | protoc --go_out=plugins=grpc:. pb/*.proto 19 | @echo 20 | 21 | cmd: proto 22 | @echo == Building CLI == 23 | mkdir -p build/cmd 24 | go build \ 25 | -o build/fsdup \ 26 | -ldflags \ 27 | "-X main.buildversion=${VERSION} -X main.buildcommit=$(shell git rev-parse --short HEAD) -X main.builddate=$(shell date +%s)" \ 28 | cmd/fsdup/main.go 29 | @echo 30 | @echo "--> fsdup CLI built at build/fsdup" 31 | @echo 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | fsdup 2 | ===== 3 | `fsdup` is a file system deduplication tool, mainly for deduping files _inside_ the NTFS filesystem. 4 | As of today, `fsdup` is a **work in progress** and barely tested, so please don't use it for anything important. 5 | 6 | Usage 7 | ----- 8 | Use `fsdup index` to index/dedup a disk or filesystem and store the resulting chunks in Ceph or on disk. After the 9 | index process, you'll have a manifest file describing the indexed disk/filesystem which can then be used 10 | to map it to a local drive using the `fsdup map` command, or export the image using `fsdup export`. 11 | 12 | Example 13 | ------- 14 | ``` 15 | $ fsdup index /dev/loop1 mydisk.manifest 16 | # Writes chunks to 'index' folder (default) and generates mydisk.manifest 17 | 18 | $ fsdup index -store /mnt/myindex ... 19 | # Writes chunks to '/mnt/myindex' folder instead 20 | 21 | $ fsdup index -store 'ceph:ceph.conf?pool=chunkindex' ... 22 | # Writes chunks to Ceph pool 'chunkindex' instead, using 23 | # the 'ceph.conf' config file 24 | 25 | $ fsdup print mydisk.manifest 26 | idx 0000000000 diskoff 0000000000000 - 0000000196608 len 196608 gap chunk a4bfc70231b80a636dfc2aeff92bfbc18c3225a424be097e88173d5fce1f68ae chunkoff 0 - 196608 27 | idx 0000000001 diskoff 0000000196608 - 0000039288832 len 39092224 sparse - 28 | idx 0000000002 diskoff 0000039288832 - 0000039759872 len 471040 gap chunk b8976c52c266cca71fbc16a4195b7a28cf7b2088d862e2590c246e3a9620864f chunkoff 0 - 471040 29 | ... 30 | 31 | $ fsdup export mydisk.manifest mydisk.img 32 | # Creates an image file from chunk index using the manifest 33 | 34 | $ fsdup map mydisk.manifest 35 | # Creates a block device /dev/nbdX that allows reading the image file without exporting it 36 | 37 | $ fsdup stat *.manifest 38 | Manifests: 79 39 | Number of unique chunks: 6210530 40 | Total image size: 22.0 TB (24167171188224 bytes) 41 | Total on disk size: 9.3 TB (10181703710208 bytes) 42 | Total sparse size: 12.7 TB (13985467478016 bytes) 43 | Total chunk size: 6.2 TB (6805815365169 bytes) 44 | Average chunk size: 1.0 MB (1095851 bytes) 45 | Dedup ratio: 1.5 : 1 46 | Space reduction ratio: 33.2 % 47 | ``` 48 | 49 | Commands 50 | -------- 51 | ``` 52 | Syntax: 53 | fsdup index [-debug] [-nowrite] [-store STORE] [-offset OFFSET] [-minsize MINSIZE] [-exact] INFILE MANIFEST 54 | fsdup map [-debug] MANIFEST 55 | fsdup export [-debug] MANIFEST OUTFILE 56 | fsdup print [-debug] MANIFEST 57 | fsdup stat [-debug] MANIFEST... 58 | ``` 59 | 60 | Author 61 | ------ 62 | Philipp Heckel 63 | 64 | License 65 | ------- 66 | [Apache License 2.0](LICENSE) 67 | -------------------------------------------------------------------------------- /chunker.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "fmt" 5 | "golang.org/x/crypto/blake2b" 6 | ) 7 | 8 | const ( 9 | DefaultDedupFileSizeMinBytes = 128 * 1024 10 | 11 | // Max size for any chunk produced by any of the chunkers. Note that lowering this 12 | // probably has terrible consequences for existing manifests, because the buffers all use this 13 | // value. Splitting this value and the buffer size would be a way to solve this. 14 | DefaultChunkSizeMaxBytes = 32 * 1024 * 1024 15 | ) 16 | 17 | type Chunker interface { 18 | Dedup() (*manifest, error) 19 | } 20 | 21 | type chunk struct { 22 | size int64 23 | data []byte 24 | checksum []byte 25 | } 26 | 27 | func NewChunk(maxSize int64) *chunk { 28 | return &chunk{ 29 | size: 0, 30 | data: make([]byte, maxSize), 31 | checksum: nil, 32 | } 33 | } 34 | 35 | func (c *chunk) Reset() { 36 | c.size = 0 37 | c.checksum = nil 38 | } 39 | 40 | func (c *chunk) Write(data []byte) { 41 | copy(c.data[c.size:c.size+int64(len(data))], data) 42 | c.checksum = nil // reset! 43 | c.size += int64(len(data)) 44 | } 45 | 46 | func (c *chunk) Checksum() []byte { 47 | if c.checksum == nil { 48 | checksum := blake2b.Sum256(c.data[:c.size]) 49 | c.checksum = checksum[:] 50 | } 51 | 52 | return c.checksum 53 | } 54 | 55 | func (c *chunk) ChecksumString() string { 56 | return fmt.Sprintf("%x", c.Checksum()) 57 | } 58 | 59 | func (c *chunk) Data() []byte { 60 | return c.data[:c.size] 61 | } 62 | 63 | func (c *chunk) Size() int64 { 64 | return c.size 65 | } 66 | 67 | func (c *chunk) Remaining() int64 { 68 | return int64(len(c.data)) - c.size 69 | } 70 | 71 | func (c *chunk) Full() bool { 72 | return c.Remaining() <= 0 73 | } 74 | 75 | -------------------------------------------------------------------------------- /chunker_fixed.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "sync" 7 | ) 8 | 9 | type fixedChunker struct { 10 | reader io.ReaderAt 11 | store ChunkStore 12 | start int64 13 | sizeInBytes int64 14 | chunkMaxSize int64 15 | writeConcurrency int64 16 | skip *manifest 17 | } 18 | 19 | func NewFixedChunker(reader io.ReaderAt, index ChunkStore, offset int64, size int64, chunkMaxSize int64, 20 | writeConcurrency int64) *fixedChunker { 21 | 22 | skip := NewManifest(chunkMaxSize) 23 | return NewFixedChunkerWithSkip(reader, index, offset, size, chunkMaxSize, writeConcurrency, skip) 24 | } 25 | 26 | func NewFixedChunkerWithSkip(reader io.ReaderAt, store ChunkStore, offset int64, size int64, 27 | chunkMaxSize int64, writeConcurrency int64, skip *manifest) *fixedChunker { 28 | 29 | return &fixedChunker{ 30 | reader: reader, 31 | store: store, 32 | start: offset, 33 | sizeInBytes: size, 34 | chunkMaxSize: chunkMaxSize, 35 | writeConcurrency: writeConcurrency, 36 | skip: skip, 37 | } 38 | } 39 | 40 | func (d *fixedChunker) Dedup() (*manifest, error) { 41 | out := NewManifest(d.chunkMaxSize) 42 | 43 | sliceOffsets := d.skip.Offsets() 44 | 45 | currentOffset := int64(0) 46 | breakpointIndex := 0 47 | breakpoint := int64(0) 48 | 49 | wg := sync.WaitGroup{} 50 | writeChan := make(chan *chunk) 51 | errChan := make(chan error) 52 | chunkPool := &sync.Pool{ 53 | New: func() interface{} { 54 | return NewChunk(d.chunkMaxSize) 55 | }, 56 | } 57 | 58 | for i := int64(0); i < d.writeConcurrency; i++ { 59 | go func() { 60 | wg.Add(1) 61 | defer wg.Done() 62 | 63 | for c := range writeChan { 64 | if err := d.store.Write(c.Checksum(), c.Data()); err != nil { 65 | errChan <- err 66 | } 67 | chunkPool.Put(c) 68 | } 69 | }() 70 | } 71 | 72 | buffer := make([]byte, d.chunkMaxSize) 73 | 74 | statusf("Creating gap chunks ...") 75 | chunkBytes := int64(0) 76 | chunkCount := int64(0) 77 | 78 | for currentOffset < d.sizeInBytes { 79 | select { 80 | case err := <-errChan: 81 | return nil, err 82 | default: 83 | } 84 | 85 | hasNextBreakpoint := breakpointIndex < len(sliceOffsets) 86 | 87 | if hasNextBreakpoint { 88 | // At this point, we figure out if the space from the current offset to the 89 | // next breakpoint will fit in a full chunk. 90 | 91 | breakpoint = sliceOffsets[breakpointIndex] 92 | bytesToBreakpoint := breakpoint - currentOffset 93 | 94 | if bytesToBreakpoint > d.chunkMaxSize { 95 | // We can fill an entire chunk, because there are enough bytes to the next breakpoint 96 | 97 | chunkEndOffset := minInt64(currentOffset + d.chunkMaxSize, d.sizeInBytes) 98 | 99 | bytesRead, err := d.reader.ReadAt(buffer, d.start + currentOffset) 100 | if err != nil { 101 | return nil, err 102 | } else if int64(bytesRead) != d.chunkMaxSize { 103 | return nil, fmt.Errorf("cannot read all bytes from disk, %d read\n", bytesRead) 104 | } 105 | 106 | achunk := chunkPool.Get().(*chunk) 107 | achunk.Reset() 108 | achunk.Write(buffer[:bytesRead]) 109 | 110 | debugf("offset %d - %d, NEW chunk %x, size %d\n", 111 | currentOffset, chunkEndOffset, achunk.Checksum(), achunk.Size()) 112 | 113 | out.Add(&chunkSlice{ 114 | checksum: achunk.Checksum(), 115 | kind: kindGap, 116 | diskfrom: currentOffset, 117 | diskto: currentOffset + achunk.Size(), 118 | chunkfrom: 0, 119 | chunkto: achunk.Size(), 120 | length: achunk.Size(), 121 | }) 122 | 123 | chunkBytes += achunk.Size() 124 | chunkCount++ 125 | statusf("Creating gap chunk(s) (%d chunk(s), %s) ...", chunkCount, convertBytesToHumanReadable(chunkBytes)) 126 | 127 | writeChan <- achunk 128 | currentOffset = chunkEndOffset 129 | } else { 130 | // There are NOT enough bytes to the next breakpoint to fill an entire chunk 131 | 132 | if bytesToBreakpoint > 0 { 133 | // Create and emit a chunk from the current position to the breakpoint. 134 | // This may create small chunks and is inefficient. 135 | // FIXME this should just buffer the current chunk and not emit is right away. It should FILL UP a chunk later! 136 | 137 | bytesRead, err := d.reader.ReadAt(buffer[:bytesToBreakpoint], d.start + currentOffset) 138 | if err != nil { 139 | return nil, err 140 | } else if int64(bytesRead) != bytesToBreakpoint { 141 | return nil, fmt.Errorf("cannot read all bytes from disk, %d read\n", bytesRead) 142 | } 143 | 144 | achunk := chunkPool.Get().(*chunk) 145 | achunk.Reset() 146 | achunk.Write(buffer[:bytesRead]) 147 | 148 | out.Add(&chunkSlice{ 149 | checksum: achunk.Checksum(), 150 | kind: kindGap, 151 | diskfrom: currentOffset, 152 | diskto: currentOffset + achunk.Size(), 153 | chunkfrom: 0, 154 | chunkto: achunk.Size(), 155 | length: achunk.Size(), 156 | }) 157 | 158 | chunkBytes += achunk.Size() 159 | chunkCount++ 160 | statusf("Creating gap chunk(s) (%d chunk(s), %s) ...", chunkCount, convertBytesToHumanReadable(chunkBytes)) 161 | 162 | debugf("offset %d - %d, NEW2 chunk %x, size %d\n", 163 | currentOffset, currentOffset + bytesToBreakpoint, achunk.Checksum(), achunk.Size()) 164 | 165 | writeChan <- achunk 166 | currentOffset += bytesToBreakpoint 167 | } 168 | 169 | // Now we are AT the breakpoint. 170 | // Simply add this entry to the manifest. 171 | 172 | part := d.skip.Get(breakpoint) 173 | partSize := part.chunkto - part.chunkfrom 174 | 175 | debugf("offset %d - %d, size %d -> FILE chunk %x, offset %d - %d\n", 176 | currentOffset, currentOffset + partSize, partSize, part.checksum, part.chunkfrom, part.chunkto) 177 | 178 | currentOffset += partSize 179 | breakpointIndex++ 180 | } 181 | } else { 182 | chunkEndOffset := minInt64(currentOffset + d.chunkMaxSize, d.sizeInBytes) 183 | chunkSize := chunkEndOffset - currentOffset 184 | 185 | bytesRead, err := d.reader.ReadAt(buffer[:chunkSize], d.start + currentOffset) 186 | if err != nil { 187 | panic(err) 188 | } else if int64(bytesRead) != chunkSize { 189 | panic(fmt.Errorf("cannot read bytes from disk, %d read\n", bytesRead)) 190 | } 191 | 192 | achunk := chunkPool.Get().(*chunk) 193 | achunk.Reset() 194 | achunk.Write(buffer[:bytesRead]) 195 | 196 | debugf("offset %d - %d, NEW3 chunk %x, size %d\n", 197 | currentOffset, chunkEndOffset, achunk.Checksum(), achunk.Size()) 198 | 199 | out.Add(&chunkSlice{ 200 | checksum: achunk.Checksum(), 201 | kind: kindGap, 202 | diskfrom: currentOffset, 203 | diskto: currentOffset + achunk.Size(), 204 | chunkfrom: 0, 205 | chunkto: achunk.Size(), 206 | length: achunk.Size(), 207 | }) 208 | 209 | chunkBytes += achunk.Size() 210 | chunkCount++ 211 | statusf("Indexing gap chunks (%d chunk(s), %s) ...", chunkCount, convertBytesToHumanReadable(chunkBytes)) 212 | 213 | writeChan <- achunk 214 | currentOffset = chunkEndOffset 215 | } 216 | } 217 | 218 | statusf("Waiting for remaining chunk writes ...") 219 | close(writeChan) 220 | wg.Wait() 221 | 222 | statusf("Indexed %s of gaps (%d chunk(s))\n", convertBytesToHumanReadable(chunkBytes), chunkCount) 223 | return out, nil 224 | } 225 | -------------------------------------------------------------------------------- /chunker_gpt.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | const ( 8 | gptSignatureMagic = "EFI PART" 9 | gptSignatureOffset = 512 10 | gptHeaderOffset = 512 11 | gptHeaderLength = 512 12 | gptLogicalSectorSize = 512 13 | gptFirstEntrySectorOffset = 72 14 | gptFirstEntrySectorLength = 8 15 | gptEntryCountOffset = 80 16 | gptEntryCountLength = 4 17 | gptEntrySizeOffset = 84 18 | gptEntrySizeLength = 4 19 | gptEntryFirstSectorRelativeOffset = 32 20 | gptEntryFirstSectorRelativeLength = 8 21 | ) 22 | 23 | type gptDiskChunker struct { 24 | reader io.ReaderAt 25 | store ChunkStore 26 | start int64 27 | size int64 28 | exact bool 29 | noFile bool 30 | minSize int64 31 | chunkMaxSize int64 32 | writeConcurrency int64 33 | manifest *manifest 34 | } 35 | 36 | func NewGptDiskChunker(reader io.ReaderAt, store ChunkStore, offset int64, size int64, exact bool, noFile bool, 37 | minSize int64, chunkMaxSize int64, writeConcurrency int64) *gptDiskChunker { 38 | 39 | return &gptDiskChunker{ 40 | reader: reader, 41 | store: store, 42 | start: offset, 43 | size: size, 44 | exact: exact, 45 | noFile: noFile, 46 | minSize: minSize, 47 | chunkMaxSize: chunkMaxSize, 48 | writeConcurrency: writeConcurrency, 49 | manifest: NewManifest(chunkMaxSize), 50 | } 51 | } 52 | 53 | func (d *gptDiskChunker) Dedup() (*manifest, error) { 54 | statusf("Detected GPT disk ...\n") 55 | 56 | if err := d.dedupNtfsPartitions(); err != nil { 57 | return nil, err 58 | } 59 | 60 | if err := d.dedupRest(); err != nil { 61 | return nil, err 62 | } 63 | 64 | statusf("GPT disk fully indexed\n") 65 | return d.manifest, nil 66 | } 67 | 68 | func (d *gptDiskChunker) dedupNtfsPartitions() error { 69 | buffer := make([]byte, gptHeaderLength) 70 | _, err := d.reader.ReadAt(buffer, d.start + gptHeaderOffset) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | // Read basic information, then re-read buffer 76 | firstEntrySector := parseUintLE(buffer, gptFirstEntrySectorOffset, gptFirstEntrySectorLength) 77 | entryCount := parseUintLE(buffer, gptEntryCountOffset, gptEntryCountLength) 78 | entrySize := parseUintLE(buffer, gptEntrySizeOffset, gptEntrySizeLength) 79 | 80 | buffer = make([]byte, entryCount * entrySize) 81 | _, err = d.reader.ReadAt(buffer, d.start + firstEntrySector * gptLogicalSectorSize) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | // Walk the entries, index partitions if supported 87 | for i := int64(0); i < entryCount; i++ { 88 | entryOffset := i * entrySize 89 | 90 | partitionFirstSector := parseUintLE(buffer, entryOffset+gptEntryFirstSectorRelativeOffset, gptEntryFirstSectorRelativeLength) 91 | partitionOffset := d.start + partitionFirstSector*gptLogicalSectorSize 92 | debugf("Reading GPT entry %d, partition begins at sector %d, offset %d\n", 93 | i+1, partitionFirstSector, partitionOffset) 94 | 95 | if partitionOffset == 0 { 96 | continue 97 | } 98 | 99 | partitionType, err := probeType(d.reader, partitionOffset) // TODO fix global func call 100 | if err != nil { 101 | continue 102 | } 103 | 104 | if partitionType == typeNtfs { 105 | debugf("NTFS partition found at offset %d\n", partitionOffset) 106 | ntfs := NewNtfsChunker(d.reader, d.store, partitionOffset, d.exact, d.noFile, d.minSize, d.chunkMaxSize, d.writeConcurrency) 107 | manifest, err := ntfs.Dedup() 108 | if err != nil { 109 | return err 110 | } 111 | 112 | d.manifest.MergeAtOffset(partitionOffset, manifest) 113 | } 114 | } 115 | 116 | return nil 117 | } 118 | 119 | func (d *gptDiskChunker) dedupRest() error { 120 | chunker := NewFixedChunkerWithSkip(d.reader, d.store, d.start, d.size, d.chunkMaxSize, d.writeConcurrency, d.manifest) 121 | 122 | gapManifest, err := chunker.Dedup() 123 | if err != nil { 124 | return err 125 | } 126 | 127 | d.manifest.Merge(gapManifest) 128 | return nil 129 | } 130 | -------------------------------------------------------------------------------- /chunker_mbr.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | const ( 8 | mbrLength = 512 9 | mbrSectorSize = 512 10 | mbrSignatureOffset = 510 11 | mbrSignatureLength = 2 12 | mbrSignatureMagic = 0xaa55 // 0x55aa as little endian 13 | mbrEntryCount = 4 14 | mbrEntryFirstOffset = 446 15 | mbrEntryLength = 16 16 | mbrFirstSectorRelativeOffset = 8 17 | mbrEntryFirstSectorRelativeLength = 4 18 | ) 19 | 20 | type mbrDiskChunker struct { 21 | reader io.ReaderAt 22 | store ChunkStore 23 | start int64 24 | size int64 25 | exact bool 26 | noFile bool 27 | minSize int64 28 | chunkMaxSize int64 29 | writeConcurrency int64 30 | manifest *manifest 31 | } 32 | 33 | func NewMbrDiskChunker(reader io.ReaderAt, store ChunkStore, offset int64, size int64, exact bool, noFile bool, 34 | minSize int64, chunkMaxSize int64, writeConcurrency int64) *mbrDiskChunker { 35 | 36 | return &mbrDiskChunker{ 37 | reader: reader, 38 | store: store, 39 | start: offset, 40 | size: size, 41 | exact: exact, 42 | noFile: noFile, 43 | minSize: minSize, 44 | chunkMaxSize: chunkMaxSize, 45 | writeConcurrency: writeConcurrency, 46 | manifest: NewManifest(chunkMaxSize), 47 | } 48 | } 49 | 50 | func (d *mbrDiskChunker) Dedup() (*manifest, error) { 51 | statusf("Detected MBR disk\n") 52 | 53 | if err := d.dedupNtfsPartitions(); err != nil { 54 | return nil, err 55 | } 56 | 57 | if err := d.dedupRest(); err != nil { 58 | return nil, err 59 | } 60 | 61 | statusf("MBR disk fully indexed\n") 62 | return d.manifest, nil 63 | } 64 | 65 | func (d *mbrDiskChunker) dedupNtfsPartitions() error { 66 | buffer := make([]byte, mbrLength) 67 | _, err := d.reader.ReadAt(buffer, d.start) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | for i := int64(0); i < mbrEntryCount; i++ { 73 | entryOffset := mbrEntryFirstOffset + i * mbrEntryLength 74 | 75 | partitionFirstSector := parseUintLE(buffer, entryOffset+mbrFirstSectorRelativeOffset, mbrEntryFirstSectorRelativeLength) 76 | partitionOffset := d.start + partitionFirstSector*mbrSectorSize 77 | debugf("Reading MBR entry at %d, partition begins at sector %d, offset %d\n", 78 | entryOffset, partitionFirstSector, partitionOffset) 79 | 80 | if partitionOffset == 0 { 81 | continue 82 | } 83 | 84 | partitionType, err := probeType(d.reader, partitionOffset) // TODO fix global func call 85 | if err != nil { 86 | continue 87 | } 88 | 89 | if partitionType == typeNtfs { 90 | ntfs := NewNtfsChunker(d.reader, d.store, partitionOffset, d.exact, d.noFile, d.minSize, d.chunkMaxSize, d.writeConcurrency) 91 | manifest, err := ntfs.Dedup() 92 | if err != nil { 93 | return err 94 | } 95 | 96 | d.manifest.MergeAtOffset(partitionOffset, manifest) 97 | } 98 | } 99 | 100 | return nil 101 | } 102 | 103 | func (d *mbrDiskChunker) dedupRest() error { 104 | chunker := NewFixedChunkerWithSkip(d.reader, d.store, d.start, d.size, d.chunkMaxSize, d.writeConcurrency, d.manifest) 105 | 106 | gapManifest, err := chunker.Dedup() 107 | if err != nil { 108 | return err 109 | } 110 | 111 | d.manifest.Merge(gapManifest) 112 | return nil 113 | } 114 | -------------------------------------------------------------------------------- /chunker_ntfs.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io" 7 | ) 8 | 9 | // ntfsChunker reads and deduplicates files within the NTFS 10 | // file system. It implements the chunker interface. 11 | // 12 | // Assuming the input is an NTFS partition, it works in 13 | // three passes: 14 | // 15 | // Pass 1: Find FILE entries > $minSize according to $MFT (F): 16 | // _____________________________________________ 17 | // |__|FFF|_____|FFF|_______|FF|_________|FF|____| 18 | // 19 | // Pass 2: Find unused sections according to $Bitmap and mark as sparse (S): 20 | // _____________________________________________ 21 | // |__|FFF|_|SS||FFF|__|SS|_|FF|___|SSSSS|FF|____| 22 | // 23 | // Pass 3: Find gaps: 24 | // _____________________________________________ 25 | // |GG|FFF|G|SS||FFF|GG|SS|G|FF|GGG|SSSSS|FF|GGGG| 26 | // 27 | // 28 | // Glossary: 29 | // run: mft data run, pointing to chunkParts with data on disk 30 | // chunk: data blob, containing data from one or many runs 31 | // chunk part: pointer to one or many parts of a chunk; a run's data can be spread across multiple chunks 32 | // 33 | type ntfsChunker struct { 34 | reader io.ReaderAt 35 | start int64 36 | sizeInBytes int64 37 | exact bool 38 | noFile bool 39 | minSize int64 40 | chunkMaxSize int64 41 | writeConcurrency int64 42 | 43 | totalSectors int64 44 | sectorSize int64 45 | sectorsPerCluster int64 46 | clusterSize int64 47 | store ChunkStore 48 | buffer []byte // cannot be larger than DefaultChunkSizeMaxBytes, the logic relies on it! 49 | chunk *chunk 50 | 51 | // Output manifest 52 | out *manifest 53 | } 54 | 55 | type entry struct { 56 | offset int64 57 | resident bool 58 | data bool 59 | inuse bool 60 | allocatedSize int64 61 | dataSize int64 62 | runs []run 63 | } 64 | 65 | type run struct { 66 | sparse bool 67 | fromOffset int64 68 | toOffset int64 69 | firstCluster int64 // signed! 70 | clusterCount int64 71 | size int64 72 | } 73 | 74 | const ( 75 | // NTFS boot sector (absolute aka relative to file system start) 76 | ntfsBootRecordSize = 512 77 | ntfsBootMagicOffset = 3 78 | ntfsBootMagic = "NTFS " 79 | ntfsBootSectorSizeOffset = 11 80 | ntfsBootSectorSizeLength = 2 81 | ntfsBootSectorsPerClusterOffset = 13 82 | ntfsBootSectorsPerClusterLength = 1 83 | ntfsBootTotalSectorsOffset = 40 84 | ntfsBootTotalSectorsLength = 8 85 | ntfsBootMftClusterNumberOffset = 48 86 | ntfsBootMftClusterNumberLength = 8 87 | 88 | // FILE entry (relative to FILE offset) 89 | // https://flatcap.org/linux-ntfs/ntfs/concepts/file_record.html 90 | ntfsEntryMagic = "FILE" 91 | ntfsEntryTypicalSize = 1024 92 | ntfsEntryUpdateSequenceOffsetOffset = 4 93 | ntfsEntryUpdateSequenceOffsetLength = 2 94 | ntfsEntryUpdateSequenceSizeOffset = 6 95 | ntfsEntryUpdateSequenceSizeLength = 2 96 | ntfsEntryUpdateSequenceNumberLength = 2 97 | ntfsEntryAllocatedSizeOffset = 28 98 | ntfsEntryAllocatedSizeLength = 4 99 | ntfsEntryFirstAttrOffset = 20 100 | ntfsEntryFirstAttrLength = 2 101 | ntfsEntryFlagsOffset = 22 102 | ntfsEntryFlagsLength = 2 103 | ntfsEntryFlagInUse = 1 104 | 105 | // $Bitmap file 106 | ntfsBitmapFileEntryIndex = 6 107 | ntfsBitmapMinSparseClusters = 0 // arbitrary number 108 | ntfsBitsPerByte = 8 109 | 110 | // Attribute header / footer (relative to attribute offset) 111 | // https://flatcap.org/linux-ntfs/ntfs/concepts/attribute_header.html 112 | ntfsAttrTypeOffset = 0 113 | ntfsAttrTypeLength = 4 114 | ntfsAttrLengthOffset = 4 115 | ntfsAttrLengthLength = 4 116 | ntfsAttrTypeData = 0x80 117 | ntfsAttrTypeEndMarker = -1 // 0xFFFFFFFF 118 | 119 | // Attribute 0x80 / $DATA (relative to attribute offset) 120 | // https://flatcap.org/linux-ntfs/ntfs/attributes/data.html 121 | ntfsAttrDataResidentOffset = 8 122 | ntfsAttrDataResidentLength = 1 123 | ntfsAttrDataRealSizeOffset = 48 124 | ntfsAttrDataRealSizeLength = 4 125 | ntfsAttrDataRunsOffset = 32 126 | ntfsAttrDataRunsLength = 2 127 | 128 | // Data run structure within $DATA attribute 129 | // https://flatcap.org/linux-ntfs/ntfs/concepts/data_runs.html 130 | ntfsAttrDataRunsHeaderOffset = 0 131 | ntfsAttrDataRunsHeaderLength = 1 132 | ntfsAttrDataRunsHeaderEndMarker = 0 133 | ) 134 | 135 | var ErrUnexpectedMagic = errors.New("unexpected magic") 136 | 137 | func NewNtfsChunker(reader io.ReaderAt, store ChunkStore, offset int64, exact bool, noFile bool, 138 | minSize int64, chunkMaxSize int64, writeConcurrency int64) *ntfsChunker { 139 | 140 | return &ntfsChunker{ 141 | reader: reader, 142 | store: store, 143 | start: offset, 144 | exact: exact, 145 | noFile: noFile, 146 | minSize: minSize, 147 | chunkMaxSize: chunkMaxSize, 148 | writeConcurrency: writeConcurrency, 149 | chunk: NewChunk(chunkMaxSize), 150 | buffer: make([]byte, chunkMaxSize), 151 | out: NewManifest(chunkMaxSize), 152 | } 153 | } 154 | 155 | func (d *ntfsChunker) Dedup() (*manifest, error) { 156 | // Read NTFS boot sector 157 | boot := make([]byte, ntfsBootRecordSize) 158 | _, err := d.reader.ReadAt(boot, d.start) 159 | if err != nil { 160 | return nil, err 161 | } 162 | 163 | // Read magic string to ensure this is an NTFS filesystem 164 | if bytes.Compare([]byte(ntfsBootMagic), boot[ntfsBootMagicOffset:ntfsBootMagicOffset+len(ntfsBootMagic)]) != 0 { 165 | return nil, errors.New("invalid boot sector, invalid magic") 166 | } 167 | 168 | // Read basic information 169 | d.sectorSize = parseUintLE(boot, ntfsBootSectorSizeOffset, ntfsBootSectorSizeLength) 170 | d.sectorsPerCluster = parseUintLE(boot, ntfsBootSectorsPerClusterOffset, ntfsBootSectorsPerClusterLength) 171 | d.totalSectors = parseUintLE(boot, ntfsBootTotalSectorsOffset, ntfsBootTotalSectorsLength) 172 | d.clusterSize = d.sectorSize * d.sectorsPerCluster 173 | d.sizeInBytes = d.sectorSize * d.totalSectors + d.sectorSize // Backup boot sector at the end! 174 | 175 | statusf("NTFS partition of size %s found at offset %d\n", convertBytesToHumanReadable(d.sizeInBytes), d.start) 176 | 177 | // Read $MFT entry 178 | mftClusterNumber := parseUintLE(boot, ntfsBootMftClusterNumberOffset, ntfsBootMftClusterNumberLength) 179 | mftOffset := mftClusterNumber * d.clusterSize 180 | 181 | debugf("sector size = %d, sectors per cluster = %d, cluster size = %d, mft cluster number = %d, mft offset = %d\n", 182 | d.sectorSize, d.sectorsPerCluster, d.clusterSize, mftClusterNumber, mftOffset) 183 | 184 | mft, err := d.readEntry(mftOffset) 185 | if err != nil { 186 | return nil, err 187 | } 188 | 189 | // Find and checksum FILE entries 190 | if !d.noFile { 191 | if err := d.dedupFiles(mft); err != nil { 192 | return nil, err 193 | } 194 | } 195 | 196 | // Find unused/empty sections based on the $Bitmap 197 | if !d.exact { 198 | if err := d.dedupUnused(mft); err != nil { 199 | return nil, err 200 | } 201 | } 202 | 203 | // Dedup the rest (gap areas) 204 | if err := d.dedupGaps(); err != nil { 205 | return nil, err 206 | } 207 | 208 | statusf("NTFS partition successfully indexed\n") 209 | 210 | return d.out, nil 211 | } 212 | 213 | func (d *ntfsChunker) readEntry(offset int64) (*entry, error) { 214 | err := readAndCompare(d.reader, d.start + offset, []byte(ntfsEntryMagic)) 215 | if err != nil { 216 | return nil, err 217 | } 218 | 219 | // Read full entry into buffer (we're guessing size 1024 here) 220 | buffer := make([]byte, ntfsEntryTypicalSize) 221 | _, err = d.reader.ReadAt(buffer, d.start + offset) 222 | if err != nil { 223 | return nil, err 224 | } 225 | 226 | // Read entry length, and re-read the buffer if it differs 227 | allocatedSize := parseUintLE(buffer, ntfsEntryAllocatedSizeOffset, ntfsEntryAllocatedSizeLength) 228 | if int(allocatedSize) > len(buffer) { 229 | buffer = make([]byte, allocatedSize) 230 | _, err = d.reader.ReadAt(buffer, d.start + offset) 231 | if err != nil { 232 | return nil, err 233 | } 234 | } 235 | 236 | // Apply fix-up 237 | // see https://flatcap.org/linux-ntfs/ntfs/concepts/fixup.html 238 | updateSequenceOffset := parseIntLE(buffer, ntfsEntryUpdateSequenceOffsetOffset, ntfsEntryUpdateSequenceOffsetLength) 239 | updateSequenceSizeInWords := parseIntLE(buffer, ntfsEntryUpdateSequenceSizeOffset, ntfsEntryUpdateSequenceSizeLength) 240 | updateSequenceArrayOffset := updateSequenceOffset + ntfsEntryUpdateSequenceNumberLength 241 | updateSequenceArrayLength := 2*updateSequenceSizeInWords - 2 // see https://flatcap.org/linux-ntfs/ntfs/concepts/file_record.html 242 | updateSequenceArray := buffer[updateSequenceArrayOffset:updateSequenceArrayOffset+updateSequenceArrayLength] 243 | 244 | for i := int64(0); i < updateSequenceArrayLength/2; i++ { 245 | buffer[(i+1)*d.sectorSize-2] = updateSequenceArray[i] 246 | buffer[(i+1)*d.sectorSize-1] = updateSequenceArray[i+1] 247 | } 248 | 249 | entry := &entry{ 250 | offset: offset, 251 | resident: false, 252 | data: false, 253 | inuse: false, 254 | allocatedSize: allocatedSize, 255 | } 256 | 257 | // Read flags 258 | flags := parseUintLE(buffer, ntfsEntryFlagsOffset, ntfsEntryFlagsLength) 259 | entry.inuse = flags & ntfsEntryFlagInUse == ntfsEntryFlagInUse 260 | 261 | if !entry.inuse { 262 | return entry, nil 263 | } 264 | 265 | // Read attributes 266 | relativeFirstAttrOffset := parseUintLE(buffer, ntfsEntryFirstAttrOffset, ntfsEntryFirstAttrLength) 267 | firstAttrOffset := relativeFirstAttrOffset 268 | attrOffset := firstAttrOffset 269 | 270 | for { 271 | attrType := parseIntLE(buffer, attrOffset + ntfsAttrTypeOffset, ntfsAttrTypeLength) 272 | 273 | if attrType == ntfsAttrTypeEndMarker { 274 | break 275 | } 276 | 277 | attrLen := parseUintLE(buffer, attrOffset + ntfsAttrLengthOffset, ntfsAttrLengthLength) 278 | if attrLen == 0 { // FIXME this should really never happen! 279 | break 280 | } 281 | 282 | if attrType == ntfsAttrTypeData { 283 | nonResident := parseUintLE(buffer, attrOffset + ntfsAttrDataResidentOffset, ntfsAttrDataResidentLength) 284 | entry.resident = nonResident == 0 285 | 286 | if !entry.resident { 287 | dataRealSize := parseUintLE(buffer, attrOffset + ntfsAttrDataRealSizeOffset, ntfsAttrDataRealSizeLength) 288 | 289 | relativeDataRunsOffset := parseIntLE(buffer, attrOffset + ntfsAttrDataRunsOffset, ntfsAttrDataRunsLength) 290 | dataRunFirstOffset := attrOffset + int64(relativeDataRunsOffset) 291 | 292 | entry.dataSize = dataRealSize 293 | entry.runs = d.readRuns(buffer, dataRunFirstOffset) 294 | } 295 | } 296 | 297 | attrOffset += attrLen 298 | } 299 | 300 | entry.data = entry.runs != nil 301 | return entry, nil 302 | } 303 | 304 | func (d *ntfsChunker) dedupFiles(mft *entry) error { 305 | statusf("Reading NTFS $MFT ...") 306 | 307 | processedEntries := int64(0) 308 | dedupedEntries := int64(0) 309 | totalFileSize := int64(0) 310 | maxEntries := int64(0) 311 | 312 | for _, run := range mft.runs { 313 | maxEntries += run.clusterCount * d.sectorsPerCluster * d.sectorSize / mft.allocatedSize 314 | } 315 | 316 | for _, run := range mft.runs { 317 | debugf("Reading run (from = %d, to = %d, clusters = %d, size = %d, sparse = %t)\n", 318 | run.fromOffset, run.toOffset, run.clusterCount, run.size, run.sparse) 319 | 320 | startOffset := run.firstCluster * d.sectorsPerCluster * d.sectorSize 321 | endOffset := startOffset + run.clusterCount * d.sectorsPerCluster * d.sectorSize 322 | 323 | offset := startOffset + mft.allocatedSize // Skip $MFT entry itself! 324 | 325 | for offset < endOffset { 326 | processedEntries++ 327 | 328 | entry, err := d.readEntry(offset) 329 | if err == ErrUnexpectedMagic { 330 | offset += d.sectorSize 331 | debugf("Entry at offset %d cannot be read: %s\n", offset, err.Error()) 332 | continue 333 | } else if err != nil { 334 | return err 335 | } 336 | 337 | if !entry.inuse { 338 | offset += entry.allocatedSize 339 | debugf("Entry at offset %d ignored: deleted file\n", offset) 340 | continue 341 | } 342 | 343 | if !entry.data { 344 | offset += entry.allocatedSize 345 | debugf("Entry at offset %d ignored: no data attribute\n", offset) 346 | continue 347 | } 348 | 349 | if entry.resident { 350 | offset += entry.allocatedSize 351 | debugf("Entry at offset %d ignored: data is resident\n", offset) 352 | continue 353 | } 354 | 355 | if entry.dataSize < d.minSize { 356 | offset += entry.allocatedSize 357 | debugf("Entry at offset %d skipped: %d byte(s) is too small\n", offset, entry.dataSize) 358 | continue 359 | } 360 | 361 | statusf("Indexing file entry %d/%d (%s / %d indexed, %d skipped) ...", 362 | processedEntries, maxEntries, convertBytesToHumanReadable(totalFileSize), dedupedEntries, processedEntries - dedupedEntries) 363 | 364 | bytesIndexed, err := d.dedupFile(entry) 365 | if err != nil { 366 | offset += entry.allocatedSize 367 | debugf("Entry at offset %d failed to be deduped:\n", offset, err.Error()) 368 | continue 369 | } 370 | 371 | debugf("Entry at offset %d successfully indexed\n", offset) 372 | 373 | offset += entry.allocatedSize 374 | totalFileSize += bytesIndexed 375 | dedupedEntries++ 376 | } 377 | } 378 | 379 | statusf("Indexed %s in %d file(s) (%d skipped)\n", 380 | convertBytesToHumanReadable(totalFileSize), dedupedEntries, processedEntries - dedupedEntries) 381 | 382 | return nil 383 | } 384 | 385 | func (d *ntfsChunker) readRuns(entry []byte, offset int64) []run { 386 | runs := make([]run, 0) 387 | firstCluster := int64(0) 388 | 389 | for { 390 | header := uint64(parseIntLE(entry, offset + ntfsAttrDataRunsHeaderOffset, ntfsAttrDataRunsHeaderLength)) 391 | 392 | if header == ntfsAttrDataRunsHeaderEndMarker { 393 | break 394 | } 395 | 396 | clusterCountLength := int64(header & 0x0F) // right nibble 397 | clusterCountOffset := offset + ntfsAttrDataRunsHeaderLength 398 | clusterCount := parseUintLE(entry, clusterCountOffset, clusterCountLength) 399 | 400 | firstClusterLength := int64(header & 0xF0 >> 4) // left nibble 401 | firstClusterOffset := clusterCountOffset + clusterCountLength 402 | firstCluster += parseIntLE(entry, firstClusterOffset, firstClusterLength) // relative to previous, can be negative, so signed! 403 | 404 | sparse := firstClusterLength == 0 405 | 406 | fromOffset := int64(firstCluster) * int64(d.clusterSize) 407 | toOffset := fromOffset + int64(clusterCount) * int64(d.clusterSize) 408 | fullRun := entry[offset+ntfsAttrDataRunsHeaderOffset:offset+ntfsAttrDataRunsHeaderOffset+clusterCountLength+firstClusterLength+1] 409 | 410 | debugf("data run offset = %d, header = 0x%x, full run = %x, sparse = %t, to to = 0x%x, offset to = 0x%x, " + 411 | "cluster count = %d, first cluster = %d, from offset = %d, to offset = %d\n", 412 | offset, header, fullRun, sparse, clusterCountLength, firstClusterLength, clusterCount, firstCluster, 413 | fromOffset, toOffset) 414 | 415 | runs = append(runs, run{ 416 | sparse: sparse, 417 | firstCluster: firstCluster, 418 | clusterCount: clusterCount, 419 | fromOffset: fromOffset, 420 | toOffset: toOffset, 421 | size: toOffset - fromOffset, 422 | }) 423 | 424 | offset += firstClusterLength + clusterCountLength + 1 425 | } 426 | 427 | return runs 428 | } 429 | 430 | func (d *ntfsChunker) dedupFile(entry *entry) (int64, error) { 431 | remainingToEndOfFile := entry.dataSize 432 | bytesIndexed := int64(0) 433 | 434 | slices := make(map[int64]*chunkSlice, 0) 435 | d.chunk.Reset() 436 | 437 | for _, run := range entry.runs { 438 | debugf("- Processing run at cluster %d, offset %d, cluster count = %d, size = %d\n", 439 | run.firstCluster, run.fromOffset, run.clusterCount, run.size) 440 | 441 | if run.sparse { 442 | debugf("- Sparse run, skipping %d bytes\n", d.clusterSize * run.clusterCount) 443 | remainingToEndOfFile -= d.clusterSize * run.clusterCount 444 | } else { 445 | runOffset := run.fromOffset 446 | runSize := minInt64(remainingToEndOfFile, run.size) // only read to filesize, doesnt always align with clusters! 447 | 448 | bytesIndexed += runSize 449 | remainingToEndOfFile -= runSize 450 | remainingToEndOfRun := runSize 451 | 452 | for remainingToEndOfRun > 0 { 453 | remainingToFullChunk := d.chunk.Remaining() 454 | runBytesMaxToBeRead := minInt64(minInt64(remainingToEndOfRun, remainingToFullChunk), int64(len(d.buffer))) 455 | 456 | debugf("- Reading disk section at offset %d to max %d bytes (remaining to end of run = %d, remaining to full chunk = %d, run buffer size = %d)\n", 457 | runOffset, runBytesMaxToBeRead, remainingToEndOfRun, remainingToFullChunk, len(d.buffer)) 458 | 459 | runBytesRead, err := d.reader.ReadAt(d.buffer[:runBytesMaxToBeRead], d.start + runOffset) 460 | if err != nil { 461 | return 0, err 462 | } 463 | 464 | // Add run to chunk(s) 465 | debugf("- Bytes read = %d, current chunk size = %d, chunk max = %d\n", 466 | runBytesRead, d.chunk.Size(), d.chunkMaxSize) 467 | 468 | slices[runOffset] = &chunkSlice{ 469 | checksum: nil, // fill this when chunk is finalized! 470 | kind: kindFile, 471 | diskfrom: runOffset, 472 | diskto: runOffset + int64(runBytesRead), 473 | chunkfrom: d.chunk.Size(), 474 | chunkto: d.chunk.Size() + int64(runBytesRead), 475 | length: int64(runBytesRead), 476 | } 477 | 478 | d.chunk.Write(d.buffer[:runBytesRead]) 479 | 480 | debugf("- Adding %d bytes to chunk, new chunk size is %d\n", runBytesRead, d.chunk.Size()) 481 | 482 | // Emit full chunk, write file and add to chunk map 483 | if d.chunk.Full() { 484 | debugf("- Chunk full. Emitting chunk %x, size = %d\n", d.chunk.Checksum(), d.chunk.Size()) 485 | 486 | // Add slices to disk map 487 | for sliceOffset, slice := range slices { 488 | slice.checksum = d.chunk.Checksum() 489 | debugf("- Adding disk section %d - %d, mapping to chunk %x, offset %d - %d\n", 490 | sliceOffset, sliceOffset+ slice.chunkto- slice.chunkfrom, slice.checksum, slice.chunkfrom, slice.chunkto) 491 | d.out.Add(slice) 492 | } 493 | 494 | slices = make(map[int64]*chunkSlice, 0) // clear! 495 | 496 | // Write chunk 497 | if err := d.store.Write(d.chunk.Checksum(), d.chunk.Data()); err != nil { 498 | return 0, err 499 | } 500 | 501 | d.chunk.Reset() 502 | } 503 | 504 | remainingToEndOfRun -= int64(runBytesRead) 505 | runOffset += int64(runBytesRead) 506 | } 507 | 508 | // Add sparse section for files that are not cluster-aligned (most files!) 509 | if !d.exact { 510 | if runOffset % d.sectorSize != 0 { 511 | remainingToEndOfCluster := d.sectorSize - runOffset % d.sectorSize 512 | debugf("- File end is not cluster aligned, emitting sparse section %d - %d\n", 513 | runOffset, runOffset + remainingToEndOfCluster) 514 | 515 | d.out.Add(&chunkSlice{ 516 | checksum: nil, 517 | kind: kindSparse, 518 | diskfrom: runOffset, 519 | diskto: runOffset + remainingToEndOfCluster, 520 | chunkfrom: 0, 521 | chunkto: remainingToEndOfCluster, 522 | length: remainingToEndOfCluster, 523 | }) 524 | } 525 | } 526 | } 527 | } 528 | 529 | // Finish last chunk 530 | if d.chunk.Size() > 0 { 531 | // Add slices to disk map 532 | for sliceOffset, slice := range slices { 533 | slice.checksum = d.chunk.Checksum() 534 | debugf("- Adding disk section %d - %d, mapping to chunk %x, offset %d - %d\n", 535 | sliceOffset, sliceOffset+ slice.chunkto- slice.chunkfrom, slice.checksum, slice.chunkfrom, slice.chunkto) 536 | d.out.Add(slice) 537 | } 538 | 539 | debugf("- End of file. Emitting last chunk %x, size = %d\n", d.chunk.Checksum(), d.chunk.Size()) 540 | if err := d.store.Write(d.chunk.Checksum(), d.chunk.Data()); err != nil { 541 | return 0, err 542 | } 543 | } 544 | 545 | return bytesIndexed, nil 546 | } 547 | 548 | // dedupUnused reads the NTFS $Bitmap file to find unused clusters and 549 | // creates sparse entry in the manifest for them. 550 | // 551 | // The logic is a little simplified right now, as it treats the bit-map 552 | // as a byte-map, only looking at 8 empty clusters in a row (= 8 bits, 1 byte). 553 | func (d *ntfsChunker) dedupUnused(mft *entry) error { 554 | statusf("Indexing unused space ...") 555 | 556 | // Find $Bitmap entry 557 | var err error 558 | bitmap := mft 559 | 560 | for i := 0; i < ntfsBitmapFileEntryIndex; i++ { 561 | debugf("reading entry %d\n", bitmap.offset + bitmap.allocatedSize) 562 | bitmap, err = d.readEntry(bitmap.offset + bitmap.allocatedSize) 563 | if err != nil { 564 | return err 565 | } 566 | } 567 | 568 | // FIXME This relies solely on the offset. It does not verify that 569 | // what we have found is in fact the $Bitmap! 570 | 571 | // Read $Bitmap 572 | debugf("$Bitmap is at offset %d\n", bitmap.offset) 573 | 574 | sparseBytes := int64(0) 575 | 576 | remainingToEndOfFile := bitmap.dataSize 577 | buffer := make([]byte, d.clusterSize) 578 | 579 | lastWasZero := false 580 | cluster := int64(0) 581 | sparseClusterGroupStart := int64(0) 582 | sparseClusterGroupEnd := int64(0) 583 | 584 | for _, run := range bitmap.runs { 585 | debugf(" - Processing run at cluster %d, offset %d, cluster count = %d, size = %d\n", 586 | run.firstCluster, run.fromOffset, run.clusterCount, run.size) 587 | 588 | runOffset := run.fromOffset 589 | runSize := minInt64(remainingToEndOfFile, run.size) // only read to filesize, doesnt always align with clusters! 590 | 591 | remainingToEndOfFile -= runSize 592 | remainingToEndOfRun := runSize 593 | 594 | for remainingToEndOfRun > 0 { 595 | runBytesMaxToBeRead := minInt64(remainingToEndOfRun, int64(len(buffer))) 596 | 597 | debugf(" -> Reading disk section at offset %d to max %d bytes (remaining to end of run = %d, run buffer size = %d)\n", 598 | runOffset, runBytesMaxToBeRead, remainingToEndOfRun, len(buffer)) 599 | 600 | runBytesRead, err := d.reader.ReadAt(buffer[:runBytesMaxToBeRead], d.start + runOffset) 601 | if err != nil { 602 | return err 603 | } 604 | 605 | for i := 0; i < runBytesRead; i++ { 606 | if buffer[i] == 0 { 607 | if lastWasZero { 608 | sparseClusterGroupEnd = cluster 609 | } else { 610 | lastWasZero = true 611 | sparseClusterGroupStart = cluster 612 | sparseClusterGroupEnd = cluster 613 | } 614 | } else { 615 | if lastWasZero { 616 | lastWasZero = false 617 | isLargeEnough := (sparseClusterGroupEnd-sparseClusterGroupStart)*ntfsBitsPerByte > ntfsBitmapMinSparseClusters 618 | 619 | if isLargeEnough { 620 | sparseSectionStartOffset := sparseClusterGroupStart * d.clusterSize * ntfsBitsPerByte 621 | sparseSectionEndOffset := sparseClusterGroupEnd * d.clusterSize * ntfsBitsPerByte 622 | sparseSectionLength := sparseSectionEndOffset - sparseSectionStartOffset 623 | 624 | debugf("- Detected large sparse section %d - %d (%d bytes)\n", 625 | sparseSectionStartOffset, sparseSectionEndOffset, sparseSectionLength) 626 | 627 | d.out.Add(&chunkSlice{ 628 | checksum: nil, 629 | kind: kindSparse, 630 | diskfrom: sparseSectionStartOffset, 631 | diskto: sparseSectionEndOffset, 632 | chunkfrom: 0, 633 | chunkto: sparseSectionLength, 634 | length: sparseSectionLength, 635 | }) 636 | 637 | sparseBytes += sparseSectionLength 638 | statusf("Finding unused space via $Bitmap (%s marked sparse) ...", convertBytesToHumanReadable(sparseBytes)) 639 | } 640 | } 641 | } 642 | 643 | cluster++ 644 | } 645 | 646 | remainingToEndOfRun -= int64(runBytesRead) 647 | runOffset += int64(runBytesRead) 648 | } 649 | } 650 | 651 | statusf("Indexed %s of unused space\n", convertBytesToHumanReadable(sparseBytes)) 652 | 653 | return nil 654 | } 655 | 656 | func (d *ntfsChunker) dedupGaps() error { 657 | chunker := NewFixedChunkerWithSkip(d.reader, d.store, d.start, d.sizeInBytes, d.chunkMaxSize, d.writeConcurrency, d.out) 658 | 659 | gapManifest, err := chunker.Dedup() 660 | if err != nil { 661 | return err 662 | } 663 | 664 | d.out.Merge(gapManifest) 665 | return nil 666 | } 667 | -------------------------------------------------------------------------------- /cmd/fsdup/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "github.com/ncw/swift" 8 | "heckel.io/fsdup" 9 | "net/url" 10 | "os" 11 | "regexp" 12 | "strconv" 13 | "strings" 14 | "time" 15 | ) 16 | 17 | // TODO [HIGH] "map": Make caching efficient 18 | // TODO [LOW] Find zeros in gaps, mark as sparse 19 | // TODO [LOW] Find zeros in FILE runs, mark as sparse 20 | // TODO [LOW] rename "size" to "length" 21 | // TODO [LOW] chunkPart.to|from -> offset|length 22 | // TODO [LOW] different debug levels -d -dd -ddd -q 23 | 24 | var ( 25 | buildversion = "0.0.0-DEV" 26 | builddate = "0" 27 | buildcommit = "dev version" 28 | ) 29 | 30 | func main() { 31 | versionFlag := flag.Bool("version", false, "Displays the version of this program") 32 | debugFlag := flag.Bool("debug", false, "Enable debug information") 33 | quietFlag := flag.Bool("quiet", false, "Do not print status information") 34 | 35 | flag.Parse() 36 | 37 | fsdup.Debug = *debugFlag 38 | fsdup.Quiet = *quietFlag 39 | 40 | if *versionFlag { 41 | displayVersion() 42 | } 43 | 44 | if flag.NArg() < 2 { 45 | usage() 46 | } 47 | 48 | command := flag.Args()[0] 49 | args := flag.Args()[1:] 50 | 51 | switch command { 52 | case "index": 53 | indexCommand(args) 54 | case "map": 55 | mapCommand(args) 56 | case "export": 57 | exportCommand(args) 58 | case "import": 59 | importCommand(args) 60 | case "print": 61 | printCommand(args) 62 | case "stat": 63 | statCommand(args) 64 | case "server": 65 | serverCommand(args) 66 | case "upload": 67 | uploadCommand(args) 68 | default: 69 | usage() 70 | } 71 | } 72 | 73 | func exit(code int, message string) { 74 | fmt.Println(message) 75 | os.Exit(code) 76 | } 77 | 78 | func usage() { 79 | fmt.Println("Syntax:") 80 | fmt.Println(" fsdup [-quiet] [-debug] COMMAND [options ...]") 81 | fmt.Println("") 82 | fmt.Println("Commands:") 83 | fmt.Println(" index [-nowrite] [-store STORE] [-offset OFFSET] [-minsize MINSIZE] [-exact] [-nofile] INFILE MANIFEST") 84 | fmt.Println(" import [-store STORE] INFILE MANIFEST") 85 | fmt.Println(" export [-store STORE] MANIFEST OUTFILE") 86 | fmt.Println(" upload [-server ADDR:PORT] INFILE MANIFEST") 87 | fmt.Println(" map [-store STORE] [-cache CACHE] MANIFEST OUTFILE") 88 | fmt.Println(" print [disk|chunks] MANIFEST") 89 | fmt.Println(" server [-store STORE] [ADDR]:PORT") 90 | fmt.Println(" stat MANIFEST...") 91 | 92 | os.Exit(1) 93 | } 94 | 95 | func displayVersion() { 96 | unixtime, _ := strconv.Atoi(builddate) 97 | datestr := time.Unix(int64(unixtime), 0).Format("01/02/06 15:04") 98 | 99 | fmt.Printf("fsdup version %s, built at %s from %s\n", buildversion, datestr, buildcommit) 100 | fmt.Printf("Distributed under the Apache License 2.0, see LICENSE file for details\n") 101 | fmt.Printf("Copyright (C) 2019 Philipp Heckel\n") 102 | 103 | os.Exit(1) 104 | } 105 | 106 | func indexCommand(args []string) { 107 | flags := flag.NewFlagSet("index", flag.ExitOnError) 108 | debugFlag := flags.Bool("debug", fsdup.Debug, "Enable debug mode") 109 | noWriteFlag := flags.Bool("nowrite", false, "Do not write chunk data, only manifest") 110 | storeFlag := flags.String("store", "index", "Location of the chunk store") 111 | metaFlag := flags.String("meta", "", "Location of the metadata store") 112 | offsetFlag := flags.Int64("offset", 0, "Start reading file at given offset") 113 | exactFlag := flags.Bool("exact", false, "Ignore the NTFS bitmap, i.e. include unused blocks") 114 | noFileFlag := flags.Bool("nofile", false, "Don't do NTFS FILE deduping, just do gaps and unused space") 115 | minSizeFlag := flags.String("minsize", fmt.Sprintf("%d", fsdup.DefaultDedupFileSizeMinBytes), "Minimum file size to consider for deduping") 116 | maxChunkSizeFlag := flags.String("maxchunksize", fmt.Sprintf("%d", fsdup.DefaultChunkSizeMaxBytes), "Maximum size per chunk") 117 | writeConcurrencyFlag := flags.Int("writeconcurrency", 20, "Number of concurrent write requests against the store") 118 | 119 | flags.Parse(args) 120 | 121 | if flags.NArg() < 2 { 122 | usage() 123 | } 124 | 125 | if *debugFlag { 126 | fsdup.Debug = *debugFlag 127 | } 128 | 129 | offset := *offsetFlag 130 | exact := *exactFlag 131 | noFile := *noFileFlag 132 | minSize, err := convertToBytes(*minSizeFlag) 133 | if err != nil { 134 | exit(2, "Invalid min size value: " + err.Error()) 135 | } 136 | 137 | chunkMaxSize, err := convertToBytes(*maxChunkSizeFlag) 138 | if err != nil { 139 | exit(2, "Invalid max chunk size value: " + err.Error()) 140 | } 141 | writeConcurrency := int64(*writeConcurrencyFlag) 142 | 143 | file := flags.Arg(0) 144 | manifestId := flags.Arg(1) 145 | 146 | var store fsdup.ChunkStore 147 | if *noWriteFlag { 148 | store = fsdup.NewDummyChunkStore() 149 | } else { 150 | store, err = createChunkStore(*storeFlag) 151 | if err != nil { 152 | exit(2, "Invalid syntax: " + string(err.Error())) 153 | } 154 | } 155 | 156 | metaStore, err := createMetaStore(*metaFlag) 157 | if err != nil { 158 | exit(2, "Invalid syntax: " + string(err.Error())) 159 | } 160 | 161 | // Go index! 162 | if err := fsdup.Index(file, store, metaStore, manifestId, offset, exact, noFile, minSize, chunkMaxSize, writeConcurrency); err != nil { 163 | exit(2, "Cannot index file: " + string(err.Error())) 164 | } 165 | } 166 | 167 | func mapCommand(args []string) { 168 | flags := flag.NewFlagSet("map", flag.ExitOnError) 169 | debugFlag := flags.Bool("debug", fsdup.Debug, "Enable debug mode") 170 | storeFlag := flags.String("store", "index", "Location of the chunk store") 171 | metaFlag := flags.String("meta", "", "Location of the metadata store") 172 | cacheFlag := flags.String("cache", "cache", "Location of the chunk cache") 173 | 174 | flags.Parse(args) 175 | 176 | if flags.NArg() < 2 { 177 | usage() 178 | } 179 | 180 | if *debugFlag { 181 | fsdup.Debug = *debugFlag 182 | } 183 | 184 | manifestId := flags.Arg(0) 185 | targetFile := flags.Arg(1) 186 | 187 | store, err := createChunkStore(*storeFlag) 188 | if err != nil { 189 | exit(2, "Invalid syntax: " + string(err.Error())) 190 | } 191 | 192 | metaStore, err := createMetaStore(*metaFlag) 193 | if err != nil { 194 | exit(2, "Invalid syntax: " + string(err.Error())) 195 | } 196 | 197 | cache := fsdup.NewFileChunkStore(*cacheFlag) 198 | 199 | if err := fsdup.Map(manifestId, store, metaStore, cache, targetFile); err != nil { 200 | exit(2, "Cannot map drive file: " + string(err.Error())) 201 | } 202 | } 203 | 204 | func exportCommand(args []string) { 205 | flags := flag.NewFlagSet("export", flag.ExitOnError) 206 | debugFlag := flags.Bool("debug", fsdup.Debug, "Enable debug mode") 207 | storeFlag := flags.String("store", "index", "Location of the chunk store") 208 | metaFlag := flags.String("meta", "", "Location of the metadata store") 209 | 210 | flags.Parse(args) 211 | 212 | if flags.NArg() < 2 { 213 | usage() 214 | } 215 | 216 | if *debugFlag { 217 | fsdup.Debug = *debugFlag 218 | } 219 | 220 | manifestId := flags.Arg(0) 221 | outputFile := flags.Arg(1) 222 | 223 | store, err := createChunkStore(*storeFlag) 224 | if err != nil { 225 | exit(2, "Invalid syntax: " + string(err.Error())) 226 | } 227 | 228 | metaStore, err := createMetaStore(*metaFlag) 229 | if err != nil { 230 | exit(2, "Invalid syntax: " + string(err.Error())) 231 | } 232 | 233 | if err := fsdup.Export(manifestId, store, metaStore, outputFile); err != nil { 234 | exit(2, "Cannot export file: " + string(err.Error())) 235 | } 236 | } 237 | 238 | func importCommand(args []string) { 239 | flags := flag.NewFlagSet("import", flag.ExitOnError) 240 | debugFlag := flags.Bool("debug", fsdup.Debug, "Enable debug mode") 241 | storeFlag := flags.String("store", "index", "Location of the chunk store") 242 | metaFlag := flags.String("meta", "", "Location of the metadata store") 243 | 244 | flags.Parse(args) 245 | 246 | if flags.NArg() < 2 { 247 | usage() 248 | } 249 | 250 | if *debugFlag { 251 | fsdup.Debug = *debugFlag 252 | } 253 | 254 | inputFile := flags.Arg(0) 255 | manifestId := flags.Arg(1) 256 | 257 | store, err := createChunkStore(*storeFlag) 258 | if err != nil { 259 | exit(2, "Invalid syntax: " + string(err.Error())) 260 | } 261 | 262 | metaStore, err := createMetaStore(*metaFlag) 263 | if err != nil { 264 | exit(2, "Invalid syntax: " + string(err.Error())) 265 | } 266 | 267 | if err := fsdup.Import(manifestId, store, metaStore, inputFile); err != nil { 268 | exit(2, "Cannot import file: " + string(err.Error())) 269 | } 270 | } 271 | 272 | func printCommand(args []string) { 273 | flags := flag.NewFlagSet("print", flag.ExitOnError) 274 | debugFlag := flags.Bool("debug", fsdup.Debug, "Enable debug mode") 275 | metaFlag := flags.String("meta", "", "Location of the metadata store") 276 | 277 | flags.Parse(args) 278 | 279 | if flags.NArg() < 1 { 280 | usage() 281 | } 282 | 283 | if *debugFlag { 284 | fsdup.Debug = *debugFlag 285 | } 286 | 287 | metaStore, err := createMetaStore(*metaFlag) 288 | if err != nil { 289 | exit(2, "Invalid syntax: " + string(err.Error())) 290 | } 291 | 292 | var what string 293 | var manifestId string 294 | 295 | if flags.NArg() == 1 { 296 | what = "disk" 297 | manifestId = flags.Arg(0) 298 | } else { 299 | what = flags.Arg(0) 300 | manifestId = flags.Arg(1) 301 | } 302 | 303 | manifest, err := metaStore.ReadManifest(manifestId) 304 | if err != nil { 305 | exit(2, "Cannot read manifest: " + string(err.Error())) 306 | } 307 | 308 | switch what { 309 | case "disk": 310 | manifest.PrintDisk() 311 | case "chunks": 312 | if err := manifest.PrintChunks(); err != nil { 313 | exit(2, "Cannot print chunks: " + string(err.Error())) 314 | } 315 | default: 316 | usage() 317 | } 318 | } 319 | 320 | func statCommand(args []string) { 321 | flags := flag.NewFlagSet("stat", flag.ExitOnError) 322 | debugFlag := flags.Bool("debug", fsdup.Debug, "Enable debug mode") 323 | verboseFlag := flags.Bool("verbose", false, "Enable verbose mode") 324 | metaFlag := flags.String("meta", "", "Location of the metadata store") 325 | 326 | flags.Parse(args) 327 | 328 | if flags.NArg() < 1 { 329 | usage() 330 | } 331 | 332 | if *debugFlag { 333 | fsdup.Debug = *debugFlag 334 | } 335 | 336 | metaStore, err := createMetaStore(*metaFlag) 337 | if err != nil { 338 | exit(2, "Invalid syntax: " + string(err.Error())) 339 | } 340 | 341 | manifestIds := flags.Args() 342 | 343 | if err := fsdup.Stat(manifestIds, metaStore, *verboseFlag); err != nil { 344 | exit(2, "Cannot create manifest stats: " + string(err.Error())) 345 | } 346 | } 347 | 348 | func serverCommand(args []string) { 349 | flags := flag.NewFlagSet("server", flag.ExitOnError) 350 | debugFlag := flags.Bool("debug", fsdup.Debug, "Enable debug mode") 351 | storeFlag := flags.String("store", "index", "Location of the chunk store") 352 | metaFlag := flags.String("meta", "", "Location of the metadata store") 353 | 354 | flags.Parse(args) 355 | 356 | if flags.NArg() < 1 { 357 | usage() 358 | } 359 | 360 | if *debugFlag { 361 | fsdup.Debug = *debugFlag 362 | } 363 | 364 | store, err := createChunkStore(*storeFlag) 365 | if err != nil { 366 | exit(2, "Invalid syntax: " + string(err.Error())) 367 | } 368 | 369 | metaStore, err := createMetaStore(*metaFlag) 370 | if err != nil { 371 | exit(2, "Invalid syntax: " + string(err.Error())) 372 | } 373 | 374 | listenAddr := flags.Arg(0) 375 | if err := fsdup.ListenAndServe(listenAddr, store, metaStore); err != nil { 376 | exit(1, err.Error()) 377 | } 378 | } 379 | 380 | func uploadCommand(args []string) { 381 | flags := flag.NewFlagSet("upload", flag.ExitOnError) 382 | debugFlag := flags.Bool("debug", fsdup.Debug, "Enable debug mode") 383 | serverFlag := flags.String("server", ":9991", "Server address") 384 | metaFlag := flags.String("meta", "", "Location of the metadata store") 385 | flags.Parse(args) 386 | 387 | if flags.NArg() < 2 { 388 | usage() 389 | } 390 | 391 | if *debugFlag { 392 | fsdup.Debug = *debugFlag 393 | } 394 | 395 | metaStore, err := createMetaStore(*metaFlag) 396 | if err != nil { 397 | exit(2, "Invalid syntax: " + string(err.Error())) 398 | } 399 | 400 | inputFile := flags.Arg(0) 401 | manifestId := flags.Arg(1) 402 | 403 | if err := fsdup.Upload(manifestId, metaStore, inputFile, *serverFlag); err != nil { 404 | exit(2, "Cannot upload chunks for file: " + string(err.Error())) 405 | } 406 | } 407 | 408 | func createChunkStore(spec string) (fsdup.ChunkStore, error) { 409 | if regexp.MustCompile(`^(ceph|swift|gcloud|remote):`).MatchString(spec) { 410 | uri, err := url.ParseRequestURI(spec) 411 | if err != nil { 412 | return nil, err 413 | } 414 | 415 | if uri.Scheme == "ceph" { 416 | return createCephChunkStore(uri) 417 | } else if uri.Scheme == "swift" { 418 | return createSwiftChunkStore(uri) 419 | } else if uri.Scheme == "gcloud" { 420 | return createGcloudChunkStore(uri) 421 | } else if uri.Scheme == "remote" { 422 | return createRemoteChunkStore(uri) 423 | } 424 | 425 | return nil, errors.New("store type not supported") 426 | } 427 | 428 | return fsdup.NewFileChunkStore(spec), nil 429 | } 430 | 431 | func createCephChunkStore(uri *url.URL) (fsdup.ChunkStore, error) { 432 | var configFile string 433 | var pool string 434 | 435 | if uri.Opaque != "" { 436 | configFile = uri.Opaque 437 | } else if uri.Path != "" { 438 | configFile = uri.Path 439 | } else { 440 | configFile = "/etc/ceph/ceph.conf" 441 | } 442 | 443 | if _, err := os.Stat(configFile); err != nil { 444 | return nil, err 445 | } 446 | 447 | pool = uri.Query().Get("pool") 448 | if pool == "" { 449 | return nil, errors.New("invalid syntax for ceph store type, should be ceph:FILE?pool=POOL") 450 | } 451 | 452 | compressStr := uri.Query().Get("compress") 453 | compress := compressStr == "yes" || compressStr == "true" 454 | 455 | return fsdup.NewCephStore(configFile, pool, compress), nil 456 | } 457 | 458 | func createSwiftChunkStore(uri *url.URL) (fsdup.ChunkStore, error) { 459 | connection := &swift.Connection{} 460 | 461 | container := uri.Query().Get("container") 462 | if container == "" { 463 | return nil, errors.New("invalid syntax for swift store type, container parameter is required") 464 | } 465 | 466 | err := connection.ApplyEnvironment() 467 | if err != nil { 468 | return nil, err 469 | } 470 | 471 | // TODO provide way to override environment variables 472 | 473 | return fsdup.NewSwiftStore(connection, container), nil 474 | } 475 | 476 | func createGcloudChunkStore(uri *url.URL) (fsdup.ChunkStore, error) { 477 | project := uri.Query().Get("project") 478 | if project == "" { 479 | return nil, errors.New("invalid syntax for gcloud store type, project parameter is required") 480 | } 481 | 482 | bucket := uri.Query().Get("bucket") 483 | if bucket == "" { 484 | return nil, errors.New("invalid syntax for gcloud store type, bucket parameter is required") 485 | } 486 | 487 | return fsdup.NewGcloudStore(project, bucket), nil 488 | } 489 | 490 | func createRemoteChunkStore(uri *url.URL) (fsdup.ChunkStore, error) { 491 | return fsdup.NewRemoteChunkStore(uri.Opaque), nil 492 | } 493 | 494 | func createMetaStore(spec string) (fsdup.MetaStore, error) { 495 | if regexp.MustCompile(`^(remote|mysql):`).MatchString(spec) { 496 | uri, err := url.ParseRequestURI(spec) 497 | if err != nil { 498 | return nil, err 499 | } 500 | 501 | if uri.Scheme == "remote" { 502 | return createRemoteMetaStore(uri) 503 | } else if uri.Scheme == "mysql" { 504 | return createMysqlMetaStore(uri) 505 | } 506 | 507 | return nil, errors.New("meta store type not supported") 508 | } 509 | 510 | return createFileMetaStore() 511 | } 512 | 513 | func createFileMetaStore() (fsdup.MetaStore, error) { 514 | return fsdup.NewFileMetaStore(), nil 515 | } 516 | 517 | func createRemoteMetaStore(uri *url.URL) (fsdup.MetaStore, error) { 518 | return fsdup.NewRemoteMetaStore(uri.Opaque), nil 519 | } 520 | 521 | func createMysqlMetaStore(uri *url.URL) (fsdup.MetaStore, error) { 522 | return fsdup.NewMysqlMetaStore(uri.Opaque) 523 | } 524 | 525 | func convertToBytes(s string) (int64, error) { 526 | r := regexp.MustCompile(`^(\d+)([bBkKmMgGtT])?$`) 527 | matches := r.FindStringSubmatch(s) 528 | 529 | if matches == nil { 530 | return 0, errors.New("cannot convert to bytes: " + s) 531 | } 532 | 533 | value, err := strconv.Atoi(matches[1]) 534 | if err != nil { 535 | return 0, err 536 | } 537 | 538 | unit := strings.ToLower(matches[2]) 539 | switch unit { 540 | case "k": 541 | return int64(value) * (1 << 10), nil 542 | case "m": 543 | return int64(value) * (1 << 20), nil 544 | case "g": 545 | return int64(value) * (1 << 30), nil 546 | case "t": 547 | return int64(value) * (1 << 40), nil 548 | default: 549 | return int64(value), nil 550 | } 551 | } 552 | -------------------------------------------------------------------------------- /export.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | ) 7 | 8 | func Export(manifestId string, store ChunkStore, metaStore MetaStore, outputFile string) error { 9 | manifest, err := metaStore.ReadManifest(manifestId) 10 | if err != nil { 11 | return err 12 | } 13 | 14 | if err := truncateExportFile(outputFile, manifest.Size()); err != nil { 15 | return err 16 | } 17 | 18 | // Open file 19 | out, err := os.OpenFile(outputFile, os.O_WRONLY, 0666) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | buffer := make([]byte, manifest.chunkMaxSize) 25 | offset := int64(0) 26 | 27 | for _, breakpoint := range manifest.Offsets() { 28 | part := manifest.Get(breakpoint) 29 | sparse := part.checksum == nil 30 | length := part.chunkto - part.chunkfrom 31 | 32 | if sparse { 33 | debugf("%013d Skipping sparse section of %d bytes\n", offset, length) 34 | } else { 35 | debugf("%013d Writing chunk %x, offset %d - %d (size %d)\n", offset, part.checksum, part.chunkfrom, part.chunkto, length) 36 | 37 | read, err := store.ReadAt(part.checksum, buffer[:length], part.chunkfrom) 38 | if err != nil { 39 | return err 40 | } else if int64(read) != length { 41 | return errors.New("cannot read all required bytes from chunk") 42 | } 43 | 44 | written, err := out.WriteAt(buffer[:length], offset) 45 | if err != nil { 46 | return err 47 | } else if int64(written) != length { 48 | return errors.New("cannot write all bytes to output file") 49 | } 50 | } 51 | 52 | offset += length 53 | } 54 | 55 | err = out.Close() 56 | if err != nil { 57 | return err 58 | } 59 | 60 | return nil 61 | } 62 | 63 | // truncateExportFile wipes the output file (truncate to zero, then to target size) 64 | func truncateExportFile(outputFile string, size int64) error { 65 | if _, err := os.Stat(outputFile); err != nil { 66 | file, err := os.Create(outputFile) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | err = file.Close() 72 | if err != nil { 73 | return err 74 | } 75 | } else { 76 | if err := os.Truncate(outputFile, 0); err != nil { 77 | return err 78 | } 79 | } 80 | 81 | if err := os.Truncate(outputFile, size); err != nil { 82 | return err 83 | } 84 | 85 | return nil 86 | } -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module heckel.io/fsdup 2 | 3 | go 1.12 4 | 5 | require ( 6 | cloud.google.com/go v0.49.0 // indirect 7 | cloud.google.com/go/storage v1.4.0 8 | github.com/binwiederhier/buse-go v0.0.0-20190620221802-88a95178f4c4 9 | github.com/ceph/go-ceph v0.0.0-20181217221554-e32f9f0f2e94 10 | github.com/go-sql-driver/mysql v1.4.1 11 | github.com/gofrs/uuid v3.3.0+incompatible // indirect 12 | github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect 13 | github.com/golang/protobuf v1.3.2 14 | github.com/jstemmer/go-junit-report v0.9.1 // indirect 15 | github.com/ncw/swift v1.0.49 16 | go.opencensus.io v0.22.2 // indirect 17 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 18 | golang.org/x/exp v0.0.0-20191129062945-2f5052295587 // indirect 19 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect 20 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect 21 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect 22 | golang.org/x/sys v0.0.0-20191224085550-c709ea063b76 // indirect 23 | golang.org/x/tools v0.0.0-20191213221258-04c2e8eff935 // indirect 24 | google.golang.org/appengine v1.6.5 // indirect 25 | google.golang.org/genproto v0.0.0-20191223191004-3caeed10a8bf // indirect 26 | google.golang.org/grpc v1.26.0 27 | ) 28 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= 4 | cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= 5 | cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= 6 | cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= 7 | cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y= 8 | cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= 9 | cloud.google.com/go v0.49.0 h1:CH+lkubJzcPYB1Ggupcq0+k8Ni2ILdG2lYjDIgavDBQ= 10 | cloud.google.com/go v0.49.0/go.mod h1:hGvAdzcWNbyuxS3nWhD7H2cIJxjRRTRLQVB0bdputVY= 11 | cloud.google.com/go/bigquery v1.0.1 h1:hL+ycaJpVE9M7nLoiXb/Pn10ENE2u+oddxbD8uu0ZVU= 12 | cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= 13 | cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM= 14 | cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= 15 | cloud.google.com/go/pubsub v1.0.1 h1:W9tAK3E57P75u0XLLR82LZyw8VpAnhmyTOxW9qzmyj8= 16 | cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= 17 | cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= 18 | cloud.google.com/go/storage v1.4.0 h1:KDdqY5VTXBTqpSbctVTt0mVvfanP6JZzNzLE0qNY100= 19 | cloud.google.com/go/storage v1.4.0/go.mod h1:ZusYJWlOshgSBGbt6K3GnB3MT3H1xs2id9+TCl4fDBA= 20 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 21 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 22 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 23 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 24 | github.com/binwiederhier/buse-go v0.0.0-20190620221802-88a95178f4c4 h1:c5ufUfsSSuKX/xgnwhwRpy6HY+6PR4Pw2KG4LPuj3Bg= 25 | github.com/binwiederhier/buse-go v0.0.0-20190620221802-88a95178f4c4/go.mod h1:LT1PPZZ/xW98ibFT1olG8UziTSJCFmnjQWo/zegotzE= 26 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 27 | github.com/ceph/go-ceph v0.0.0-20181217221554-e32f9f0f2e94 h1:m3fyIqe28zGggNHc9hE2R3Zk/CJC3Lg8KFSNfXu8mpk= 28 | github.com/ceph/go-ceph v0.0.0-20181217221554-e32f9f0f2e94/go.mod h1:DhWkbjUxN0QRc0xQvpI9QhzqQSzYysRuZVcqSfiStds= 29 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 30 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 31 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 32 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 33 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 34 | github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= 35 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 36 | github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= 37 | github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 38 | github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= 39 | github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= 40 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= 41 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 42 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= 43 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 44 | github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 h1:uHTyIjqVhYRhLbJ8nIiOJHkEZZ+5YoOsAbD3sk82NiE= 45 | github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 46 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 47 | github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 48 | github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= 49 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 50 | github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= 51 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 52 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 53 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 54 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 55 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 56 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 57 | github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= 58 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 59 | github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= 60 | github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= 61 | github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 62 | github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= 63 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 64 | github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= 65 | github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= 66 | github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= 67 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 68 | github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= 69 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 70 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc= 71 | github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= 72 | github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= 73 | github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= 74 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 75 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 76 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 77 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 78 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 79 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 80 | github.com/ncw/swift v1.0.49 h1:eQaKIjSt/PXLKfYgzg01nevmO+CMXfXGRhB1gOhDs7E= 81 | github.com/ncw/swift v1.0.49/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= 82 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 83 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 84 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 85 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 86 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 87 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 88 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 89 | go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= 90 | go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= 91 | go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= 92 | go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= 93 | go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 94 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 95 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 96 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 97 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= 98 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 99 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 100 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 101 | golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= 102 | golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= 103 | golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw= 104 | golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= 105 | golang.org/x/exp v0.0.0-20191129062945-2f5052295587 h1:5Uz0rkjCFu9BC9gCRN7EkwVvhNyQgGWb8KNJrPwBoHY= 106 | golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= 107 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 108 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 109 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 110 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 111 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 112 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 113 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 114 | golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 115 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= 116 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 117 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= 118 | golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= 119 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 120 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 121 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 122 | golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= 123 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 124 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 125 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 126 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 127 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 128 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 129 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 130 | golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 131 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 132 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 133 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= 134 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 135 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= 136 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 137 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 138 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 139 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= 140 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 141 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= 142 | golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 143 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 144 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 145 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 146 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 147 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= 148 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 149 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 150 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 151 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 152 | golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= 153 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 154 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 155 | golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 156 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 157 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc= 158 | golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 159 | golang.org/x/sys v0.0.0-20191224085550-c709ea063b76 h1:Dho5nD6R3PcW2SH1or8vS0dszDaXRxIw55lBX7XiE5g= 160 | golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 161 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 162 | golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 163 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 164 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 165 | golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 166 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 167 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 168 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 169 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 170 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 171 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 172 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 173 | golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 174 | golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 175 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 176 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 177 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 178 | golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 179 | golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 180 | golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 181 | golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 182 | golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2 h1:EtTFh6h4SAKemS+CURDMTDIANuduG5zKEXShyy18bGA= 183 | golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 184 | golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 185 | golang.org/x/tools v0.0.0-20191213221258-04c2e8eff935 h1:kJQZhwFzSwJS2BxboKjdZzWczQOZx8VuH7Y8hhuGUtM= 186 | golang.org/x/tools v0.0.0-20191213221258-04c2e8eff935/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 187 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 188 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 189 | google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= 190 | google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= 191 | google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 192 | google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= 193 | google.golang.org/api v0.14.0 h1:uMf5uLi4eQMRrMKhCplNik4U4H8Z6C1br3zOtAa/aDE= 194 | google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= 195 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 196 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 197 | google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 198 | google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= 199 | google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 200 | google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= 201 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 202 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 203 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 204 | google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 205 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 206 | google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 207 | google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 208 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 209 | google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= 210 | google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9 h1:6XzpBoANz1NqMNfDXzc2QmHmbb1vyMsvRfoP5rM+K1I= 211 | google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 212 | google.golang.org/genproto v0.0.0-20191223191004-3caeed10a8bf h1:1x8rC5/IgdLMPbPTvlQTN28+rcy8XL9Q19UWUMDyqYs= 213 | google.golang.org/genproto v0.0.0-20191223191004-3caeed10a8bf/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= 214 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 215 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 216 | google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8= 217 | google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 218 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 219 | google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= 220 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 221 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 222 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 223 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 224 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 225 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 226 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 227 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 228 | honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 229 | honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 230 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 231 | honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= 232 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 233 | rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= 234 | -------------------------------------------------------------------------------- /import.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "encoding/hex" 5 | "errors" 6 | "fmt" 7 | "os" 8 | ) 9 | 10 | func Import(manifestId string, store ChunkStore, metaStore MetaStore, inputFile string) error { 11 | manifest, err := metaStore.ReadManifest(manifestId) 12 | if err != nil { 13 | return err 14 | } 15 | 16 | // Open file 17 | in, err := os.OpenFile(inputFile, os.O_RDONLY, 0666) 18 | if err != nil { 19 | return err 20 | } 21 | defer in.Close() 22 | 23 | stat, err := in.Stat() 24 | if err != nil { 25 | return err 26 | } else if stat.Size() != manifest.Size() { 27 | return errors.New("size in manifest does not match file size. wrong input file?") 28 | } 29 | 30 | chunkSlices, err := manifest.ChunkSlices() 31 | if err != nil { 32 | return err 33 | } 34 | 35 | imported := int64(0) 36 | skipped := int64(0) 37 | buffer := make([]byte, manifest.chunkMaxSize) 38 | 39 | for _, checksumStr := range manifest.ChecksumsByDiskOffset(chunkSlices) { 40 | slices := chunkSlices[checksumStr] 41 | 42 | statusf("Importing chunk %d (%d skipped, %d total) ...", imported + 1, skipped, len(chunkSlices)) 43 | debugf("Importing chunk %s (%d slices) ...\n", checksumStr, len(slices)) 44 | 45 | checksum, err := hex.DecodeString(checksumStr) // FIXME this is ugly. checksum should be its own type. 46 | if err != nil { 47 | return err 48 | } 49 | 50 | if err := store.Stat(checksum); err == nil { 51 | debugf("Skipping chunk. Already exists in index.\n") 52 | skipped++ 53 | imported++ 54 | continue 55 | } 56 | 57 | chunkSize := int64(0) 58 | 59 | for i, slice := range slices { 60 | debugf("idx %-5d diskoff %13d - %13d len %-10d chunkoff %13d - %13d\n", 61 | i, slice.diskfrom, slice.diskto, slice.length, slice.chunkfrom, slice.chunkto) 62 | 63 | read, err := in.ReadAt(buffer[slice.chunkfrom:slice.chunkto], slice.diskfrom) 64 | if err != nil { 65 | return err 66 | } else if int64(read) != slice.length { 67 | return errors.New(fmt.Sprintf("cannot read full chunk from input file, read only %d bytes, but %d expectecd", read, slice.length)) 68 | } 69 | 70 | chunkSize += slice.length 71 | } 72 | 73 | if err := store.Write(checksum, buffer[:chunkSize]); err != nil { 74 | return err 75 | } 76 | 77 | imported++ 78 | } 79 | 80 | err = in.Close() 81 | if err != nil { 82 | return err 83 | } 84 | 85 | statusf("Imported %d chunks (%d skipped)\n", imported, skipped) 86 | 87 | return nil 88 | } 89 | -------------------------------------------------------------------------------- /index.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "os" 7 | "os/exec" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | type fileType int 13 | 14 | const ( 15 | typeNtfs fileType = iota + 1 16 | typeMbrDisk 17 | typeGptDisk 18 | typeUnknown 19 | ) 20 | 21 | const ( 22 | probeTypeBufferLength = 1024 23 | ) 24 | 25 | func Index(inputFile string, store ChunkStore, metaStore MetaStore, manifestId string, offset int64, exact bool, 26 | noFile bool, minSize int64, chunkMaxSize int64, writeConcurrency int64) error { 27 | file, err := os.Open(inputFile) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | defer file.Close() 33 | 34 | var chunker Chunker 35 | 36 | size, err := readFileSize(file, inputFile) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | // Probe type to figure out which chunker to pick 42 | fileType, err := probeType(file, offset) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | switch fileType { 48 | case typeNtfs: 49 | chunker = NewNtfsChunker(file, store, offset, exact, noFile, minSize, chunkMaxSize, writeConcurrency) 50 | case typeMbrDisk: 51 | chunker = NewMbrDiskChunker(file, store, offset, size, exact, noFile, minSize, chunkMaxSize, writeConcurrency) 52 | case typeGptDisk: 53 | chunker = NewGptDiskChunker(file, store, offset, size, exact, noFile, minSize, chunkMaxSize, writeConcurrency) 54 | default: 55 | chunker = NewFixedChunker(file, store, offset, size, chunkMaxSize, writeConcurrency) 56 | } 57 | 58 | manifest, err := chunker.Dedup() 59 | if err != nil { 60 | return err 61 | } 62 | 63 | if Debug { 64 | debugf("Manifest:\n") 65 | manifest.PrintDisk() 66 | } 67 | 68 | if err := metaStore.WriteManifest(manifestId, manifest); err != nil { 69 | return err 70 | } 71 | 72 | return nil 73 | } 74 | 75 | // Determine file size for file or block device 76 | func readFileSize(file *os.File, inputFile string) (int64, error) { 77 | stat, err := file.Stat() 78 | if err != nil { 79 | return 0, err 80 | } 81 | 82 | if stat.Mode() & os.ModeDevice == os.ModeDevice { 83 | // TODO This is ugly, but it works. 84 | 85 | out, err := exec.Command("blockdev", "--getsize64", inputFile).Output() 86 | if err != nil { 87 | return 0, err 88 | } 89 | 90 | size, err := strconv.ParseInt(strings.Trim(string(out), "\n"), 10, 64) 91 | if err != nil { 92 | return 0, err 93 | } 94 | 95 | return size, nil 96 | } else { 97 | return stat.Size(), nil 98 | } 99 | } 100 | 101 | func probeType(reader io.ReaderAt, offset int64) (fileType, error) { 102 | buffer := make([]byte, probeTypeBufferLength) 103 | _, err := reader.ReadAt(buffer, offset) 104 | if err != nil { 105 | return -1, err 106 | } 107 | 108 | // Be aware that the probing order is important. 109 | // NTFS and GPT also have an MBR signature! 110 | 111 | // Detect NTFS 112 | if bytes.Compare([]byte(ntfsBootMagic), buffer[ntfsBootMagicOffset:ntfsBootMagicOffset+len(ntfsBootMagic)]) == 0 { 113 | return typeNtfs, nil 114 | } 115 | 116 | // Detect GPT 117 | if bytes.Compare([]byte(gptSignatureMagic), buffer[gptSignatureOffset:gptSignatureOffset+len(gptSignatureMagic)]) == 0 { 118 | return typeGptDisk, nil 119 | } 120 | 121 | // Detect MBR 122 | if mbrSignatureMagic == parseUintLE(buffer, mbrSignatureOffset, mbrSignatureLength) { 123 | return typeMbrDisk, nil 124 | } 125 | 126 | return typeUnknown, nil 127 | } 128 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "strings" 7 | ) 8 | 9 | var ( 10 | Debug = false // Toggle with -debug CLI flag! // TODO fix this with debug/log levels 11 | Quiet = false // TODO fix this with log levels 12 | 13 | statusLastLength = 0 14 | ) 15 | 16 | func debugf(format string, args ...interface{}) { 17 | if Quiet { 18 | return 19 | } 20 | 21 | if Debug { 22 | log.Printf(format, args...) 23 | } 24 | } 25 | 26 | func statusf(format string, args ...interface{}) { 27 | if Quiet { 28 | return 29 | } 30 | 31 | if Debug { 32 | fmt.Printf(format + "\n", args...) 33 | } else { 34 | status := fmt.Sprintf(format, args...) 35 | statusNewLength := len(status) 36 | 37 | if statusNewLength < statusLastLength { 38 | fmt.Printf("\r%s\r%s", strings.Repeat(" ", statusLastLength), status) 39 | } else { 40 | fmt.Printf("\r%s", status) 41 | } 42 | 43 | statusLastLength = statusNewLength 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /manifest.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/golang/protobuf/proto" 7 | "heckel.io/fsdup/pb" 8 | "io/ioutil" 9 | "sort" 10 | ) 11 | 12 | type kind int 13 | 14 | const ( 15 | kindFile kind = 1 16 | kindSparse kind = 2 17 | kindGap kind = 3 18 | ) 19 | 20 | type manifest struct { 21 | id string 22 | diskMap map[int64]*chunkSlice 23 | size int64 24 | chunkMaxSize int64 25 | offsets []int64 // cache, don't forget to update for all write operations! 26 | } 27 | 28 | type chunkSlice struct { 29 | checksum []byte 30 | kind kind 31 | diskfrom int64 32 | diskto int64 33 | chunkfrom int64 34 | chunkto int64 35 | length int64 36 | } 37 | 38 | func NewManifest(chunkMaxSize int64) *manifest { 39 | return &manifest{ 40 | id: randString(32), 41 | size: 0, 42 | chunkMaxSize: chunkMaxSize, 43 | diskMap: make(map[int64]*chunkSlice, 0), 44 | offsets: nil, 45 | } 46 | } 47 | 48 | func NewManifestFromFile(file string) (*manifest, error) { 49 | in, err := ioutil.ReadFile(file) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | pbmanifest := &pb.ManifestV1{} 55 | if err := proto.Unmarshal(in, pbmanifest); err != nil { 56 | return nil, err 57 | } 58 | 59 | return NewManifestFromProto(pbmanifest) 60 | } 61 | 62 | func NewManifestFromProto(pbmanifest *pb.ManifestV1) (*manifest, error) { 63 | chunkMaxSize := int64(DefaultChunkSizeMaxBytes) 64 | if pbmanifest.ChunkMaxSize != 0 { 65 | chunkMaxSize = pbmanifest.ChunkMaxSize 66 | } 67 | 68 | manifest := NewManifest(chunkMaxSize) 69 | manifest.id = pbmanifest.Id 70 | manifest.chunkMaxSize = chunkMaxSize 71 | 72 | offset := int64(0) 73 | for _, slice := range pbmanifest.Slices { 74 | manifest.Add(&chunkSlice{ 75 | checksum: slice.Checksum, 76 | kind: kind(slice.Kind), 77 | diskfrom: offset, 78 | diskto: offset + slice.Length, 79 | chunkfrom: slice.Offset, 80 | chunkto: slice.Offset + slice.Length, 81 | length: slice.Length, 82 | }) 83 | 84 | offset += slice.Length 85 | } 86 | 87 | return manifest, nil 88 | } 89 | 90 | // Breakpoints returns a sorted list of slice offsets, useful for sequential disk traversal 91 | func (m *manifest) Offsets() []int64 { 92 | if m.offsets != nil { 93 | return m.offsets 94 | } 95 | 96 | offsets := make([]int64, 0, len(m.diskMap)) 97 | for offset, _ := range m.diskMap { 98 | offsets = append(offsets, offset) 99 | } 100 | 101 | sort.Slice(offsets, func(i, j int) bool { 102 | return offsets[i] < offsets[j] 103 | }) 104 | 105 | m.offsets = offsets 106 | return offsets 107 | } 108 | 109 | // Chunks returns a map of chunks in this manifest. It does not contain the chunk data. 110 | // The key is a hex representation of the chunk checksum. 111 | func (m *manifest) Chunks() map[string]*chunk { 112 | chunks := make(map[string]*chunk, 0) 113 | 114 | for _, slice := range m.diskMap { 115 | // This is a weird way to get the chunk size, but hey ... 116 | checksumStr := fmt.Sprintf("%x", slice.checksum) 117 | 118 | if _, ok := chunks[checksumStr]; !ok { 119 | chunks[checksumStr] = &chunk{ 120 | checksum: slice.checksum, 121 | size: slice.chunkto, 122 | } 123 | } else { 124 | chunks[checksumStr].size = maxInt64(chunks[checksumStr].size, slice.chunkto) 125 | } 126 | } 127 | 128 | return chunks 129 | } 130 | 131 | // SlicesBetween efficiently finds the slices storing the given disk offset 132 | func (m *manifest) SlicesBetween(from int64, to int64) ([]*chunkSlice, error) { 133 | offsets := m.Offsets() 134 | 135 | fromIndex := sort.Search(len(offsets), func(i int) bool { 136 | return i+1 == len(offsets) || offsets[i+1] > from 137 | }) 138 | 139 | toIndex := sort.Search(len(offsets), func(i int) bool { 140 | return i+1 == len(offsets) || offsets[i+1] > to 141 | }) 142 | 143 | if fromIndex == len(offsets) || toIndex == len(offsets) { 144 | return nil, errors.New("cannot find slice at given offset") 145 | } 146 | 147 | slices := make([]*chunkSlice, 0) 148 | 149 | for i := fromIndex; i <= toIndex; i++ { 150 | slices = append(slices, m.diskMap[offsets[i]]) 151 | } 152 | 153 | return slices, nil 154 | } 155 | 156 | // Slices returns a map of chunks and its sections on disk. 157 | // The key is a hex representation of the chunk checksum. 158 | func (m *manifest) ChunkSlices() (map[string][]*chunkSlice, error) { 159 | // First, we'll sort all slices into a map grouped by chunk checksum. This 160 | // produces a map with potentially overlapping slices: 161 | // 162 | // slices[aabbee..] = ( 163 | // (from:16384, to:36864), 164 | // (from:0, to:8192), 165 | // (from:8192, to:16384), 166 | // (from:36864, to:65536), 167 | // (from:0, to:16384), < overlaps with two slices 168 | // ) 169 | 170 | checksumSlicesMap := make(map[string][]*chunkSlice, 0) 171 | 172 | for _, slice := range m.diskMap { 173 | if slice.checksum == nil { 174 | continue 175 | } 176 | 177 | checksumStr := fmt.Sprintf("%x", slice.checksum) 178 | 179 | if _, ok := checksumSlicesMap[checksumStr]; !ok { 180 | checksumSlicesMap[checksumStr] = make([]*chunkSlice, 0) 181 | } 182 | 183 | checksumSlicesMap[checksumStr] = append(checksumSlicesMap[checksumStr], slice) 184 | } 185 | 186 | // Now, we'll sort each disk slice list by "from" (smallest first), 187 | // and if the "from" fields are equal, by the "to" field (highest first); 188 | // this is to prefer larger sections. 189 | // 190 | // slices[aabbee..] = ( 191 | // (from:0, to:16384), < overlaps with next two slices 192 | // (from:0, to:8192), 193 | // (from:8192, to:16384), 194 | // (from:16384, to:36864), 195 | // (from:36864, to:65536), 196 | // ) 197 | 198 | for _, slices := range checksumSlicesMap { 199 | sort.Slice(slices, func(i, j int) bool { 200 | if slices[i].chunkfrom > slices[j].chunkfrom { 201 | return false 202 | } else if slices[i].chunkfrom < slices[j].chunkfrom { 203 | return true 204 | } else { 205 | return slices[i].chunkto > slices[j].chunkto 206 | } 207 | }) 208 | } 209 | 210 | // Now, we walk the list and find connecting pieces 211 | // 212 | // slices[aabbee..] = ( 213 | // (from:0, to:16384), 214 | // (from:16384, to:36864), 215 | // (from:36864, to:65536), 216 | // ) 217 | 218 | for checksumStr, slices := range checksumSlicesMap { 219 | if len(slices) == 1 { 220 | continue 221 | } 222 | 223 | newSlices := make([]*chunkSlice, 1) 224 | newSlices[0] = slices[0] 225 | 226 | for c, n := 0, 1; c < len(slices) && n < len(slices); n++ { 227 | current := slices[c] 228 | next := slices[n] 229 | 230 | if current.chunkto == next.chunkfrom { 231 | newSlices = append(newSlices, next) 232 | c = n 233 | } 234 | } 235 | 236 | checksumSlicesMap[checksumStr] = newSlices 237 | } 238 | 239 | return checksumSlicesMap, nil 240 | } 241 | 242 | // ChecksumsByDiskOffset orders the given list by first slice disk offset. This 243 | // is useful to read all chunks as sequential as possible. 244 | func (m *manifest) ChecksumsByDiskOffset(chunkSlices map[string][]*chunkSlice) []string { 245 | checksumStrs := make([]string, 0) 246 | for checksumStr, _ := range chunkSlices { 247 | checksumStrs = append(checksumStrs, checksumStr) 248 | } 249 | 250 | sort.Slice(checksumStrs, func(i, j int) bool { 251 | return chunkSlices[checksumStrs[i]][0].diskfrom < chunkSlices[checksumStrs[j]][0].diskfrom 252 | }) 253 | 254 | return checksumStrs 255 | } 256 | 257 | // Add adds a chunk slice to the manifest at the given from 258 | func (m *manifest) Add(slice *chunkSlice) { 259 | m.diskMap[slice.diskfrom] = slice 260 | m.resetCaches() 261 | } 262 | 263 | // Get receives a chunk slice from the manifest at the given from. 264 | // Note that the from must match exactly. No soft matching is performed. 265 | func (m *manifest) Get(offset int64) *chunkSlice { 266 | return m.diskMap[offset] 267 | } 268 | 269 | func (m *manifest) Size() int64 { 270 | size := int64(0) 271 | 272 | for offset, _ := range m.diskMap { 273 | slice := m.diskMap[offset] 274 | size = maxInt64(size, offset + slice.chunkto- slice.chunkfrom) 275 | } 276 | 277 | return size 278 | } 279 | 280 | func (m *manifest) Merge(other *manifest) { 281 | for offset, part := range other.diskMap { 282 | m.diskMap[offset] = part 283 | } 284 | 285 | m.resetCaches() 286 | } 287 | 288 | func (m *manifest) MergeAtOffset(offset int64, other *manifest) { 289 | for sliceOffset, part := range other.diskMap { 290 | m.diskMap[offset+sliceOffset] = part 291 | } 292 | 293 | m.resetCaches() 294 | } 295 | 296 | func (m *manifest) WriteToFile(file string) error { 297 | buffer, err := proto.Marshal(m.Proto()) 298 | if err != nil { 299 | return err 300 | } 301 | 302 | if err := ioutil.WriteFile(file, buffer, 0644); err != nil { 303 | return err 304 | } 305 | 306 | return nil 307 | } 308 | 309 | func (m *manifest) Proto() *pb.ManifestV1 { 310 | pbmanifest := &pb.ManifestV1{ 311 | Size: m.Size(), 312 | Slices: make([]*pb.Slice, len(m.diskMap)), 313 | ChunkMaxSize: m.chunkMaxSize, 314 | } 315 | 316 | for i, offset := range m.Offsets() { 317 | slice := m.diskMap[offset] 318 | pbmanifest.Slices[i] = &pb.Slice{ 319 | Checksum: slice.checksum, 320 | Offset: slice.chunkfrom, 321 | Length: slice.chunkto - slice.chunkfrom, 322 | Kind: int32(slice.kind), 323 | } 324 | } 325 | 326 | return pbmanifest 327 | } 328 | 329 | func (m *manifest) PrintDisk() { 330 | fmt.Printf("id = %s\n", m.id) 331 | fmt.Printf("max chunk size = %d\n", m.chunkMaxSize) 332 | fmt.Printf("slices:\n") 333 | 334 | for i, offset := range m.Offsets() { 335 | slice := m.diskMap[offset] 336 | 337 | if slice.checksum == nil { 338 | fmt.Printf("idx %-10d diskoff %13d - %13d len %-13d sparse -\n", 339 | i, offset, offset + slice.chunkto- slice.chunkfrom, slice.chunkto- slice.chunkfrom) 340 | } else { 341 | kind := "unknown" 342 | if slice.kind == kindGap { 343 | kind = "gap" 344 | } else if slice.kind == kindFile { 345 | kind = "file" 346 | } 347 | 348 | fmt.Printf("idx %-10d diskoff %13d - %13d len %-13d %-10s chunk %64x chunkoff %10d - %10d\n", 349 | i, offset, offset + slice.chunkto- slice.chunkfrom, slice.chunkto- slice.chunkfrom, kind, slice.checksum, slice.chunkfrom, slice.chunkto) 350 | } 351 | } 352 | } 353 | 354 | func (m *manifest) PrintChunks() error { 355 | chunkSlices, err := m.ChunkSlices() 356 | if err != nil { 357 | return err 358 | } 359 | 360 | for _, checksumStr := range m.ChecksumsByDiskOffset(chunkSlices) { 361 | slices := chunkSlices[checksumStr] 362 | for i, slice := range slices { 363 | fmt.Printf("chunk %s idx %-5d diskoff %13d - %13d len %-13d chunkoff %10d - %10d\n", 364 | checksumStr, i, slice.diskfrom, slice.diskto, slice.length, slice.chunkfrom, slice.chunkto) 365 | } 366 | } 367 | 368 | return nil 369 | } 370 | 371 | func (m *manifest) resetCaches() { 372 | m.offsets = nil 373 | } 374 | 375 | func (k kind) toString() string { 376 | if k == kindFile { 377 | return "file" 378 | } else if k == kindSparse { 379 | return "sparse" 380 | } else if k == kindGap { 381 | return "gap" 382 | } else { 383 | return "unknown" 384 | } 385 | } 386 | 387 | func kindFromString(s string) (kind, error) { 388 | if s == "file" { 389 | return kindFile, nil 390 | } else if s == "sparse" { 391 | return kindSparse, nil 392 | } else if s == "gap" { 393 | return kindGap, nil 394 | } else { 395 | return kindFile, errors.New("invalid kind string " + s) 396 | } 397 | } -------------------------------------------------------------------------------- /map.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/binwiederhier/buse-go/buse" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "os/signal" 11 | "strings" 12 | ) 13 | 14 | // Read: 15 | // written? ---n---> cached? ---n---> store 16 | // | y 17 | 18 | type manifestImage struct { 19 | manifest *manifest 20 | store ChunkStore 21 | target *os.File 22 | cache ChunkStore 23 | chunks map[string]*chunk 24 | written map[int64]bool 25 | sliceCount map[string]int64 26 | buffer []byte 27 | } 28 | 29 | func Map(manifestId string, store ChunkStore, metaStore MetaStore, cache ChunkStore, targetFile string) error { 30 | manifest, err := metaStore.ReadManifest(manifestId) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | var target *os.File 36 | if targetFile != "" { 37 | target, err = os.OpenFile(targetFile, os.O_CREATE | os.O_RDWR | os.O_TRUNC, 0600) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | if err := target.Truncate(manifest.Size()); err != nil { 43 | return err 44 | } 45 | } 46 | 47 | deviceName, err := findNextNbdDevice() 48 | if err != nil { 49 | return err 50 | } 51 | 52 | debugf("Creating device %s ...\n", deviceName) 53 | 54 | image := NewManifestImage(manifest, store, cache, target) 55 | device, err := buse.CreateDevice(deviceName, uint(manifest.Size()), image) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | sig := make(chan os.Signal) 61 | signal.Notify(sig, os.Interrupt) 62 | go func() { 63 | if err := device.Connect(); err != nil { 64 | debugf("Buse device stopped with error: %s", err) 65 | } else { 66 | log.Println("Buse device stopped gracefully.") 67 | } 68 | }() 69 | 70 | <-sig 71 | 72 | // Received SIGTERM, cleanup 73 | debugf("SIGINT, disconnecting...\n") 74 | device.Disconnect() 75 | 76 | return nil 77 | } 78 | 79 | func NewManifestImage(manifest *manifest, store ChunkStore, cache ChunkStore, target *os.File) *manifestImage { 80 | sliceCount := make(map[string]int64, 0) 81 | 82 | // Create slice count map for cache accounting 83 | for _, sliceOffset := range manifest.Offsets() { 84 | slice := manifest.Get(sliceOffset) 85 | if slice.checksum != nil { 86 | checksumStr := fmt.Sprintf("%x", slice.checksum) 87 | 88 | if _, ok := sliceCount[checksumStr]; ok { 89 | sliceCount[checksumStr]++ 90 | } else { 91 | sliceCount[checksumStr] = 1 92 | } 93 | } 94 | } 95 | 96 | return &manifestImage{ 97 | manifest: manifest, 98 | store: store, 99 | target: target, 100 | cache: cache, 101 | chunks: manifest.Chunks(), // cache ! 102 | written: make(map[int64]bool, 0), 103 | sliceCount: sliceCount, 104 | buffer: make([]byte, manifest.chunkMaxSize), 105 | } 106 | } 107 | 108 | func (d *manifestImage) ReadAt(p []byte, off uint) error { 109 | debugf("READ offset %d, len %d\n", off, len(p)) 110 | 111 | if err := d.syncSlices(int64(off), int64(off) + int64(len(p))); err != nil { 112 | return d.wrapError(err) 113 | } 114 | 115 | read, err := d.target.ReadAt(p, int64(off)) 116 | if err != nil { 117 | return err 118 | } else if read != len(p) { 119 | return errors.New("cannot read from target file") 120 | } 121 | 122 | return nil 123 | } 124 | 125 | func (d *manifestImage) WriteAt(p []byte, off uint) error { 126 | if d.target == nil { 127 | debugf("Failed to write to device at offset %d, to %d: Cannot write to read only device\n", off, len(p)) 128 | return errors.New("cannot write to read only device") 129 | } 130 | 131 | if err := d.syncSlices(int64(off), int64(off) + int64(len(p))); err != nil { 132 | return err 133 | } 134 | 135 | debugf("WRITE offset %d, len %d\n", off, len(p)) 136 | 137 | written, err := d.target.WriteAt(p, int64(off)) 138 | if err != nil { 139 | return err 140 | } else if written != len(p) { 141 | return errors.New("could not write all bytes") 142 | } 143 | 144 | return nil 145 | } 146 | 147 | func (d *manifestImage) syncSlices(from int64, to int64) error { 148 | slices, err := d.manifest.SlicesBetween(from, to) 149 | if err != nil { 150 | return d.wrapError(err) 151 | } 152 | 153 | for _, slice := range slices { 154 | if err := d.syncSlice(slice); err != nil { 155 | return err 156 | } 157 | } 158 | 159 | return nil 160 | } 161 | 162 | func (d *manifestImage) syncSlice(slice *chunkSlice) error { 163 | if _, ok := d.written[slice.diskfrom]; ok { 164 | return nil 165 | } 166 | 167 | if slice.checksum == nil { 168 | d.written[slice.diskfrom] = true 169 | return nil 170 | } 171 | 172 | buffer := d.buffer 173 | length := slice.chunkto - slice.chunkfrom 174 | debugf("Syncing diskoff %d - %d (len %d) -> checksum %x, %d to %d\n", 175 | slice.diskfrom, slice.diskto, length, slice.checksum, slice.chunkfrom, slice.chunkto) 176 | 177 | checksumStr := fmt.Sprintf("%x", slice.checksum) 178 | 179 | read, err := d.cache.ReadAt(slice.checksum, buffer[:length], slice.chunkfrom) 180 | if err != nil { 181 | debugf("Chunk %x not in cache. Retrieving full chunk ...\n", slice.checksum) 182 | 183 | // Read entire chunk, store to cache 184 | chunk := d.chunks[checksumStr] 185 | 186 | // FIXME: This will fill up the local cache will all chunks and never delete it 187 | read, err = d.store.ReadAt(slice.checksum, buffer[:chunk.size], 0) 188 | if err != nil { 189 | return err 190 | } else if int64(read) != chunk.size { 191 | return errors.New(fmt.Sprintf("cannot read entire chunk, read only %d bytes", read)) 192 | } 193 | 194 | if err := d.cache.Write(slice.checksum, buffer[:chunk.size]); err != nil { 195 | return err 196 | } 197 | 198 | buffer = buffer[slice.chunkfrom:slice.chunkto] 199 | } else if int64(read) != length { 200 | return errors.New(fmt.Sprintf("cannot read entire slice, read only %d bytes", read)) 201 | } 202 | 203 | _, err = d.target.WriteAt(buffer[:length], slice.diskfrom) 204 | if err != nil { 205 | return err 206 | } 207 | 208 | // Accounting 209 | d.written[slice.diskfrom] = true 210 | d.sliceCount[checksumStr]-- 211 | 212 | // Remove from cache if it will never be requested again 213 | if d.sliceCount[checksumStr] == 0 { 214 | if err := d.cache.Remove(slice.checksum); err != nil { 215 | return err 216 | } 217 | } 218 | 219 | return nil 220 | } 221 | 222 | func (d *manifestImage) Disconnect() { 223 | // No thanks 224 | } 225 | 226 | func (d *manifestImage) Flush() error { 227 | return nil 228 | } 229 | 230 | func (d *manifestImage) Trim(off, length uint) error { 231 | return nil 232 | } 233 | 234 | func findNextNbdDevice() (string, error) { 235 | for i := 0; i < 256; i++ { 236 | sizeFile := fmt.Sprintf("/sys/class/block/nbd%d/size", i) 237 | 238 | if _, err := os.Stat(sizeFile); err == nil { 239 | b, err := ioutil.ReadFile(sizeFile) 240 | if err != nil { 241 | return "", err 242 | } 243 | 244 | if strings.Trim(string(b), "\n") == "0" { 245 | return fmt.Sprintf("/dev/nbd%d", i), nil 246 | } 247 | } 248 | } 249 | 250 | return "", errors.New("cannot find free nbd device, driver not loaded?") 251 | } 252 | 253 | func (d *manifestImage) wrapError(err error) error { 254 | fmt.Printf("Error: %s\n", err.Error()) 255 | return err 256 | } 257 | -------------------------------------------------------------------------------- /meta_store.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | type MetaStore interface { 4 | ReadManifest(manifestId string) (*manifest, error) 5 | WriteManifest(manifestId string, manifest *manifest) error 6 | } 7 | -------------------------------------------------------------------------------- /meta_store_file.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | type fileMetaStore struct { 4 | // Nothing 5 | } 6 | 7 | func NewFileMetaStore() *fileMetaStore { 8 | debugf("Creating file metadata store\n") 9 | return &fileMetaStore{} 10 | } 11 | 12 | func (s *fileMetaStore) ReadManifest(manifestId string) (*manifest, error) { 13 | return NewManifestFromFile(manifestId) 14 | } 15 | 16 | func (s* fileMetaStore) WriteManifest(manifestId string, manifest *manifest) error { 17 | return manifest.WriteToFile(manifestId) 18 | } -------------------------------------------------------------------------------- /meta_store_mysql.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/hex" 6 | "fmt" 7 | ) 8 | 9 | type mysqlMetaStore struct { 10 | db *sql.DB 11 | } 12 | 13 | var _ MetaStore = &mysqlMetaStore{} 14 | 15 | func NewMysqlMetaStore(dataSource string) (*mysqlMetaStore, error) { 16 | debugf("Creating MySQL metadata store using datasource %s\n", dataSource) 17 | 18 | db, err := sql.Open("mysql", dataSource) // "fsdup:fsdup@/fsdup" 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | return &mysqlMetaStore{ 24 | db: db, 25 | }, nil 26 | } 27 | 28 | func (s *mysqlMetaStore) ReadManifest(manifestId string) (*manifest, error) { 29 | manifest := NewManifest(DefaultChunkSizeMaxBytes) // FIXME 30 | rows, err := s.db.Query( 31 | "SELECT checksum, chunkOffset, chunkLength, kind FROM manifest WHERE manifestId = ? ORDER BY offset ASC", 32 | manifestId) 33 | if err != nil { 34 | return nil, err 35 | } 36 | defer rows.Close() 37 | 38 | diskfrom := int64(0) 39 | for rows.Next() { 40 | var chunkOffset, chunkLength int64 41 | var kindStr string 42 | var checksum sql.NullString 43 | 44 | if err := rows.Scan(&checksum, &chunkOffset, &chunkLength, &kindStr); err != nil { 45 | return nil, err 46 | } 47 | 48 | kind, err := kindFromString(kindStr) 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | var checksumBytes []byte 54 | if checksum.Valid { 55 | checksumBytes, err = hex.DecodeString(checksum.String) 56 | if err != nil { 57 | return nil, err 58 | } 59 | } 60 | 61 | manifest.Add(&chunkSlice{ 62 | checksum: checksumBytes, // Can be nil! 63 | kind: kind, 64 | diskfrom: diskfrom, 65 | diskto: diskfrom + chunkLength, 66 | chunkfrom: chunkOffset, 67 | chunkto: chunkOffset + chunkLength, 68 | length: chunkLength, 69 | }) 70 | 71 | diskfrom += chunkLength 72 | } 73 | 74 | return manifest, nil 75 | } 76 | 77 | func (s* mysqlMetaStore) WriteManifest(manifestId string, manifest *manifest) error { 78 | tx, err := s.db.Begin() 79 | if err != nil { 80 | return err 81 | } 82 | defer tx.Rollback() 83 | 84 | stmt, err := tx.Prepare(` 85 | INSERT INTO manifest 86 | SET 87 | manifestId = ?, 88 | offset = ?, 89 | checksum = ?, 90 | chunkOffset = ?, 91 | chunkLength = ?, 92 | kind = ? 93 | `) 94 | if err != nil { 95 | return err 96 | } 97 | defer stmt.Close() // danger! 98 | 99 | for _, offset := range manifest.Offsets() { 100 | slice := manifest.Get(offset) 101 | if slice.kind == kindSparse { 102 | _, err = stmt.Exec(manifestId, offset, &sql.NullString{}, 103 | slice.chunkfrom, slice.length, slice.kind.toString()) 104 | } else { 105 | _, err = stmt.Exec(manifestId, offset, fmt.Sprintf("%x", slice.checksum), 106 | slice.chunkfrom, slice.length, slice.kind.toString()) 107 | } 108 | if err != nil { 109 | return err 110 | } 111 | } 112 | 113 | return tx.Commit() 114 | } 115 | -------------------------------------------------------------------------------- /meta_store_remote.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "context" 5 | "google.golang.org/grpc" 6 | "heckel.io/fsdup/pb" 7 | "sync" 8 | ) 9 | 10 | type remoteMetaStore struct { 11 | serverAddr string 12 | client pb.HubClient 13 | sync.Mutex 14 | } 15 | 16 | func NewRemoteMetaStore(serverAddr string) *remoteMetaStore { 17 | debugf("NewRemoteMetaStore(%s)", serverAddr) 18 | 19 | return &remoteMetaStore{ 20 | serverAddr: serverAddr, 21 | client: nil, 22 | } 23 | } 24 | 25 | func (s *remoteMetaStore) ReadManifest(manifestId string) (*manifest, error) { 26 | if err := s.ensureConnected(); err != nil { 27 | return nil, err 28 | } 29 | 30 | debugf("remoteMetaStore.ReadManifest(%s)", manifestId) 31 | 32 | response, err := s.client.ReadManifest(context.Background(), &pb.ReadManifestRequest{Id: manifestId}) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | manifest, err := NewManifestFromProto(response.Manifest) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | return manifest, nil 43 | } 44 | 45 | func (s *remoteMetaStore) WriteManifest(manifestId string, manifest *manifest) error { 46 | if err := s.ensureConnected(); err != nil { 47 | return err 48 | } 49 | 50 | debugf("remoteMetaStore.WriteManifest(%s)", manifestId) 51 | 52 | _, err := s.client.WriteManifest(context.Background(), &pb.WriteManifestRequest{Id: manifestId, Manifest: manifest.Proto()}) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | return nil 58 | } 59 | 60 | func (s *remoteMetaStore) ensureConnected() error { 61 | s.Lock() 62 | defer s.Unlock() 63 | 64 | if s.client != nil { 65 | return nil 66 | } 67 | 68 | conn, err := grpc.Dial(s.serverAddr, grpc.WithBlock(), grpc.WithInsecure(), 69 | grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(128 * 1024 * 1024), grpc.MaxCallRecvMsgSize(128 * 1024 * 1024))) // FIXME 70 | if err != nil { 71 | return err 72 | } 73 | 74 | s.client = pb.NewHubClient(conn) 75 | 76 | return nil 77 | } -------------------------------------------------------------------------------- /pb/manifest.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // source: pb/manifest.proto 3 | 4 | package pb 5 | 6 | import ( 7 | fmt "fmt" 8 | proto "github.com/golang/protobuf/proto" 9 | math "math" 10 | ) 11 | 12 | // Reference imports to suppress errors if they are not otherwise used. 13 | var _ = proto.Marshal 14 | var _ = fmt.Errorf 15 | var _ = math.Inf 16 | 17 | // This is a compile-time assertion to ensure that this generated file 18 | // is compatible with the proto package it is being compiled against. 19 | // A compilation error at this line likely means your copy of the 20 | // proto package needs to be updated. 21 | const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package 22 | 23 | type Slice struct { 24 | Checksum []byte `protobuf:"bytes,1,opt,name=Checksum,proto3" json:"Checksum,omitempty"` 25 | Offset int64 `protobuf:"varint,2,opt,name=Offset,proto3" json:"Offset,omitempty"` 26 | Length int64 `protobuf:"varint,3,opt,name=Length,proto3" json:"Length,omitempty"` 27 | Kind int32 `protobuf:"varint,4,opt,name=Kind,proto3" json:"Kind,omitempty"` 28 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 29 | XXX_unrecognized []byte `json:"-"` 30 | XXX_sizecache int32 `json:"-"` 31 | } 32 | 33 | func (m *Slice) Reset() { *m = Slice{} } 34 | func (m *Slice) String() string { return proto.CompactTextString(m) } 35 | func (*Slice) ProtoMessage() {} 36 | func (*Slice) Descriptor() ([]byte, []int) { 37 | return fileDescriptor_2539f5857e72bfb4, []int{0} 38 | } 39 | 40 | func (m *Slice) XXX_Unmarshal(b []byte) error { 41 | return xxx_messageInfo_Slice.Unmarshal(m, b) 42 | } 43 | func (m *Slice) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 44 | return xxx_messageInfo_Slice.Marshal(b, m, deterministic) 45 | } 46 | func (m *Slice) XXX_Merge(src proto.Message) { 47 | xxx_messageInfo_Slice.Merge(m, src) 48 | } 49 | func (m *Slice) XXX_Size() int { 50 | return xxx_messageInfo_Slice.Size(m) 51 | } 52 | func (m *Slice) XXX_DiscardUnknown() { 53 | xxx_messageInfo_Slice.DiscardUnknown(m) 54 | } 55 | 56 | var xxx_messageInfo_Slice proto.InternalMessageInfo 57 | 58 | func (m *Slice) GetChecksum() []byte { 59 | if m != nil { 60 | return m.Checksum 61 | } 62 | return nil 63 | } 64 | 65 | func (m *Slice) GetOffset() int64 { 66 | if m != nil { 67 | return m.Offset 68 | } 69 | return 0 70 | } 71 | 72 | func (m *Slice) GetLength() int64 { 73 | if m != nil { 74 | return m.Length 75 | } 76 | return 0 77 | } 78 | 79 | func (m *Slice) GetKind() int32 { 80 | if m != nil { 81 | return m.Kind 82 | } 83 | return 0 84 | } 85 | 86 | type ManifestV1 struct { 87 | Version int64 `protobuf:"varint,5,opt,name=Version,proto3" json:"Version,omitempty"` 88 | Id string `protobuf:"bytes,3,opt,name=Id,proto3" json:"Id,omitempty"` 89 | Size int64 `protobuf:"varint,1,opt,name=Size,proto3" json:"Size,omitempty"` 90 | Slices []*Slice `protobuf:"bytes,2,rep,name=Slices,proto3" json:"Slices,omitempty"` 91 | ChunkMaxSize int64 `protobuf:"varint,4,opt,name=ChunkMaxSize,proto3" json:"ChunkMaxSize,omitempty"` 92 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 93 | XXX_unrecognized []byte `json:"-"` 94 | XXX_sizecache int32 `json:"-"` 95 | } 96 | 97 | func (m *ManifestV1) Reset() { *m = ManifestV1{} } 98 | func (m *ManifestV1) String() string { return proto.CompactTextString(m) } 99 | func (*ManifestV1) ProtoMessage() {} 100 | func (*ManifestV1) Descriptor() ([]byte, []int) { 101 | return fileDescriptor_2539f5857e72bfb4, []int{1} 102 | } 103 | 104 | func (m *ManifestV1) XXX_Unmarshal(b []byte) error { 105 | return xxx_messageInfo_ManifestV1.Unmarshal(m, b) 106 | } 107 | func (m *ManifestV1) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 108 | return xxx_messageInfo_ManifestV1.Marshal(b, m, deterministic) 109 | } 110 | func (m *ManifestV1) XXX_Merge(src proto.Message) { 111 | xxx_messageInfo_ManifestV1.Merge(m, src) 112 | } 113 | func (m *ManifestV1) XXX_Size() int { 114 | return xxx_messageInfo_ManifestV1.Size(m) 115 | } 116 | func (m *ManifestV1) XXX_DiscardUnknown() { 117 | xxx_messageInfo_ManifestV1.DiscardUnknown(m) 118 | } 119 | 120 | var xxx_messageInfo_ManifestV1 proto.InternalMessageInfo 121 | 122 | func (m *ManifestV1) GetVersion() int64 { 123 | if m != nil { 124 | return m.Version 125 | } 126 | return 0 127 | } 128 | 129 | func (m *ManifestV1) GetId() string { 130 | if m != nil { 131 | return m.Id 132 | } 133 | return "" 134 | } 135 | 136 | func (m *ManifestV1) GetSize() int64 { 137 | if m != nil { 138 | return m.Size 139 | } 140 | return 0 141 | } 142 | 143 | func (m *ManifestV1) GetSlices() []*Slice { 144 | if m != nil { 145 | return m.Slices 146 | } 147 | return nil 148 | } 149 | 150 | func (m *ManifestV1) GetChunkMaxSize() int64 { 151 | if m != nil { 152 | return m.ChunkMaxSize 153 | } 154 | return 0 155 | } 156 | 157 | func init() { 158 | proto.RegisterType((*Slice)(nil), "pb.Slice") 159 | proto.RegisterType((*ManifestV1)(nil), "pb.ManifestV1") 160 | } 161 | 162 | func init() { proto.RegisterFile("pb/manifest.proto", fileDescriptor_2539f5857e72bfb4) } 163 | 164 | var fileDescriptor_2539f5857e72bfb4 = []byte{ 165 | // 225 bytes of a gzipped FileDescriptorProto 166 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x90, 0xc1, 0x4a, 0x03, 0x31, 167 | 0x10, 0x86, 0x49, 0xb2, 0xbb, 0xda, 0xb1, 0x08, 0xce, 0x41, 0x82, 0xa7, 0xb8, 0xa7, 0x9c, 0x56, 168 | 0xd4, 0x47, 0xe8, 0xa9, 0x68, 0x11, 0x52, 0xe8, 0xbd, 0xe9, 0x66, 0xbb, 0xa1, 0x36, 0x1b, 0x9a, 169 | 0x14, 0xc4, 0xb7, 0xf0, 0x8d, 0xa5, 0xd3, 0x55, 0xe8, 0x6d, 0xbe, 0x2f, 0xfc, 0x93, 0x9f, 0x81, 170 | 0xbb, 0x68, 0x9f, 0xf6, 0xeb, 0xe0, 0x3b, 0x97, 0x72, 0x13, 0x0f, 0x43, 0x1e, 0x90, 0x47, 0x5b, 171 | 0x6f, 0xa1, 0x5c, 0x7e, 0xfa, 0x8d, 0xc3, 0x07, 0xb8, 0x9e, 0xf5, 0x6e, 0xb3, 0x4b, 0xc7, 0xbd, 172 | 0x64, 0x8a, 0xe9, 0xa9, 0xf9, 0x67, 0xbc, 0x87, 0xea, 0xa3, 0xeb, 0x92, 0xcb, 0x92, 0x2b, 0xa6, 173 | 0x85, 0x19, 0xe9, 0xe4, 0xdf, 0x5d, 0xd8, 0xe6, 0x5e, 0x8a, 0xb3, 0x3f, 0x13, 0x22, 0x14, 0x6f, 174 | 0x3e, 0xb4, 0xb2, 0x50, 0x4c, 0x97, 0x86, 0xe6, 0xfa, 0x87, 0x01, 0x2c, 0xc6, 0xff, 0x57, 0xcf, 175 | 0x28, 0xe1, 0x6a, 0xe5, 0x0e, 0xc9, 0x0f, 0x41, 0x96, 0x94, 0xfd, 0x43, 0xbc, 0x05, 0x3e, 0x6f, 176 | 0x69, 0xe1, 0xc4, 0xf0, 0x79, 0x7b, 0x5a, 0xb6, 0xf4, 0xdf, 0x8e, 0x4a, 0x09, 0x43, 0x33, 0x3e, 177 | 0x42, 0x45, 0xad, 0x93, 0xe4, 0x4a, 0xe8, 0x9b, 0x97, 0x49, 0x13, 0x6d, 0x43, 0xc6, 0x8c, 0x0f, 178 | 0x58, 0xc3, 0x74, 0xd6, 0x1f, 0xc3, 0x6e, 0xb1, 0xfe, 0xa2, 0x78, 0x41, 0xf1, 0x0b, 0x67, 0x2b, 179 | 0xba, 0xc3, 0xeb, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa2, 0x15, 0x7b, 0x19, 0x1c, 0x01, 0x00, 180 | 0x00, 181 | } 182 | -------------------------------------------------------------------------------- /pb/manifest.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package pb; 3 | 4 | message Slice { 5 | bytes Checksum = 1; 6 | int64 Offset = 2; 7 | int64 Length = 3; 8 | int32 Kind = 4; 9 | } 10 | 11 | message ManifestV1 { 12 | int64 Version = 5; 13 | string Id = 3; 14 | int64 Size = 1; 15 | repeated Slice Slices = 2; 16 | int64 ChunkMaxSize = 4; 17 | } -------------------------------------------------------------------------------- /pb/service.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // source: pb/service.proto 3 | 4 | package pb 5 | 6 | import ( 7 | context "context" 8 | fmt "fmt" 9 | proto "github.com/golang/protobuf/proto" 10 | grpc "google.golang.org/grpc" 11 | codes "google.golang.org/grpc/codes" 12 | status "google.golang.org/grpc/status" 13 | math "math" 14 | ) 15 | 16 | // Reference imports to suppress errors if they are not otherwise used. 17 | var _ = proto.Marshal 18 | var _ = fmt.Errorf 19 | var _ = math.Inf 20 | 21 | // This is a compile-time assertion to ensure that this generated file 22 | // is compatible with the proto package it is being compiled against. 23 | // A compilation error at this line likely means your copy of the 24 | // proto package needs to be updated. 25 | const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package 26 | 27 | type DiffRequest struct { 28 | Id string `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"` 29 | Checksums [][]byte `protobuf:"bytes,2,rep,name=Checksums,proto3" json:"Checksums,omitempty"` 30 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 31 | XXX_unrecognized []byte `json:"-"` 32 | XXX_sizecache int32 `json:"-"` 33 | } 34 | 35 | func (m *DiffRequest) Reset() { *m = DiffRequest{} } 36 | func (m *DiffRequest) String() string { return proto.CompactTextString(m) } 37 | func (*DiffRequest) ProtoMessage() {} 38 | func (*DiffRequest) Descriptor() ([]byte, []int) { 39 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{0} 40 | } 41 | 42 | func (m *DiffRequest) XXX_Unmarshal(b []byte) error { 43 | return xxx_messageInfo_DiffRequest.Unmarshal(m, b) 44 | } 45 | func (m *DiffRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 46 | return xxx_messageInfo_DiffRequest.Marshal(b, m, deterministic) 47 | } 48 | func (m *DiffRequest) XXX_Merge(src proto.Message) { 49 | xxx_messageInfo_DiffRequest.Merge(m, src) 50 | } 51 | func (m *DiffRequest) XXX_Size() int { 52 | return xxx_messageInfo_DiffRequest.Size(m) 53 | } 54 | func (m *DiffRequest) XXX_DiscardUnknown() { 55 | xxx_messageInfo_DiffRequest.DiscardUnknown(m) 56 | } 57 | 58 | var xxx_messageInfo_DiffRequest proto.InternalMessageInfo 59 | 60 | func (m *DiffRequest) GetId() string { 61 | if m != nil { 62 | return m.Id 63 | } 64 | return "" 65 | } 66 | 67 | func (m *DiffRequest) GetChecksums() [][]byte { 68 | if m != nil { 69 | return m.Checksums 70 | } 71 | return nil 72 | } 73 | 74 | type DiffResponse struct { 75 | UnknownChecksums [][]byte `protobuf:"bytes,1,rep,name=UnknownChecksums,proto3" json:"UnknownChecksums,omitempty"` 76 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 77 | XXX_unrecognized []byte `json:"-"` 78 | XXX_sizecache int32 `json:"-"` 79 | } 80 | 81 | func (m *DiffResponse) Reset() { *m = DiffResponse{} } 82 | func (m *DiffResponse) String() string { return proto.CompactTextString(m) } 83 | func (*DiffResponse) ProtoMessage() {} 84 | func (*DiffResponse) Descriptor() ([]byte, []int) { 85 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{1} 86 | } 87 | 88 | func (m *DiffResponse) XXX_Unmarshal(b []byte) error { 89 | return xxx_messageInfo_DiffResponse.Unmarshal(m, b) 90 | } 91 | func (m *DiffResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 92 | return xxx_messageInfo_DiffResponse.Marshal(b, m, deterministic) 93 | } 94 | func (m *DiffResponse) XXX_Merge(src proto.Message) { 95 | xxx_messageInfo_DiffResponse.Merge(m, src) 96 | } 97 | func (m *DiffResponse) XXX_Size() int { 98 | return xxx_messageInfo_DiffResponse.Size(m) 99 | } 100 | func (m *DiffResponse) XXX_DiscardUnknown() { 101 | xxx_messageInfo_DiffResponse.DiscardUnknown(m) 102 | } 103 | 104 | var xxx_messageInfo_DiffResponse proto.InternalMessageInfo 105 | 106 | func (m *DiffResponse) GetUnknownChecksums() [][]byte { 107 | if m != nil { 108 | return m.UnknownChecksums 109 | } 110 | return nil 111 | } 112 | 113 | type WriteChunkRequest struct { 114 | Checksum []byte `protobuf:"bytes,1,opt,name=Checksum,proto3" json:"Checksum,omitempty"` 115 | Data []byte `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"` 116 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 117 | XXX_unrecognized []byte `json:"-"` 118 | XXX_sizecache int32 `json:"-"` 119 | } 120 | 121 | func (m *WriteChunkRequest) Reset() { *m = WriteChunkRequest{} } 122 | func (m *WriteChunkRequest) String() string { return proto.CompactTextString(m) } 123 | func (*WriteChunkRequest) ProtoMessage() {} 124 | func (*WriteChunkRequest) Descriptor() ([]byte, []int) { 125 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{2} 126 | } 127 | 128 | func (m *WriteChunkRequest) XXX_Unmarshal(b []byte) error { 129 | return xxx_messageInfo_WriteChunkRequest.Unmarshal(m, b) 130 | } 131 | func (m *WriteChunkRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 132 | return xxx_messageInfo_WriteChunkRequest.Marshal(b, m, deterministic) 133 | } 134 | func (m *WriteChunkRequest) XXX_Merge(src proto.Message) { 135 | xxx_messageInfo_WriteChunkRequest.Merge(m, src) 136 | } 137 | func (m *WriteChunkRequest) XXX_Size() int { 138 | return xxx_messageInfo_WriteChunkRequest.Size(m) 139 | } 140 | func (m *WriteChunkRequest) XXX_DiscardUnknown() { 141 | xxx_messageInfo_WriteChunkRequest.DiscardUnknown(m) 142 | } 143 | 144 | var xxx_messageInfo_WriteChunkRequest proto.InternalMessageInfo 145 | 146 | func (m *WriteChunkRequest) GetChecksum() []byte { 147 | if m != nil { 148 | return m.Checksum 149 | } 150 | return nil 151 | } 152 | 153 | func (m *WriteChunkRequest) GetData() []byte { 154 | if m != nil { 155 | return m.Data 156 | } 157 | return nil 158 | } 159 | 160 | type WriteChunkResponse struct { 161 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 162 | XXX_unrecognized []byte `json:"-"` 163 | XXX_sizecache int32 `json:"-"` 164 | } 165 | 166 | func (m *WriteChunkResponse) Reset() { *m = WriteChunkResponse{} } 167 | func (m *WriteChunkResponse) String() string { return proto.CompactTextString(m) } 168 | func (*WriteChunkResponse) ProtoMessage() {} 169 | func (*WriteChunkResponse) Descriptor() ([]byte, []int) { 170 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{3} 171 | } 172 | 173 | func (m *WriteChunkResponse) XXX_Unmarshal(b []byte) error { 174 | return xxx_messageInfo_WriteChunkResponse.Unmarshal(m, b) 175 | } 176 | func (m *WriteChunkResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 177 | return xxx_messageInfo_WriteChunkResponse.Marshal(b, m, deterministic) 178 | } 179 | func (m *WriteChunkResponse) XXX_Merge(src proto.Message) { 180 | xxx_messageInfo_WriteChunkResponse.Merge(m, src) 181 | } 182 | func (m *WriteChunkResponse) XXX_Size() int { 183 | return xxx_messageInfo_WriteChunkResponse.Size(m) 184 | } 185 | func (m *WriteChunkResponse) XXX_DiscardUnknown() { 186 | xxx_messageInfo_WriteChunkResponse.DiscardUnknown(m) 187 | } 188 | 189 | var xxx_messageInfo_WriteChunkResponse proto.InternalMessageInfo 190 | 191 | type ReadChunkRequest struct { 192 | Checksum []byte `protobuf:"bytes,1,opt,name=Checksum,proto3" json:"Checksum,omitempty"` 193 | Offset int64 `protobuf:"varint,2,opt,name=Offset,proto3" json:"Offset,omitempty"` 194 | Length int64 `protobuf:"varint,3,opt,name=Length,proto3" json:"Length,omitempty"` 195 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 196 | XXX_unrecognized []byte `json:"-"` 197 | XXX_sizecache int32 `json:"-"` 198 | } 199 | 200 | func (m *ReadChunkRequest) Reset() { *m = ReadChunkRequest{} } 201 | func (m *ReadChunkRequest) String() string { return proto.CompactTextString(m) } 202 | func (*ReadChunkRequest) ProtoMessage() {} 203 | func (*ReadChunkRequest) Descriptor() ([]byte, []int) { 204 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{4} 205 | } 206 | 207 | func (m *ReadChunkRequest) XXX_Unmarshal(b []byte) error { 208 | return xxx_messageInfo_ReadChunkRequest.Unmarshal(m, b) 209 | } 210 | func (m *ReadChunkRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 211 | return xxx_messageInfo_ReadChunkRequest.Marshal(b, m, deterministic) 212 | } 213 | func (m *ReadChunkRequest) XXX_Merge(src proto.Message) { 214 | xxx_messageInfo_ReadChunkRequest.Merge(m, src) 215 | } 216 | func (m *ReadChunkRequest) XXX_Size() int { 217 | return xxx_messageInfo_ReadChunkRequest.Size(m) 218 | } 219 | func (m *ReadChunkRequest) XXX_DiscardUnknown() { 220 | xxx_messageInfo_ReadChunkRequest.DiscardUnknown(m) 221 | } 222 | 223 | var xxx_messageInfo_ReadChunkRequest proto.InternalMessageInfo 224 | 225 | func (m *ReadChunkRequest) GetChecksum() []byte { 226 | if m != nil { 227 | return m.Checksum 228 | } 229 | return nil 230 | } 231 | 232 | func (m *ReadChunkRequest) GetOffset() int64 { 233 | if m != nil { 234 | return m.Offset 235 | } 236 | return 0 237 | } 238 | 239 | func (m *ReadChunkRequest) GetLength() int64 { 240 | if m != nil { 241 | return m.Length 242 | } 243 | return 0 244 | } 245 | 246 | type ReadChunkResponse struct { 247 | Data []byte `protobuf:"bytes,1,opt,name=Data,proto3" json:"Data,omitempty"` 248 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 249 | XXX_unrecognized []byte `json:"-"` 250 | XXX_sizecache int32 `json:"-"` 251 | } 252 | 253 | func (m *ReadChunkResponse) Reset() { *m = ReadChunkResponse{} } 254 | func (m *ReadChunkResponse) String() string { return proto.CompactTextString(m) } 255 | func (*ReadChunkResponse) ProtoMessage() {} 256 | func (*ReadChunkResponse) Descriptor() ([]byte, []int) { 257 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{5} 258 | } 259 | 260 | func (m *ReadChunkResponse) XXX_Unmarshal(b []byte) error { 261 | return xxx_messageInfo_ReadChunkResponse.Unmarshal(m, b) 262 | } 263 | func (m *ReadChunkResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 264 | return xxx_messageInfo_ReadChunkResponse.Marshal(b, m, deterministic) 265 | } 266 | func (m *ReadChunkResponse) XXX_Merge(src proto.Message) { 267 | xxx_messageInfo_ReadChunkResponse.Merge(m, src) 268 | } 269 | func (m *ReadChunkResponse) XXX_Size() int { 270 | return xxx_messageInfo_ReadChunkResponse.Size(m) 271 | } 272 | func (m *ReadChunkResponse) XXX_DiscardUnknown() { 273 | xxx_messageInfo_ReadChunkResponse.DiscardUnknown(m) 274 | } 275 | 276 | var xxx_messageInfo_ReadChunkResponse proto.InternalMessageInfo 277 | 278 | func (m *ReadChunkResponse) GetData() []byte { 279 | if m != nil { 280 | return m.Data 281 | } 282 | return nil 283 | } 284 | 285 | type StatChunkRequest struct { 286 | Checksum []byte `protobuf:"bytes,1,opt,name=Checksum,proto3" json:"Checksum,omitempty"` 287 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 288 | XXX_unrecognized []byte `json:"-"` 289 | XXX_sizecache int32 `json:"-"` 290 | } 291 | 292 | func (m *StatChunkRequest) Reset() { *m = StatChunkRequest{} } 293 | func (m *StatChunkRequest) String() string { return proto.CompactTextString(m) } 294 | func (*StatChunkRequest) ProtoMessage() {} 295 | func (*StatChunkRequest) Descriptor() ([]byte, []int) { 296 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{6} 297 | } 298 | 299 | func (m *StatChunkRequest) XXX_Unmarshal(b []byte) error { 300 | return xxx_messageInfo_StatChunkRequest.Unmarshal(m, b) 301 | } 302 | func (m *StatChunkRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 303 | return xxx_messageInfo_StatChunkRequest.Marshal(b, m, deterministic) 304 | } 305 | func (m *StatChunkRequest) XXX_Merge(src proto.Message) { 306 | xxx_messageInfo_StatChunkRequest.Merge(m, src) 307 | } 308 | func (m *StatChunkRequest) XXX_Size() int { 309 | return xxx_messageInfo_StatChunkRequest.Size(m) 310 | } 311 | func (m *StatChunkRequest) XXX_DiscardUnknown() { 312 | xxx_messageInfo_StatChunkRequest.DiscardUnknown(m) 313 | } 314 | 315 | var xxx_messageInfo_StatChunkRequest proto.InternalMessageInfo 316 | 317 | func (m *StatChunkRequest) GetChecksum() []byte { 318 | if m != nil { 319 | return m.Checksum 320 | } 321 | return nil 322 | } 323 | 324 | type StatChunkResponse struct { 325 | Exists bool `protobuf:"varint,1,opt,name=Exists,proto3" json:"Exists,omitempty"` 326 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 327 | XXX_unrecognized []byte `json:"-"` 328 | XXX_sizecache int32 `json:"-"` 329 | } 330 | 331 | func (m *StatChunkResponse) Reset() { *m = StatChunkResponse{} } 332 | func (m *StatChunkResponse) String() string { return proto.CompactTextString(m) } 333 | func (*StatChunkResponse) ProtoMessage() {} 334 | func (*StatChunkResponse) Descriptor() ([]byte, []int) { 335 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{7} 336 | } 337 | 338 | func (m *StatChunkResponse) XXX_Unmarshal(b []byte) error { 339 | return xxx_messageInfo_StatChunkResponse.Unmarshal(m, b) 340 | } 341 | func (m *StatChunkResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 342 | return xxx_messageInfo_StatChunkResponse.Marshal(b, m, deterministic) 343 | } 344 | func (m *StatChunkResponse) XXX_Merge(src proto.Message) { 345 | xxx_messageInfo_StatChunkResponse.Merge(m, src) 346 | } 347 | func (m *StatChunkResponse) XXX_Size() int { 348 | return xxx_messageInfo_StatChunkResponse.Size(m) 349 | } 350 | func (m *StatChunkResponse) XXX_DiscardUnknown() { 351 | xxx_messageInfo_StatChunkResponse.DiscardUnknown(m) 352 | } 353 | 354 | var xxx_messageInfo_StatChunkResponse proto.InternalMessageInfo 355 | 356 | func (m *StatChunkResponse) GetExists() bool { 357 | if m != nil { 358 | return m.Exists 359 | } 360 | return false 361 | } 362 | 363 | type RemoveChunkRequest struct { 364 | Checksum []byte `protobuf:"bytes,1,opt,name=Checksum,proto3" json:"Checksum,omitempty"` 365 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 366 | XXX_unrecognized []byte `json:"-"` 367 | XXX_sizecache int32 `json:"-"` 368 | } 369 | 370 | func (m *RemoveChunkRequest) Reset() { *m = RemoveChunkRequest{} } 371 | func (m *RemoveChunkRequest) String() string { return proto.CompactTextString(m) } 372 | func (*RemoveChunkRequest) ProtoMessage() {} 373 | func (*RemoveChunkRequest) Descriptor() ([]byte, []int) { 374 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{8} 375 | } 376 | 377 | func (m *RemoveChunkRequest) XXX_Unmarshal(b []byte) error { 378 | return xxx_messageInfo_RemoveChunkRequest.Unmarshal(m, b) 379 | } 380 | func (m *RemoveChunkRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 381 | return xxx_messageInfo_RemoveChunkRequest.Marshal(b, m, deterministic) 382 | } 383 | func (m *RemoveChunkRequest) XXX_Merge(src proto.Message) { 384 | xxx_messageInfo_RemoveChunkRequest.Merge(m, src) 385 | } 386 | func (m *RemoveChunkRequest) XXX_Size() int { 387 | return xxx_messageInfo_RemoveChunkRequest.Size(m) 388 | } 389 | func (m *RemoveChunkRequest) XXX_DiscardUnknown() { 390 | xxx_messageInfo_RemoveChunkRequest.DiscardUnknown(m) 391 | } 392 | 393 | var xxx_messageInfo_RemoveChunkRequest proto.InternalMessageInfo 394 | 395 | func (m *RemoveChunkRequest) GetChecksum() []byte { 396 | if m != nil { 397 | return m.Checksum 398 | } 399 | return nil 400 | } 401 | 402 | type RemoveChunkResponse struct { 403 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 404 | XXX_unrecognized []byte `json:"-"` 405 | XXX_sizecache int32 `json:"-"` 406 | } 407 | 408 | func (m *RemoveChunkResponse) Reset() { *m = RemoveChunkResponse{} } 409 | func (m *RemoveChunkResponse) String() string { return proto.CompactTextString(m) } 410 | func (*RemoveChunkResponse) ProtoMessage() {} 411 | func (*RemoveChunkResponse) Descriptor() ([]byte, []int) { 412 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{9} 413 | } 414 | 415 | func (m *RemoveChunkResponse) XXX_Unmarshal(b []byte) error { 416 | return xxx_messageInfo_RemoveChunkResponse.Unmarshal(m, b) 417 | } 418 | func (m *RemoveChunkResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 419 | return xxx_messageInfo_RemoveChunkResponse.Marshal(b, m, deterministic) 420 | } 421 | func (m *RemoveChunkResponse) XXX_Merge(src proto.Message) { 422 | xxx_messageInfo_RemoveChunkResponse.Merge(m, src) 423 | } 424 | func (m *RemoveChunkResponse) XXX_Size() int { 425 | return xxx_messageInfo_RemoveChunkResponse.Size(m) 426 | } 427 | func (m *RemoveChunkResponse) XXX_DiscardUnknown() { 428 | xxx_messageInfo_RemoveChunkResponse.DiscardUnknown(m) 429 | } 430 | 431 | var xxx_messageInfo_RemoveChunkResponse proto.InternalMessageInfo 432 | 433 | type WriteManifestRequest struct { 434 | Id string `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"` 435 | Manifest *ManifestV1 `protobuf:"bytes,2,opt,name=Manifest,proto3" json:"Manifest,omitempty"` 436 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 437 | XXX_unrecognized []byte `json:"-"` 438 | XXX_sizecache int32 `json:"-"` 439 | } 440 | 441 | func (m *WriteManifestRequest) Reset() { *m = WriteManifestRequest{} } 442 | func (m *WriteManifestRequest) String() string { return proto.CompactTextString(m) } 443 | func (*WriteManifestRequest) ProtoMessage() {} 444 | func (*WriteManifestRequest) Descriptor() ([]byte, []int) { 445 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{10} 446 | } 447 | 448 | func (m *WriteManifestRequest) XXX_Unmarshal(b []byte) error { 449 | return xxx_messageInfo_WriteManifestRequest.Unmarshal(m, b) 450 | } 451 | func (m *WriteManifestRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 452 | return xxx_messageInfo_WriteManifestRequest.Marshal(b, m, deterministic) 453 | } 454 | func (m *WriteManifestRequest) XXX_Merge(src proto.Message) { 455 | xxx_messageInfo_WriteManifestRequest.Merge(m, src) 456 | } 457 | func (m *WriteManifestRequest) XXX_Size() int { 458 | return xxx_messageInfo_WriteManifestRequest.Size(m) 459 | } 460 | func (m *WriteManifestRequest) XXX_DiscardUnknown() { 461 | xxx_messageInfo_WriteManifestRequest.DiscardUnknown(m) 462 | } 463 | 464 | var xxx_messageInfo_WriteManifestRequest proto.InternalMessageInfo 465 | 466 | func (m *WriteManifestRequest) GetId() string { 467 | if m != nil { 468 | return m.Id 469 | } 470 | return "" 471 | } 472 | 473 | func (m *WriteManifestRequest) GetManifest() *ManifestV1 { 474 | if m != nil { 475 | return m.Manifest 476 | } 477 | return nil 478 | } 479 | 480 | type WriteManifestResponse struct { 481 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 482 | XXX_unrecognized []byte `json:"-"` 483 | XXX_sizecache int32 `json:"-"` 484 | } 485 | 486 | func (m *WriteManifestResponse) Reset() { *m = WriteManifestResponse{} } 487 | func (m *WriteManifestResponse) String() string { return proto.CompactTextString(m) } 488 | func (*WriteManifestResponse) ProtoMessage() {} 489 | func (*WriteManifestResponse) Descriptor() ([]byte, []int) { 490 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{11} 491 | } 492 | 493 | func (m *WriteManifestResponse) XXX_Unmarshal(b []byte) error { 494 | return xxx_messageInfo_WriteManifestResponse.Unmarshal(m, b) 495 | } 496 | func (m *WriteManifestResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 497 | return xxx_messageInfo_WriteManifestResponse.Marshal(b, m, deterministic) 498 | } 499 | func (m *WriteManifestResponse) XXX_Merge(src proto.Message) { 500 | xxx_messageInfo_WriteManifestResponse.Merge(m, src) 501 | } 502 | func (m *WriteManifestResponse) XXX_Size() int { 503 | return xxx_messageInfo_WriteManifestResponse.Size(m) 504 | } 505 | func (m *WriteManifestResponse) XXX_DiscardUnknown() { 506 | xxx_messageInfo_WriteManifestResponse.DiscardUnknown(m) 507 | } 508 | 509 | var xxx_messageInfo_WriteManifestResponse proto.InternalMessageInfo 510 | 511 | type ReadManifestRequest struct { 512 | Id string `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"` 513 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 514 | XXX_unrecognized []byte `json:"-"` 515 | XXX_sizecache int32 `json:"-"` 516 | } 517 | 518 | func (m *ReadManifestRequest) Reset() { *m = ReadManifestRequest{} } 519 | func (m *ReadManifestRequest) String() string { return proto.CompactTextString(m) } 520 | func (*ReadManifestRequest) ProtoMessage() {} 521 | func (*ReadManifestRequest) Descriptor() ([]byte, []int) { 522 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{12} 523 | } 524 | 525 | func (m *ReadManifestRequest) XXX_Unmarshal(b []byte) error { 526 | return xxx_messageInfo_ReadManifestRequest.Unmarshal(m, b) 527 | } 528 | func (m *ReadManifestRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 529 | return xxx_messageInfo_ReadManifestRequest.Marshal(b, m, deterministic) 530 | } 531 | func (m *ReadManifestRequest) XXX_Merge(src proto.Message) { 532 | xxx_messageInfo_ReadManifestRequest.Merge(m, src) 533 | } 534 | func (m *ReadManifestRequest) XXX_Size() int { 535 | return xxx_messageInfo_ReadManifestRequest.Size(m) 536 | } 537 | func (m *ReadManifestRequest) XXX_DiscardUnknown() { 538 | xxx_messageInfo_ReadManifestRequest.DiscardUnknown(m) 539 | } 540 | 541 | var xxx_messageInfo_ReadManifestRequest proto.InternalMessageInfo 542 | 543 | func (m *ReadManifestRequest) GetId() string { 544 | if m != nil { 545 | return m.Id 546 | } 547 | return "" 548 | } 549 | 550 | type ReadManifestResponse struct { 551 | Manifest *ManifestV1 `protobuf:"bytes,1,opt,name=Manifest,proto3" json:"Manifest,omitempty"` 552 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 553 | XXX_unrecognized []byte `json:"-"` 554 | XXX_sizecache int32 `json:"-"` 555 | } 556 | 557 | func (m *ReadManifestResponse) Reset() { *m = ReadManifestResponse{} } 558 | func (m *ReadManifestResponse) String() string { return proto.CompactTextString(m) } 559 | func (*ReadManifestResponse) ProtoMessage() {} 560 | func (*ReadManifestResponse) Descriptor() ([]byte, []int) { 561 | return fileDescriptor_6ff5ab49d8a5fcc4, []int{13} 562 | } 563 | 564 | func (m *ReadManifestResponse) XXX_Unmarshal(b []byte) error { 565 | return xxx_messageInfo_ReadManifestResponse.Unmarshal(m, b) 566 | } 567 | func (m *ReadManifestResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 568 | return xxx_messageInfo_ReadManifestResponse.Marshal(b, m, deterministic) 569 | } 570 | func (m *ReadManifestResponse) XXX_Merge(src proto.Message) { 571 | xxx_messageInfo_ReadManifestResponse.Merge(m, src) 572 | } 573 | func (m *ReadManifestResponse) XXX_Size() int { 574 | return xxx_messageInfo_ReadManifestResponse.Size(m) 575 | } 576 | func (m *ReadManifestResponse) XXX_DiscardUnknown() { 577 | xxx_messageInfo_ReadManifestResponse.DiscardUnknown(m) 578 | } 579 | 580 | var xxx_messageInfo_ReadManifestResponse proto.InternalMessageInfo 581 | 582 | func (m *ReadManifestResponse) GetManifest() *ManifestV1 { 583 | if m != nil { 584 | return m.Manifest 585 | } 586 | return nil 587 | } 588 | 589 | func init() { 590 | proto.RegisterType((*DiffRequest)(nil), "pb.DiffRequest") 591 | proto.RegisterType((*DiffResponse)(nil), "pb.DiffResponse") 592 | proto.RegisterType((*WriteChunkRequest)(nil), "pb.WriteChunkRequest") 593 | proto.RegisterType((*WriteChunkResponse)(nil), "pb.WriteChunkResponse") 594 | proto.RegisterType((*ReadChunkRequest)(nil), "pb.ReadChunkRequest") 595 | proto.RegisterType((*ReadChunkResponse)(nil), "pb.ReadChunkResponse") 596 | proto.RegisterType((*StatChunkRequest)(nil), "pb.StatChunkRequest") 597 | proto.RegisterType((*StatChunkResponse)(nil), "pb.StatChunkResponse") 598 | proto.RegisterType((*RemoveChunkRequest)(nil), "pb.RemoveChunkRequest") 599 | proto.RegisterType((*RemoveChunkResponse)(nil), "pb.RemoveChunkResponse") 600 | proto.RegisterType((*WriteManifestRequest)(nil), "pb.WriteManifestRequest") 601 | proto.RegisterType((*WriteManifestResponse)(nil), "pb.WriteManifestResponse") 602 | proto.RegisterType((*ReadManifestRequest)(nil), "pb.ReadManifestRequest") 603 | proto.RegisterType((*ReadManifestResponse)(nil), "pb.ReadManifestResponse") 604 | } 605 | 606 | func init() { proto.RegisterFile("pb/service.proto", fileDescriptor_6ff5ab49d8a5fcc4) } 607 | 608 | var fileDescriptor_6ff5ab49d8a5fcc4 = []byte{ 609 | // 468 bytes of a gzipped FileDescriptorProto 610 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x5d, 0x8b, 0xd3, 0x40, 611 | 0x14, 0x25, 0xc9, 0x52, 0xda, 0xdb, 0xba, 0xa6, 0x63, 0x92, 0xc6, 0xe0, 0xc3, 0x32, 0x20, 0xae, 612 | 0x2b, 0x64, 0x75, 0x7d, 0x11, 0x17, 0x04, 0x6d, 0x05, 0x17, 0x14, 0x61, 0x44, 0x7d, 0x13, 0x92, 613 | 0xcd, 0xc4, 0x86, 0xd2, 0x24, 0x76, 0x26, 0xab, 0xbf, 0xc9, 0x5f, 0x29, 0x99, 0x99, 0x7c, 0x74, 614 | 0xa2, 0x6e, 0xdf, 0x32, 0xe7, 0xde, 0x73, 0xe6, 0xcc, 0xbd, 0x87, 0x80, 0x5d, 0xc6, 0xe7, 0x8c, 615 | 0xee, 0x6e, 0xb2, 0x6b, 0x1a, 0x96, 0xbb, 0x82, 0x17, 0xc8, 0x2c, 0xe3, 0x60, 0x5e, 0xc6, 0xe7, 616 | 0xdb, 0x28, 0xcf, 0x52, 0xca, 0xb8, 0x84, 0xf1, 0x25, 0x4c, 0x57, 0x59, 0x9a, 0x12, 0xfa, 0xa3, 617 | 0xa2, 0x8c, 0xa3, 0x63, 0x30, 0xaf, 0x12, 0xdf, 0x38, 0x31, 0x4e, 0x27, 0xc4, 0xbc, 0x4a, 0xd0, 618 | 0x03, 0x98, 0x2c, 0xd7, 0xf4, 0x7a, 0xc3, 0xaa, 0x2d, 0xf3, 0xcd, 0x13, 0xeb, 0x74, 0x46, 0x3a, 619 | 0x00, 0xbf, 0x84, 0x99, 0x24, 0xb3, 0xb2, 0xc8, 0x19, 0x45, 0x67, 0x60, 0x7f, 0xce, 0x37, 0x79, 620 | 0xf1, 0x33, 0xef, 0x48, 0x86, 0x20, 0x0d, 0x70, 0xbc, 0x84, 0xf9, 0xd7, 0x5d, 0xc6, 0xe9, 0x72, 621 | 0x5d, 0xe5, 0x9b, 0xe6, 0xfa, 0x00, 0xc6, 0x4d, 0x87, 0x30, 0x31, 0x23, 0xed, 0x19, 0x21, 0x38, 622 | 0x5a, 0x45, 0x3c, 0xf2, 0x4d, 0x81, 0x8b, 0x6f, 0xec, 0x00, 0xea, 0x8b, 0x48, 0x1b, 0xf8, 0x1b, 623 | 0xd8, 0x84, 0x46, 0xc9, 0xc1, 0xca, 0x1e, 0x8c, 0x3e, 0xa6, 0x29, 0xa3, 0x5c, 0x68, 0x5b, 0x44, 624 | 0x9d, 0x6a, 0xfc, 0x3d, 0xcd, 0xbf, 0xf3, 0xb5, 0x6f, 0x49, 0x5c, 0x9e, 0xf0, 0x23, 0x98, 0xf7, 625 | 0xf4, 0xd5, 0xdb, 0x1b, 0x7b, 0x46, 0xcf, 0x5e, 0x08, 0xf6, 0x27, 0x1e, 0xf1, 0x43, 0x8d, 0xe0, 626 | 0x27, 0x30, 0xef, 0xf5, 0x2b, 0x61, 0x0f, 0x46, 0x6f, 0x7f, 0x65, 0x8c, 0x33, 0xd1, 0x3e, 0x26, 627 | 0xea, 0x84, 0x9f, 0x02, 0x22, 0x74, 0x5b, 0xdc, 0x1c, 0x3c, 0x41, 0xec, 0xc2, 0xbd, 0x3d, 0x86, 628 | 0x1a, 0x17, 0x01, 0x47, 0x0c, 0xf1, 0x83, 0x4a, 0xc6, 0xbf, 0xb2, 0x70, 0x06, 0xe3, 0xa6, 0x45, 629 | 0x0c, 0x6a, 0x7a, 0x71, 0x1c, 0x96, 0x71, 0xd8, 0x60, 0x5f, 0x9e, 0x91, 0xb6, 0x8e, 0x17, 0xe0, 630 | 0x6a, 0x9a, 0xea, 0xb2, 0x87, 0xb5, 0x87, 0x28, 0xb9, 0xe5, 0x2e, 0xfc, 0x06, 0x9c, 0xfd, 0xb6, 631 | 0x36, 0x61, 0x9d, 0x07, 0xe3, 0xff, 0x1e, 0x2e, 0x7e, 0x5b, 0x60, 0xbd, 0xab, 0x62, 0xf4, 0x18, 632 | 0x8e, 0xea, 0x94, 0xa2, 0xbb, 0x75, 0x67, 0x2f, 0xec, 0x81, 0xdd, 0x01, 0x4a, 0xfe, 0x12, 0xa0, 633 | 0xcb, 0x13, 0x72, 0xeb, 0xfa, 0x20, 0xa4, 0x81, 0xa7, 0xc3, 0x8a, 0xfc, 0x02, 0x26, 0x6d, 0x2c, 634 | 0x90, 0x53, 0x37, 0xe9, 0x29, 0x0c, 0x5c, 0x0d, 0xed, 0x98, 0xed, 0xde, 0x25, 0x53, 0x8f, 0x8d, 635 | 0x64, 0x0e, 0xc3, 0xf1, 0x0a, 0xa6, 0xbd, 0x95, 0x22, 0x4f, 0xea, 0xeb, 0xa9, 0x08, 0x16, 0x03, 636 | 0x5c, 0xf1, 0x57, 0x70, 0x67, 0x6f, 0x4f, 0xc8, 0x6f, 0x1f, 0xa7, 0xad, 0x28, 0xb8, 0xff, 0x97, 637 | 0x8a, 0x52, 0x79, 0x0d, 0xb3, 0xfe, 0xb6, 0xd0, 0xa2, 0x79, 0xa6, 0xae, 0xe1, 0x0f, 0x0b, 0x52, 638 | 0x22, 0x1e, 0x89, 0xdf, 0xd1, 0xf3, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xdf, 0x7b, 0xe4, 0x16, 639 | 0xb9, 0x04, 0x00, 0x00, 640 | } 641 | 642 | // Reference imports to suppress errors if they are not otherwise used. 643 | var _ context.Context 644 | var _ grpc.ClientConn 645 | 646 | // This is a compile-time assertion to ensure that this generated file 647 | // is compatible with the grpc package it is being compiled against. 648 | const _ = grpc.SupportPackageIsVersion4 649 | 650 | // HubClient is the client API for Hub service. 651 | // 652 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. 653 | type HubClient interface { 654 | Diff(ctx context.Context, in *DiffRequest, opts ...grpc.CallOption) (*DiffResponse, error) 655 | WriteChunk(ctx context.Context, in *WriteChunkRequest, opts ...grpc.CallOption) (*WriteChunkResponse, error) 656 | ReadChunk(ctx context.Context, in *ReadChunkRequest, opts ...grpc.CallOption) (*ReadChunkResponse, error) 657 | StatChunk(ctx context.Context, in *StatChunkRequest, opts ...grpc.CallOption) (*StatChunkResponse, error) 658 | RemoveChunk(ctx context.Context, in *RemoveChunkRequest, opts ...grpc.CallOption) (*RemoveChunkResponse, error) 659 | WriteManifest(ctx context.Context, in *WriteManifestRequest, opts ...grpc.CallOption) (*WriteManifestResponse, error) 660 | ReadManifest(ctx context.Context, in *ReadManifestRequest, opts ...grpc.CallOption) (*ReadManifestResponse, error) 661 | } 662 | 663 | type hubClient struct { 664 | cc *grpc.ClientConn 665 | } 666 | 667 | func NewHubClient(cc *grpc.ClientConn) HubClient { 668 | return &hubClient{cc} 669 | } 670 | 671 | func (c *hubClient) Diff(ctx context.Context, in *DiffRequest, opts ...grpc.CallOption) (*DiffResponse, error) { 672 | out := new(DiffResponse) 673 | err := c.cc.Invoke(ctx, "/pb.Hub/Diff", in, out, opts...) 674 | if err != nil { 675 | return nil, err 676 | } 677 | return out, nil 678 | } 679 | 680 | func (c *hubClient) WriteChunk(ctx context.Context, in *WriteChunkRequest, opts ...grpc.CallOption) (*WriteChunkResponse, error) { 681 | out := new(WriteChunkResponse) 682 | err := c.cc.Invoke(ctx, "/pb.Hub/WriteChunk", in, out, opts...) 683 | if err != nil { 684 | return nil, err 685 | } 686 | return out, nil 687 | } 688 | 689 | func (c *hubClient) ReadChunk(ctx context.Context, in *ReadChunkRequest, opts ...grpc.CallOption) (*ReadChunkResponse, error) { 690 | out := new(ReadChunkResponse) 691 | err := c.cc.Invoke(ctx, "/pb.Hub/ReadChunk", in, out, opts...) 692 | if err != nil { 693 | return nil, err 694 | } 695 | return out, nil 696 | } 697 | 698 | func (c *hubClient) StatChunk(ctx context.Context, in *StatChunkRequest, opts ...grpc.CallOption) (*StatChunkResponse, error) { 699 | out := new(StatChunkResponse) 700 | err := c.cc.Invoke(ctx, "/pb.Hub/StatChunk", in, out, opts...) 701 | if err != nil { 702 | return nil, err 703 | } 704 | return out, nil 705 | } 706 | 707 | func (c *hubClient) RemoveChunk(ctx context.Context, in *RemoveChunkRequest, opts ...grpc.CallOption) (*RemoveChunkResponse, error) { 708 | out := new(RemoveChunkResponse) 709 | err := c.cc.Invoke(ctx, "/pb.Hub/RemoveChunk", in, out, opts...) 710 | if err != nil { 711 | return nil, err 712 | } 713 | return out, nil 714 | } 715 | 716 | func (c *hubClient) WriteManifest(ctx context.Context, in *WriteManifestRequest, opts ...grpc.CallOption) (*WriteManifestResponse, error) { 717 | out := new(WriteManifestResponse) 718 | err := c.cc.Invoke(ctx, "/pb.Hub/WriteManifest", in, out, opts...) 719 | if err != nil { 720 | return nil, err 721 | } 722 | return out, nil 723 | } 724 | 725 | func (c *hubClient) ReadManifest(ctx context.Context, in *ReadManifestRequest, opts ...grpc.CallOption) (*ReadManifestResponse, error) { 726 | out := new(ReadManifestResponse) 727 | err := c.cc.Invoke(ctx, "/pb.Hub/ReadManifest", in, out, opts...) 728 | if err != nil { 729 | return nil, err 730 | } 731 | return out, nil 732 | } 733 | 734 | // HubServer is the server API for Hub service. 735 | type HubServer interface { 736 | Diff(context.Context, *DiffRequest) (*DiffResponse, error) 737 | WriteChunk(context.Context, *WriteChunkRequest) (*WriteChunkResponse, error) 738 | ReadChunk(context.Context, *ReadChunkRequest) (*ReadChunkResponse, error) 739 | StatChunk(context.Context, *StatChunkRequest) (*StatChunkResponse, error) 740 | RemoveChunk(context.Context, *RemoveChunkRequest) (*RemoveChunkResponse, error) 741 | WriteManifest(context.Context, *WriteManifestRequest) (*WriteManifestResponse, error) 742 | ReadManifest(context.Context, *ReadManifestRequest) (*ReadManifestResponse, error) 743 | } 744 | 745 | // UnimplementedHubServer can be embedded to have forward compatible implementations. 746 | type UnimplementedHubServer struct { 747 | } 748 | 749 | func (*UnimplementedHubServer) Diff(ctx context.Context, req *DiffRequest) (*DiffResponse, error) { 750 | return nil, status.Errorf(codes.Unimplemented, "method Diff not implemented") 751 | } 752 | func (*UnimplementedHubServer) WriteChunk(ctx context.Context, req *WriteChunkRequest) (*WriteChunkResponse, error) { 753 | return nil, status.Errorf(codes.Unimplemented, "method WriteChunk not implemented") 754 | } 755 | func (*UnimplementedHubServer) ReadChunk(ctx context.Context, req *ReadChunkRequest) (*ReadChunkResponse, error) { 756 | return nil, status.Errorf(codes.Unimplemented, "method ReadChunk not implemented") 757 | } 758 | func (*UnimplementedHubServer) StatChunk(ctx context.Context, req *StatChunkRequest) (*StatChunkResponse, error) { 759 | return nil, status.Errorf(codes.Unimplemented, "method StatChunk not implemented") 760 | } 761 | func (*UnimplementedHubServer) RemoveChunk(ctx context.Context, req *RemoveChunkRequest) (*RemoveChunkResponse, error) { 762 | return nil, status.Errorf(codes.Unimplemented, "method RemoveChunk not implemented") 763 | } 764 | func (*UnimplementedHubServer) WriteManifest(ctx context.Context, req *WriteManifestRequest) (*WriteManifestResponse, error) { 765 | return nil, status.Errorf(codes.Unimplemented, "method WriteManifest not implemented") 766 | } 767 | func (*UnimplementedHubServer) ReadManifest(ctx context.Context, req *ReadManifestRequest) (*ReadManifestResponse, error) { 768 | return nil, status.Errorf(codes.Unimplemented, "method ReadManifest not implemented") 769 | } 770 | 771 | func RegisterHubServer(s *grpc.Server, srv HubServer) { 772 | s.RegisterService(&_Hub_serviceDesc, srv) 773 | } 774 | 775 | func _Hub_Diff_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 776 | in := new(DiffRequest) 777 | if err := dec(in); err != nil { 778 | return nil, err 779 | } 780 | if interceptor == nil { 781 | return srv.(HubServer).Diff(ctx, in) 782 | } 783 | info := &grpc.UnaryServerInfo{ 784 | Server: srv, 785 | FullMethod: "/pb.Hub/Diff", 786 | } 787 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 788 | return srv.(HubServer).Diff(ctx, req.(*DiffRequest)) 789 | } 790 | return interceptor(ctx, in, info, handler) 791 | } 792 | 793 | func _Hub_WriteChunk_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 794 | in := new(WriteChunkRequest) 795 | if err := dec(in); err != nil { 796 | return nil, err 797 | } 798 | if interceptor == nil { 799 | return srv.(HubServer).WriteChunk(ctx, in) 800 | } 801 | info := &grpc.UnaryServerInfo{ 802 | Server: srv, 803 | FullMethod: "/pb.Hub/WriteChunk", 804 | } 805 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 806 | return srv.(HubServer).WriteChunk(ctx, req.(*WriteChunkRequest)) 807 | } 808 | return interceptor(ctx, in, info, handler) 809 | } 810 | 811 | func _Hub_ReadChunk_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 812 | in := new(ReadChunkRequest) 813 | if err := dec(in); err != nil { 814 | return nil, err 815 | } 816 | if interceptor == nil { 817 | return srv.(HubServer).ReadChunk(ctx, in) 818 | } 819 | info := &grpc.UnaryServerInfo{ 820 | Server: srv, 821 | FullMethod: "/pb.Hub/ReadChunk", 822 | } 823 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 824 | return srv.(HubServer).ReadChunk(ctx, req.(*ReadChunkRequest)) 825 | } 826 | return interceptor(ctx, in, info, handler) 827 | } 828 | 829 | func _Hub_StatChunk_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 830 | in := new(StatChunkRequest) 831 | if err := dec(in); err != nil { 832 | return nil, err 833 | } 834 | if interceptor == nil { 835 | return srv.(HubServer).StatChunk(ctx, in) 836 | } 837 | info := &grpc.UnaryServerInfo{ 838 | Server: srv, 839 | FullMethod: "/pb.Hub/StatChunk", 840 | } 841 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 842 | return srv.(HubServer).StatChunk(ctx, req.(*StatChunkRequest)) 843 | } 844 | return interceptor(ctx, in, info, handler) 845 | } 846 | 847 | func _Hub_RemoveChunk_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 848 | in := new(RemoveChunkRequest) 849 | if err := dec(in); err != nil { 850 | return nil, err 851 | } 852 | if interceptor == nil { 853 | return srv.(HubServer).RemoveChunk(ctx, in) 854 | } 855 | info := &grpc.UnaryServerInfo{ 856 | Server: srv, 857 | FullMethod: "/pb.Hub/RemoveChunk", 858 | } 859 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 860 | return srv.(HubServer).RemoveChunk(ctx, req.(*RemoveChunkRequest)) 861 | } 862 | return interceptor(ctx, in, info, handler) 863 | } 864 | 865 | func _Hub_WriteManifest_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 866 | in := new(WriteManifestRequest) 867 | if err := dec(in); err != nil { 868 | return nil, err 869 | } 870 | if interceptor == nil { 871 | return srv.(HubServer).WriteManifest(ctx, in) 872 | } 873 | info := &grpc.UnaryServerInfo{ 874 | Server: srv, 875 | FullMethod: "/pb.Hub/WriteManifest", 876 | } 877 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 878 | return srv.(HubServer).WriteManifest(ctx, req.(*WriteManifestRequest)) 879 | } 880 | return interceptor(ctx, in, info, handler) 881 | } 882 | 883 | func _Hub_ReadManifest_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 884 | in := new(ReadManifestRequest) 885 | if err := dec(in); err != nil { 886 | return nil, err 887 | } 888 | if interceptor == nil { 889 | return srv.(HubServer).ReadManifest(ctx, in) 890 | } 891 | info := &grpc.UnaryServerInfo{ 892 | Server: srv, 893 | FullMethod: "/pb.Hub/ReadManifest", 894 | } 895 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 896 | return srv.(HubServer).ReadManifest(ctx, req.(*ReadManifestRequest)) 897 | } 898 | return interceptor(ctx, in, info, handler) 899 | } 900 | 901 | var _Hub_serviceDesc = grpc.ServiceDesc{ 902 | ServiceName: "pb.Hub", 903 | HandlerType: (*HubServer)(nil), 904 | Methods: []grpc.MethodDesc{ 905 | { 906 | MethodName: "Diff", 907 | Handler: _Hub_Diff_Handler, 908 | }, 909 | { 910 | MethodName: "WriteChunk", 911 | Handler: _Hub_WriteChunk_Handler, 912 | }, 913 | { 914 | MethodName: "ReadChunk", 915 | Handler: _Hub_ReadChunk_Handler, 916 | }, 917 | { 918 | MethodName: "StatChunk", 919 | Handler: _Hub_StatChunk_Handler, 920 | }, 921 | { 922 | MethodName: "RemoveChunk", 923 | Handler: _Hub_RemoveChunk_Handler, 924 | }, 925 | { 926 | MethodName: "WriteManifest", 927 | Handler: _Hub_WriteManifest_Handler, 928 | }, 929 | { 930 | MethodName: "ReadManifest", 931 | Handler: _Hub_ReadManifest_Handler, 932 | }, 933 | }, 934 | Streams: []grpc.StreamDesc{}, 935 | Metadata: "pb/service.proto", 936 | } 937 | -------------------------------------------------------------------------------- /pb/service.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package pb; 3 | 4 | import "pb/manifest.proto"; 5 | 6 | service Hub { 7 | rpc Diff (DiffRequest) returns (DiffResponse); 8 | rpc WriteChunk (WriteChunkRequest) returns (WriteChunkResponse); 9 | rpc ReadChunk (ReadChunkRequest) returns (ReadChunkResponse); 10 | rpc StatChunk (StatChunkRequest) returns (StatChunkResponse); 11 | rpc RemoveChunk (RemoveChunkRequest) returns (RemoveChunkResponse); 12 | rpc WriteManifest (WriteManifestRequest) returns (WriteManifestResponse); 13 | rpc ReadManifest (ReadManifestRequest) returns (ReadManifestResponse); 14 | } 15 | 16 | message DiffRequest { 17 | string Id = 1; 18 | repeated bytes Checksums = 2; 19 | } 20 | 21 | message DiffResponse { 22 | repeated bytes UnknownChecksums = 1; 23 | } 24 | 25 | message WriteChunkRequest { 26 | bytes Checksum = 1; 27 | bytes Data = 2; 28 | } 29 | 30 | message WriteChunkResponse { 31 | } 32 | 33 | message ReadChunkRequest { 34 | bytes Checksum = 1; 35 | int64 Offset = 2; 36 | int64 Length = 3; 37 | } 38 | 39 | message ReadChunkResponse { 40 | bytes Data = 1; 41 | } 42 | 43 | message StatChunkRequest { 44 | bytes Checksum = 1; 45 | } 46 | 47 | message StatChunkResponse { 48 | bool Exists = 1; 49 | } 50 | 51 | message RemoveChunkRequest { 52 | bytes Checksum = 1; 53 | } 54 | 55 | message RemoveChunkResponse { 56 | } 57 | 58 | message WriteManifestRequest { 59 | string Id = 1; 60 | ManifestV1 Manifest = 2; 61 | } 62 | 63 | message WriteManifestResponse { 64 | } 65 | 66 | message ReadManifestRequest { 67 | string Id = 1; 68 | } 69 | 70 | message ReadManifestResponse { 71 | ManifestV1 Manifest = 1; 72 | } -------------------------------------------------------------------------------- /server.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "context" 5 | _ "github.com/go-sql-driver/mysql" 6 | "google.golang.org/grpc" 7 | "heckel.io/fsdup/pb" 8 | "net" 9 | ) 10 | 11 | func ListenAndServe(address string, store ChunkStore, metaStore MetaStore) error { 12 | srv := &server{ 13 | store: store, 14 | metaStore: metaStore, 15 | } 16 | 17 | listener, err := net.Listen("tcp", address) 18 | if err != nil { 19 | return err 20 | } 21 | 22 | grpcServer := grpc.NewServer(grpc.MaxSendMsgSize(128 * 1024 * 1024), grpc.MaxRecvMsgSize(128 * 1024 * 1024)) // FIXME 23 | pb.RegisterHubServer(grpcServer, srv) 24 | 25 | return grpcServer.Serve(listener) 26 | } 27 | 28 | type server struct { 29 | store ChunkStore 30 | metaStore MetaStore 31 | } 32 | 33 | func (s *server) Diff(ctx context.Context, request *pb.DiffRequest) (*pb.DiffResponse, error) { 34 | unknownChecksums := make([][]byte, 0) 35 | for _, checksum := range request.Checksums { 36 | if err := s.store.Stat(checksum); err != nil { 37 | unknownChecksums = append(unknownChecksums, checksum) 38 | } 39 | } 40 | 41 | return &pb.DiffResponse{ 42 | UnknownChecksums: unknownChecksums, 43 | }, nil 44 | } 45 | 46 | func (s *server) WriteChunk(ctx context.Context, request *pb.WriteChunkRequest) (*pb.WriteChunkResponse, error) { 47 | err := s.store.Write(request.Checksum, request.Data) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | return &pb.WriteChunkResponse{}, nil 53 | } 54 | 55 | func (s *server) ReadChunk(ctx context.Context, request *pb.ReadChunkRequest) (*pb.ReadChunkResponse, error) { 56 | response := &pb.ReadChunkResponse{ 57 | Data: make([]byte, request.Length), 58 | } 59 | 60 | _, err := s.store.ReadAt(request.Checksum, response.Data, request.Offset) 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | return response, nil 66 | } 67 | 68 | func (s *server) StatChunk(ctx context.Context, request *pb.StatChunkRequest) (*pb.StatChunkResponse, error) { 69 | err := s.store.Stat(request.Checksum) 70 | 71 | // FIXME this API is wrong 72 | if err != nil { 73 | return &pb.StatChunkResponse{Exists: false}, nil 74 | } else { 75 | return &pb.StatChunkResponse{Exists: true}, nil 76 | } 77 | } 78 | 79 | 80 | func (s *server) RemoveChunk(ctx context.Context, request *pb.RemoveChunkRequest) (*pb.RemoveChunkResponse, error) { 81 | debugf("server.RemoveChunk(%x)", request.Checksum) 82 | 83 | err := s.store.Remove(request.Checksum) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | return &pb.RemoveChunkResponse{}, nil 89 | } 90 | 91 | func (s *server) WriteManifest(ctx context.Context, request *pb.WriteManifestRequest) (*pb.WriteManifestResponse, error) { 92 | debugf("server.WriteManifest(%x)", request.Manifest.Id) 93 | 94 | manifest, err := NewManifestFromProto(request.Manifest) 95 | if err != nil { 96 | return nil, err 97 | } 98 | 99 | if err := s.metaStore.WriteManifest(request.Id, manifest); err != nil { 100 | return nil, err 101 | } 102 | 103 | return &pb.WriteManifestResponse{}, nil 104 | } 105 | 106 | func (s *server) ReadManifest(ctx context.Context, request *pb.ReadManifestRequest) (*pb.ReadManifestResponse, error) { 107 | manifest, err := s.metaStore.ReadManifest(request.Id) 108 | if err != nil { 109 | return nil, err 110 | } 111 | 112 | return &pb.ReadManifestResponse{Manifest: manifest.Proto()}, nil 113 | } -------------------------------------------------------------------------------- /stat.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "sort" 7 | ) 8 | 9 | type chunkStat struct { 10 | checksum []byte 11 | size int64 12 | sliceCount int64 13 | sliceSizes int64 14 | kind kind 15 | } 16 | 17 | func Stat(manifestIds []string, metaStore MetaStore, verbose bool) error { 18 | totalImageSize := int64(0) 19 | totalFileSize := int64(0) 20 | totalGapSize := int64(0) 21 | totalUnknownSize := int64(0) 22 | 23 | totalSparseSize := int64(0) 24 | 25 | totalChunkSize := int64(0) 26 | totalFileChunkSize := int64(0) 27 | totalGapChunkSize := int64(0) 28 | totalUnknownChunkSize := int64(0) 29 | 30 | chunkMap := make(map[string]*chunkStat, 0) // checksum -> size 31 | chunkSizes := make([]int64, 0) 32 | chunkStats := make([]*chunkStat, 0) 33 | 34 | for i, manifestId := range manifestIds { 35 | statusf("Reading manifest %d/%d ...", i, len(manifestIds)) 36 | debugf("Reading manifest file %s ...\n", manifestId) 37 | 38 | manifest, err := metaStore.ReadManifest(manifestId) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | usedSize := int64(0) 44 | totalImageSize += manifest.Size() 45 | 46 | for _, offset := range manifest.Offsets() { 47 | slice := manifest.Get(offset) 48 | 49 | // Ignore sparse sections 50 | if slice.checksum == nil { 51 | totalSparseSize += slice.chunkto - slice.chunkfrom 52 | continue 53 | } 54 | 55 | sliceSize := slice.chunkto - slice.chunkfrom 56 | usedSize += sliceSize 57 | 58 | if slice.kind == kindFile { 59 | totalFileSize += sliceSize 60 | } else if slice.kind == kindGap { 61 | totalGapSize += sliceSize 62 | } else { 63 | totalUnknownSize += sliceSize 64 | } 65 | 66 | // This is a weird way to get the chunk size, but hey ... 67 | checksumStr := fmt.Sprintf("%x", slice.checksum) 68 | 69 | if _, ok := chunkMap[checksumStr]; !ok { 70 | chunkMap[checksumStr] = &chunkStat{ 71 | checksum: slice.checksum, 72 | size: slice.chunkto, 73 | sliceCount: 1, 74 | sliceSizes: sliceSize, 75 | kind: slice.kind, // This is inaccurate, because only the first appearance of the chunk is counted! 76 | } 77 | } else { 78 | chunkMap[checksumStr].size = maxInt64(chunkMap[checksumStr].size, slice.chunkto) 79 | chunkMap[checksumStr].sliceCount++ 80 | chunkMap[checksumStr].sliceSizes += sliceSize 81 | } 82 | } 83 | } 84 | 85 | statusf("Crunching numbers ...") 86 | 87 | // Find chunk sizes by type 88 | for _, stat := range chunkMap { 89 | totalChunkSize += stat.size 90 | chunkSizes = append(chunkSizes, stat.size) 91 | chunkStats = append(chunkStats, stat) 92 | 93 | if stat.kind == kindFile { 94 | totalFileChunkSize += stat.size 95 | } else if stat.kind == kindGap { 96 | totalGapChunkSize += stat.size 97 | } else { 98 | totalUnknownChunkSize += stat.size 99 | } 100 | } 101 | 102 | // Find median chunk size 103 | sort.Slice(chunkSizes, func(i, j int) bool { 104 | return chunkSizes[i] < chunkSizes[j] 105 | }) 106 | 107 | medianChunkSize := int64(0) 108 | if len(chunkSizes) % 2 == 0 { 109 | medianChunkSize = chunkSizes[len(chunkSizes)/2] 110 | } else { 111 | medianChunkSize = chunkSizes[(len(chunkSizes)-1)/2] 112 | } 113 | 114 | // Find chunk histogram 115 | sort.Slice(chunkStats, func(i, j int) bool { 116 | return chunkStats[i].sliceSizes > chunkStats[j].sliceSizes 117 | }) 118 | 119 | manifestCount := int64(len(manifestIds)) 120 | chunkCount := int64(len(chunkMap)) 121 | averageChunkSize := int64(math.Round(float64(totalChunkSize) / float64(chunkCount))) 122 | 123 | totalUsedSize := totalImageSize - totalSparseSize 124 | dedupRatio := float64(totalUsedSize) / float64(totalChunkSize) // as x:1 ratio 125 | spaceReductionPercentage := (1 - 1/dedupRatio) * 100 // in % 126 | 127 | statusf("") 128 | 129 | fmt.Printf("Manifests: %d\n", manifestCount) 130 | fmt.Printf("Number of unique chunks: %d\n", chunkCount) 131 | fmt.Printf("Total image size: %s (%d bytes)\n", convertBytesToHumanReadable(totalImageSize), totalImageSize) 132 | fmt.Printf("- Used: %s (%d bytes)\n", convertBytesToHumanReadable(totalUsedSize), totalUsedSize) 133 | fmt.Printf(" - Files: %s (%d bytes)\n", convertBytesToHumanReadable(totalFileSize), totalFileSize) 134 | fmt.Printf(" - Gaps: %s (%d bytes)\n", convertBytesToHumanReadable(totalGapSize), totalGapSize) 135 | fmt.Printf(" - Unknown: %s (%d bytes)\n", convertBytesToHumanReadable(totalUnknownSize), totalUnknownSize) 136 | fmt.Printf("- Sparse/empty: %s (%d bytes)\n", convertBytesToHumanReadable(totalSparseSize), totalSparseSize) 137 | fmt.Printf("Total chunk size: %s (%d bytes)\n", convertBytesToHumanReadable(totalChunkSize), totalChunkSize) 138 | fmt.Printf("- File chunks: %s (%d bytes)\n", convertBytesToHumanReadable(totalFileChunkSize), totalFileChunkSize) 139 | fmt.Printf("- Gap chunks: %s (%d bytes)\n", convertBytesToHumanReadable(totalGapChunkSize), totalGapChunkSize) 140 | fmt.Printf("- Unkown chunks: %s (%d bytes)\n", convertBytesToHumanReadable(totalUnknownChunkSize), totalUnknownChunkSize) 141 | fmt.Printf("Average chunk size: %s (%d bytes)\n", convertBytesToHumanReadable(averageChunkSize), averageChunkSize) 142 | fmt.Printf("Median chunk size: %s (%d bytes)\n", convertBytesToHumanReadable(medianChunkSize), medianChunkSize) 143 | fmt.Printf("Dedup ratio: %.1f : 1\n", dedupRatio) 144 | fmt.Printf("Space reduction: %.1f %%\n", spaceReductionPercentage) 145 | 146 | if verbose { 147 | fmt.Printf("Slice histogram (top 10):\n") 148 | for i, stat := range chunkStats { 149 | fmt.Printf("- Chunk %x: %s in %d slice(s)\n", stat.checksum, convertBytesToHumanReadable(stat.sliceSizes), stat.sliceCount) 150 | if i == 10 { 151 | break 152 | } 153 | } 154 | } 155 | 156 | return nil 157 | } 158 | -------------------------------------------------------------------------------- /store.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | type ChunkStore interface { 4 | Stat(checksum []byte) error 5 | ReadAt(checksum []byte, buffer []byte, offset int64) (int, error) 6 | Write(checksum []byte, buffer []byte) error 7 | Remove(checksum []byte) error 8 | } 9 | -------------------------------------------------------------------------------- /store_ceph.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "fmt" 7 | "github.com/ceph/go-ceph/rados" 8 | "io/ioutil" 9 | ) 10 | 11 | type cephChunkStore struct { 12 | configFile string 13 | pool string 14 | compress bool 15 | ctx *rados.IOContext 16 | chunkMap map[string]bool 17 | buffer []byte 18 | } 19 | 20 | func NewCephStore(configFile string, pool string, compress bool) *cephChunkStore { 21 | return &cephChunkStore{ 22 | configFile: configFile, 23 | pool: pool, 24 | compress: compress, 25 | chunkMap: make(map[string]bool, 0), 26 | buffer: make([]byte, 128 * 1024 * 1024), // FIXME: This is a hack, this means that only chunks <= 128M are supported! 27 | } 28 | } 29 | 30 | func (idx *cephChunkStore) Stat(checksum []byte) error { 31 | if err := idx.openPool(); err != nil { 32 | return err 33 | } 34 | 35 | checksumStr := fmt.Sprintf("%x", checksum) 36 | 37 | if _, ok := idx.chunkMap[checksumStr]; ok { 38 | return nil 39 | } 40 | 41 | _, err := idx.ctx.Stat(checksumStr) 42 | return err 43 | } 44 | 45 | func (idx *cephChunkStore) ReadAt(checksum []byte, buffer []byte, offset int64) (int, error) { 46 | if err := idx.openPool(); err != nil { 47 | return 0, err 48 | } 49 | 50 | checksumStr := fmt.Sprintf("%x", checksum) 51 | 52 | if idx.compress { 53 | // FIXME this reads the ENTIRE object, gunzips it, and then only reads the requested section. 54 | // don't kill me this is a PoC 55 | 56 | read, err := idx.ctx.Read(checksumStr, idx.buffer, 0) 57 | if err != nil { 58 | return 0, err 59 | } 60 | 61 | reader, err := gzip.NewReader(bytes.NewReader(idx.buffer[:read])) 62 | if err != nil { 63 | return 0, err 64 | } 65 | 66 | decompressed, err := ioutil.ReadAll(reader) 67 | if err != nil { 68 | return 0, err 69 | } 70 | 71 | length := len(buffer) 72 | end := int64(offset)+int64(length) 73 | copy(buffer, decompressed[offset:end]) // FIXME urgh ... 74 | 75 | //fmt.Printf("%d - %d = %x\n", from, end, buffer) 76 | return length, nil 77 | 78 | } else { 79 | read, err := idx.ctx.Read(checksumStr, buffer, uint64(offset)) 80 | if err != nil { 81 | return 0, err 82 | } 83 | 84 | return read, nil 85 | } 86 | } 87 | 88 | func (idx *cephChunkStore) Write(checksum []byte, buffer []byte) error { 89 | if err := idx.openPool(); err != nil { 90 | return err 91 | } 92 | 93 | checksumStr := fmt.Sprintf("%x", checksum) 94 | 95 | if _, ok := idx.chunkMap[checksumStr]; !ok { 96 | if _, err := idx.ctx.Stat(checksumStr); err != nil { 97 | if idx.compress { 98 | var b bytes.Buffer 99 | writer := gzip.NewWriter(&b) 100 | 101 | if _, err := writer.Write(buffer); err != nil { 102 | return err 103 | } 104 | 105 | if err := writer.Close(); err != nil { 106 | return err 107 | } 108 | 109 | if err := idx.ctx.Write(checksumStr, b.Bytes(), 0); err != nil { 110 | return err 111 | } 112 | } else { 113 | if err := idx.ctx.Write(checksumStr, buffer, 0); err != nil { 114 | return err 115 | } 116 | } 117 | } 118 | 119 | idx.chunkMap[checksumStr] = true 120 | } 121 | 122 | return nil 123 | } 124 | 125 | func (idx *cephChunkStore) Remove(checksum []byte) error { 126 | if err := idx.openPool(); err != nil { 127 | return err 128 | } 129 | 130 | checksumStr := fmt.Sprintf("%x", checksum) 131 | err := idx.ctx.Delete(checksumStr) 132 | if err != nil { 133 | return err 134 | } 135 | 136 | delete(idx.chunkMap, checksumStr) 137 | 138 | return nil 139 | } 140 | 141 | func (idx *cephChunkStore) openPool() error { 142 | if idx.ctx != nil { 143 | return nil 144 | } 145 | 146 | conn, err := rados.NewConn() 147 | if err != nil { 148 | return err 149 | } 150 | 151 | if err := conn.ReadConfigFile(idx.configFile); err != nil { 152 | return err 153 | } 154 | 155 | if err := conn.Connect(); err != nil { 156 | return err 157 | } 158 | 159 | idx.ctx, err = conn.OpenIOContext(idx.pool) 160 | if err != nil { 161 | return err 162 | } 163 | 164 | return nil 165 | } 166 | -------------------------------------------------------------------------------- /store_dummy.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | type dummyChunkStore struct { 4 | // Hm? 5 | } 6 | 7 | func NewDummyChunkStore() *dummyChunkStore { 8 | return &dummyChunkStore{} 9 | } 10 | 11 | func (idx *dummyChunkStore) Stat(checksum []byte) error { 12 | return nil 13 | } 14 | 15 | func (idx *dummyChunkStore) Write(checksum []byte, buffer []byte) error { 16 | return nil 17 | } 18 | 19 | func (idx *dummyChunkStore) ReadAt(checksum []byte, buffer []byte, offset int64) (int, error) { 20 | return 0, nil 21 | } 22 | 23 | func (idx *dummyChunkStore) Remove(checksum []byte) error { 24 | return nil 25 | } -------------------------------------------------------------------------------- /store_file.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "sync" 9 | ) 10 | 11 | type fileChunkStore struct { 12 | root string 13 | chunkMap *sync.Map 14 | } 15 | 16 | func NewFileChunkStore(root string) *fileChunkStore { 17 | return &fileChunkStore{ 18 | root: root, 19 | chunkMap: &sync.Map{}, 20 | } 21 | } 22 | 23 | func (idx *fileChunkStore) Stat(checksum []byte) error { 24 | checksumStr := fmt.Sprintf("%x", checksum) 25 | 26 | if _, ok := idx.chunkMap.Load(checksumStr); ok { 27 | return nil 28 | } 29 | 30 | dir := fmt.Sprintf("%s/%s/%s", idx.root, checksumStr[0:3], checksumStr[3:6]) 31 | file := fmt.Sprintf("%s/%s", dir, checksumStr) 32 | 33 | _, err := os.Stat(file) 34 | return err 35 | } 36 | 37 | func (idx *fileChunkStore) Write(checksum []byte, buffer []byte) error { 38 | checksumStr := fmt.Sprintf("%x", checksum) 39 | 40 | if _, ok := idx.chunkMap.Load(checksumStr); !ok { 41 | dir := fmt.Sprintf("%s/%s/%s", idx.root, checksumStr[0:3], checksumStr[3:6]) 42 | file := fmt.Sprintf("%s/%s", dir, checksumStr) 43 | 44 | if _, err := os.Stat(file); err != nil { 45 | if err := os.MkdirAll(dir, 0770); err != nil { 46 | return err 47 | } 48 | 49 | err = ioutil.WriteFile(file, buffer, 0666) 50 | if err != nil { 51 | return err 52 | } 53 | } 54 | 55 | idx.chunkMap.Store(checksumStr, true) 56 | } 57 | 58 | return nil 59 | } 60 | 61 | func (idx *fileChunkStore) ReadAt(checksum []byte, buffer []byte, offset int64) (int, error) { 62 | checksumStr := fmt.Sprintf("%x", checksum) 63 | dir := fmt.Sprintf("%s/%s/%s", idx.root, checksumStr[0:3], checksumStr[3:6]) 64 | file := fmt.Sprintf("%s/%s", dir, checksumStr) 65 | 66 | if _, err := os.Stat(file); err != nil { 67 | return 0, err 68 | } 69 | 70 | chunk, err := os.OpenFile(file, os.O_RDONLY, 0666) 71 | if err != nil { 72 | return 0, err 73 | } 74 | defer chunk.Close() 75 | 76 | read, err := chunk.ReadAt(buffer, offset) 77 | if err != nil { 78 | return 0, err 79 | } else if read != len(buffer) { 80 | return 0, errors.New("cannot read full section") 81 | } 82 | 83 | return read, nil 84 | } 85 | 86 | func (idx *fileChunkStore) Remove(checksum []byte) error { 87 | checksumStr := fmt.Sprintf("%x", checksum) 88 | dir1 := fmt.Sprintf("%s/%s", idx.root, checksumStr[0:3]) 89 | dir2 := fmt.Sprintf("%s/%s/%s", idx.root, checksumStr[0:3], checksumStr[3:6]) 90 | file := fmt.Sprintf("%s/%s", dir2, checksumStr) 91 | 92 | if err := os.Remove(file); err != nil { 93 | return err 94 | } 95 | 96 | os.Remove(dir2) 97 | os.Remove(dir1) 98 | 99 | idx.chunkMap.Delete(checksumStr) 100 | 101 | return nil 102 | } -------------------------------------------------------------------------------- /store_gcloud.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "cloud.google.com/go/storage" 5 | "context" 6 | "fmt" 7 | "io" 8 | "sync" 9 | ) 10 | 11 | type gcloudStore struct { 12 | projectID string 13 | bucketName string 14 | 15 | client *storage.Client 16 | bucket *storage.BucketHandle 17 | chunkMap *sync.Map 18 | } 19 | 20 | func NewGcloudStore(projectID string, bucket string) *gcloudStore { 21 | return &gcloudStore{ 22 | projectID: projectID, 23 | bucketName: bucket, 24 | chunkMap: &sync.Map{}, 25 | } 26 | } 27 | 28 | func (idx *gcloudStore) Stat(checksum []byte) error { 29 | if err := idx.createClient(); err != nil { 30 | return err 31 | } 32 | 33 | checksumStr := fmt.Sprintf("%x", checksum) 34 | 35 | if _, ok := idx.chunkMap.Load(checksumStr); ok { 36 | return nil 37 | } 38 | 39 | object := idx.bucket.Object(checksumStr) 40 | _, err := object.Attrs(context.Background()) 41 | 42 | return err 43 | } 44 | 45 | func (idx *gcloudStore) ReadAt(checksum []byte, buffer []byte, offset int64) (int, error) { 46 | if err := idx.createClient(); err != nil { 47 | return 0, err 48 | } 49 | 50 | checksumStr := fmt.Sprintf("%x", checksum) 51 | object := idx.bucket.Object(checksumStr) 52 | reader, err := object.NewRangeReader(context.Background(), offset, offset + int64(len(buffer))) 53 | if err != nil { 54 | return 0, err 55 | } 56 | 57 | read, err := io.ReadFull(reader, buffer) 58 | if err != nil { 59 | return read, err 60 | } 61 | 62 | if err := reader.Close(); err != nil { 63 | return 0, err 64 | } 65 | 66 | return read, nil 67 | } 68 | 69 | func (idx *gcloudStore) Write(checksum []byte, buffer []byte) error { 70 | if err := idx.createClient(); err != nil { 71 | return err 72 | } 73 | 74 | checksumStr := fmt.Sprintf("%x", checksum) 75 | 76 | if _, ok := idx.chunkMap.Load(checksumStr); !ok { 77 | object := idx.bucket.Object(checksumStr) 78 | if _, err := object.Attrs(context.Background()); err != nil { 79 | writer := object.NewWriter(context.Background()) 80 | if _, err := writer.Write(buffer); err != nil { 81 | return err 82 | } 83 | 84 | if err := writer.Close(); err != nil { 85 | return err 86 | } 87 | } 88 | 89 | idx.chunkMap.Store(checksumStr, true) 90 | } 91 | 92 | return nil 93 | } 94 | 95 | func (idx *gcloudStore) Remove(checksum []byte) error { 96 | if err := idx.createClient(); err != nil { 97 | return err 98 | } 99 | 100 | checksumStr := fmt.Sprintf("%x", checksum) 101 | object := idx.bucket.Object(checksumStr) 102 | if err := object.Delete(context.Background()); err != nil { 103 | return err 104 | } 105 | 106 | idx.chunkMap.Delete(checksumStr) 107 | 108 | return nil 109 | } 110 | 111 | func (idx *gcloudStore) createClient() error { 112 | if idx.client != nil { 113 | return nil 114 | } 115 | 116 | ctx := context.Background() 117 | client, err := storage.NewClient(ctx) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | bucket := client.Bucket(idx.bucketName) 123 | _, err = bucket.Attrs(ctx) 124 | if err == storage.ErrBucketNotExist { 125 | if err := bucket.Create(ctx, idx.projectID, nil); err != nil { 126 | return err 127 | } 128 | } 129 | 130 | idx.client = client 131 | idx.bucket = bucket 132 | 133 | return nil 134 | } 135 | -------------------------------------------------------------------------------- /store_remote.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "google.golang.org/grpc" 8 | "heckel.io/fsdup/pb" 9 | "sync" 10 | ) 11 | 12 | type remoteChunkStore struct { 13 | serverAddr string 14 | client pb.HubClient 15 | sync.Mutex 16 | } 17 | 18 | func NewRemoteChunkStore(serverAddr string) *remoteChunkStore { 19 | return &remoteChunkStore{ 20 | serverAddr: serverAddr, 21 | client: nil, 22 | } 23 | } 24 | 25 | func (idx *remoteChunkStore) Stat(checksum []byte) error { 26 | return nil 27 | } 28 | 29 | func (idx *remoteChunkStore) Write(checksum []byte, buffer []byte) error { 30 | if err := idx.ensureConnected(); err != nil { 31 | return err 32 | } 33 | 34 | _, err := idx.client.WriteChunk(context.Background(), &pb.WriteChunkRequest{ 35 | Checksum: checksum, 36 | Data: buffer, 37 | }) 38 | 39 | return err 40 | } 41 | 42 | func (idx *remoteChunkStore) ReadAt(checksum []byte, buffer []byte, offset int64) (int, error) { 43 | if err := idx.ensureConnected(); err != nil { 44 | return 0, err 45 | } 46 | 47 | response, err := idx.client.ReadChunk(context.Background(), &pb.ReadChunkRequest{ 48 | Checksum: checksum, 49 | Offset: offset, 50 | Length: int64(len(buffer)), 51 | }) 52 | 53 | if err != nil { 54 | return 0, err 55 | } 56 | 57 | if len(buffer) != len(response.Data) { 58 | return 0, errors.New(fmt.Sprintf("unexpected chunk returned from server, expected %d bytes, but got %d", 59 | len(buffer), len(response.Data))) 60 | } 61 | 62 | copied := copy(buffer, response.Data) 63 | if copied != len(buffer) { 64 | return copied, errors.New(fmt.Sprintf("could not copy entire response to buffer, only %d bytes copied, but %d bytes requested", 65 | copied, len(buffer))) 66 | } 67 | 68 | return copied, nil 69 | } 70 | 71 | func (idx *remoteChunkStore) Remove(checksum []byte) error { 72 | if err := idx.ensureConnected(); err != nil { 73 | return err 74 | } 75 | 76 | _, err := idx.client.RemoveChunk(context.Background(), &pb.RemoveChunkRequest{ 77 | Checksum: checksum, 78 | }) 79 | 80 | return err 81 | } 82 | 83 | 84 | func (idx *remoteChunkStore) ensureConnected() error { 85 | idx.Lock() 86 | defer idx.Unlock() 87 | 88 | if idx.client != nil { 89 | return nil 90 | } 91 | 92 | conn, err := grpc.Dial(idx.serverAddr, grpc.WithBlock(), grpc.WithInsecure(), 93 | grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(128 * 1024 * 1024), grpc.MaxCallRecvMsgSize(128 * 1024 * 1024))) // FIXME 94 | if err != nil { 95 | return err 96 | } 97 | 98 | idx.client = pb.NewHubClient(conn) 99 | 100 | return nil 101 | } -------------------------------------------------------------------------------- /store_swift.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "github.com/ncw/swift" 8 | "sync" 9 | ) 10 | 11 | type swiftChunkStore struct { 12 | connection *swift.Connection 13 | container string 14 | chunkMap sync.Map 15 | } 16 | 17 | func NewSwiftStore(connection *swift.Connection, container string) *swiftChunkStore { 18 | return &swiftChunkStore{ 19 | connection: connection, 20 | container: container, 21 | chunkMap: sync.Map{}, 22 | } 23 | } 24 | 25 | func (idx *swiftChunkStore) Stat(checksum []byte) error { 26 | if err := idx.openConnection(); err != nil { 27 | return err 28 | } 29 | 30 | checksumStr := fmt.Sprintf("%x", checksum) 31 | 32 | if _, ok := idx.chunkMap.Load(checksumStr); ok { 33 | return nil 34 | } 35 | 36 | _, _, err := idx.connection.Object(idx.container, checksumStr) 37 | if err == nil { 38 | idx.chunkMap.Store(checksumStr, true) 39 | } 40 | 41 | return err 42 | } 43 | 44 | func (idx *swiftChunkStore) ReadAt(checksum []byte, buffer []byte, offset int64) (int, error) { 45 | if err := idx.openConnection(); err != nil { 46 | return 0, err 47 | } 48 | 49 | checksumStr := fmt.Sprintf("%x", checksum) 50 | 51 | requestHeaders := make(swift.Headers) 52 | requestHeaders["Range"] = fmt.Sprintf("bytes=%d-%d", offset, len(buffer)-1) 53 | 54 | var responseBuffer bytes.Buffer 55 | _, err := idx.connection.ObjectGet(idx.container, checksumStr, &responseBuffer, false, requestHeaders) 56 | if err != nil { 57 | return 0, err 58 | } 59 | 60 | if responseBuffer.Len() != len(buffer) { 61 | return 0, errors.New(fmt.Sprintf("cannot read %d chunk bytes, response was %s bytes instead", len(buffer), responseBuffer.Len())) 62 | } 63 | 64 | copied := copy(buffer, responseBuffer.Bytes()) 65 | if copied != len(buffer) { 66 | return 0, errors.New(fmt.Sprintf("cannot copy %d chunk bytes, only %s bytes copied instead", len(buffer), copied)) 67 | } 68 | 69 | return len(buffer), nil 70 | } 71 | 72 | func (idx *swiftChunkStore) Write(checksum []byte, buffer []byte) error { 73 | if err := idx.openConnection(); err != nil { 74 | return err 75 | } 76 | 77 | if err := idx.Stat(checksum); err == nil { 78 | return nil // Exists! 79 | } 80 | 81 | checksumStr := fmt.Sprintf("%x", checksum) 82 | 83 | if _, ok := idx.chunkMap.Load(checksumStr); !ok { 84 | 85 | if err := idx.connection.ObjectPutBytes(idx.container, checksumStr, buffer, "application/x-fsdup-chunk"); err != nil { 86 | return err 87 | } 88 | 89 | idx.chunkMap.Store(checksumStr, true) 90 | } 91 | 92 | return nil 93 | } 94 | 95 | func (idx *swiftChunkStore) Remove(checksum []byte) error { 96 | if err := idx.openConnection(); err != nil { 97 | return err 98 | } 99 | 100 | checksumStr := fmt.Sprintf("%x", checksum) 101 | err := idx.connection.ObjectDelete(idx.container, checksumStr) 102 | if err != nil { 103 | return err 104 | } 105 | 106 | idx.chunkMap.Delete(checksumStr) 107 | 108 | return nil 109 | } 110 | 111 | func (idx *swiftChunkStore) openConnection() error { 112 | if idx.connection.Authenticated() { 113 | return nil 114 | } 115 | 116 | err := idx.connection.Authenticate() 117 | if err != nil { 118 | return err 119 | } 120 | 121 | return nil 122 | } 123 | -------------------------------------------------------------------------------- /upload.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "context" 5 | "encoding/hex" 6 | "errors" 7 | "fmt" 8 | "google.golang.org/grpc" 9 | "heckel.io/fsdup/pb" 10 | "os" 11 | ) 12 | 13 | func Upload(manifestId string, metaStore MetaStore, inputFile string, serverAddr string) error { 14 | manifest, err := metaStore.ReadManifest(manifestId) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | // Get list of all chunks 20 | checksums := make([][]byte, 0) 21 | for checksumStr, _ := range manifest.Chunks() { 22 | checksum, err := hex.DecodeString(checksumStr) 23 | if err != nil { 24 | return err 25 | } 26 | 27 | if len(checksum) > 0 { 28 | checksums = append(checksums, checksum) 29 | } 30 | } 31 | 32 | // Find unknown chunks 33 | conn, err := grpc.Dial(serverAddr, grpc.WithBlock(), grpc.WithInsecure(), 34 | grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(128 * 1024 * 1024))) // FIXME 35 | if err != nil { 36 | return err 37 | } 38 | defer conn.Close() 39 | 40 | client := pb.NewHubClient(conn) 41 | 42 | response, err := client.Diff(context.Background(), &pb.DiffRequest{ 43 | Id: manifestId, 44 | Checksums: checksums, 45 | }, ) 46 | 47 | if err != nil { 48 | return err 49 | } 50 | 51 | unknowns := make(map[string]bool, 0) 52 | for _, checksum := range response.UnknownChecksums { 53 | unknowns[fmt.Sprintf("%x", checksum)] = true 54 | } 55 | 56 | // Open file 57 | in, err := os.OpenFile(inputFile, os.O_RDONLY, 0666) 58 | if err != nil { 59 | return err 60 | } 61 | defer in.Close() 62 | 63 | stat, err := in.Stat() 64 | if err != nil { 65 | return err 66 | } else if stat.Size() != manifest.Size() { 67 | return errors.New("size in manifest does not match file size. wrong input file?") 68 | } 69 | 70 | chunkSlices, err := manifest.ChunkSlices() 71 | if err != nil { 72 | return err 73 | } 74 | 75 | buffer := make([]byte, manifest.chunkMaxSize) 76 | 77 | for _, checksumStr := range manifest.ChecksumsByDiskOffset(chunkSlices) { 78 | if _, ok := unknowns[checksumStr]; !ok { 79 | continue 80 | } 81 | 82 | slices := chunkSlices[checksumStr] 83 | 84 | checksum, err := hex.DecodeString(checksumStr) // FIXME this is ugly. checksum should be its own type. 85 | if err != nil { 86 | return err 87 | } 88 | 89 | chunkSize := int64(0) 90 | 91 | for i, slice := range slices { 92 | debugf("idx %-5d diskoff %13d - %13d len %-10d chunkoff %13d - %13d\n", 93 | i, slice.diskfrom, slice.diskto, slice.length, slice.chunkfrom, slice.chunkto) 94 | 95 | read, err := in.ReadAt(buffer[slice.chunkfrom:slice.chunkto], slice.diskfrom) 96 | if err != nil { 97 | return err 98 | } else if int64(read) != slice.length { 99 | return errors.New(fmt.Sprintf("cannot read full chunk from input file, read only %d bytes, but %d expectecd", read, slice.length)) 100 | } 101 | 102 | chunkSize += slice.length 103 | } 104 | 105 | debugf("uploading %x\n", checksum) 106 | 107 | _, err = client.WriteChunk(context.Background(), &pb.WriteChunkRequest{ 108 | Checksum: checksum, 109 | Data: buffer[:chunkSize], 110 | }) 111 | 112 | if err != nil { 113 | return err 114 | } 115 | } 116 | 117 | _, err = client.WriteManifest(context.Background(), &pb.WriteManifestRequest{ 118 | Id: manifestId, 119 | Manifest: manifest.Proto(), 120 | }) 121 | 122 | if err != nil { 123 | return err 124 | } 125 | 126 | return nil 127 | } 128 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package fsdup 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "io" 8 | "math/rand" 9 | "time" 10 | ) 11 | 12 | func init() { 13 | rand.Seed(time.Now().UnixNano()) 14 | } 15 | 16 | func parseIntLE(b []byte, offset int64, length int64) int64 { 17 | bpad := make([]byte, 8) 18 | 19 | // Pad b to 8 bytes (uint64/int64 size), determine if negative 20 | // and fill the array with the little endian number and the sign 21 | sign := byte(0) 22 | if b[offset+length-1] & 0x80 == 0x80 { 23 | sign = 0xFF 24 | } 25 | 26 | for i := int64(0); i < length; i++ { 27 | bpad[i] = b[offset+i] 28 | } 29 | 30 | for i := length; i < 8; i++ { 31 | bpad[i] = sign 32 | } 33 | 34 | return int64(binary.LittleEndian.Uint64(bpad)) 35 | } 36 | 37 | func parseUintLE(b []byte, offset int64, length int64) int64 { 38 | bpad := make([]byte, 8) 39 | 40 | for i := int64(0); i < length; i++ { 41 | bpad[i] = b[offset+i] 42 | } 43 | 44 | return int64(binary.LittleEndian.Uint64(bpad)) 45 | } 46 | 47 | func minInt64(a, b int64) int64 { 48 | if a < b { 49 | return a 50 | } else { 51 | return b 52 | } 53 | } 54 | 55 | func maxInt64(a, b int64) int64 { 56 | if a > b { 57 | return a 58 | } else { 59 | return b 60 | } 61 | } 62 | 63 | func readAndCompare(reader io.ReaderAt, offset int64, expected []byte) error { 64 | actual := make([]byte, len(expected)) 65 | n, err := reader.ReadAt(actual, offset) 66 | 67 | if err != nil { 68 | return err 69 | } else if n != len(actual) || bytes.Compare(expected, actual) != 0 { 70 | return ErrUnexpectedMagic 71 | } 72 | 73 | return nil 74 | } 75 | 76 | func convertBytesToHumanReadable(b int64) string { 77 | const unit = 1024 78 | 79 | if b < unit { 80 | return fmt.Sprintf("%d byte(s)", b) 81 | } 82 | 83 | div, exp := int64(unit), 0 84 | for n := b / unit; n >= unit; n /= unit { 85 | div *= unit 86 | exp++ 87 | } 88 | 89 | return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp]) 90 | } 91 | 92 | var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") 93 | 94 | func randString(n int) string { 95 | b := make([]rune, n) 96 | for i := range b { 97 | b[i] = letterRunes[rand.Intn(len(letterRunes))] 98 | } 99 | return string(b) 100 | } --------------------------------------------------------------------------------