├── .build.yml ├── LICENSE ├── README.md ├── blobsfile.go ├── blobsfile_test.go ├── go.mod ├── go.sum ├── index.go └── index_test.go /.build.yml: -------------------------------------------------------------------------------- 1 | image: ubuntu/latest 2 | sources: 3 | - https://git.sr.ht/~tsileo/blobsfile 4 | tasks: 5 | - setup: | 6 | mkdir go 7 | export GOPATH=/home/build/go 8 | wget https://dl.google.com/go/go1.13.4.linux-amd64.tar.gz 9 | sudo tar -C /usr/local -xzf go1.13.4.linux-amd64.tar.gz 10 | - test: | 11 | cd blobsfile 12 | /usr/local/go/bin/go test -v -bench=. . 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 Thomas Sileo 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BlobsFile 2 | 3 | [![builds.sr.ht status](https://builds.sr.ht/~tsileo/blobsfile.svg)](https://builds.sr.ht/~tsileo/blobsfile?) 4 |    [![Godoc Reference](https://godoc.org/a4.io/blobsfile?status.svg)](https://godoc.org/a4.io/blobsfile) 5 | 6 | *BlobsFile* is an append-only (i.e. no update and no delete) content-addressed *blob store* (using [BLAKE2b](https://blake2.net/) as hash function). 7 | 8 | It draws inspiration from Facebook's [Haystack](http://202.118.11.61/papers/case%20studies/facebook.pdf), blobs are stored in flat files (called _BlobFile_) and indexed by a small [kv](https://github.com/cznic/kv) database for fast lookup. 9 | 10 | *BlobsFile* is [BlobStash](https://github.com/tsileo/blobstash)'s storage engine. 11 | 12 | ## Features 13 | 14 | - Durable (data is fsynced before returning) 15 | - Immutable (append-only, can't mutate or delete blobs) 16 | - Optional compression (Snappy or Zstandard) 17 | - Extra parity data is added to each _BlobFile_ (using Reed-Solomon error correcting code), allowing the database to repair itself in case of corruption. 18 | - The test suite is literraly punching holes at random places 19 | -------------------------------------------------------------------------------- /blobsfile.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Package blobsfile implement the BlobsFile backend for storing blobs. 4 | 5 | It stores multiple blobs (optionally compressed with Snappy) inside "BlobsFile"/fat file/packed file 6 | (256MB by default). 7 | Blobs are indexed by a kv file (that can be rebuild from the blobsfile). 8 | 9 | New blobs are appended to the current file, and when the file exceed the limit, a new fie is created. 10 | 11 | */ 12 | package blobsfile // import "a4.io/blobsfile" 13 | 14 | import ( 15 | "bytes" 16 | "encoding/binary" 17 | "encoding/hex" 18 | "errors" 19 | "expvar" 20 | "fmt" 21 | "io" 22 | "os" 23 | "path/filepath" 24 | "strings" 25 | "sync" 26 | "time" 27 | 28 | "a4.io/blobstash/pkg/rangedb" 29 | "github.com/golang/snappy" 30 | "github.com/klauspost/reedsolomon" 31 | "golang.org/x/crypto/blake2b" 32 | ) 33 | 34 | const ( 35 | // Version is the current BlobsFile binary format version 36 | Version = 1 37 | 38 | headerMagic = "\x00Blobs" 39 | headerSize = len(headerMagic) + 58 // magic + 58 reserved bytes 40 | 41 | // 38 bytes of meta-data are stored for each blob: 32 byte hash + 2 byte flag + 4 byte blob len 42 | blobOverhead = 38 43 | hashSize = 32 44 | 45 | // Reed-Solomon config 46 | dataShards = 10 // 10 data shards 47 | parityShards = 2 // 2 parity shards 48 | 49 | defaultMaxBlobsFileSize = 256 << 20 // 256MB 50 | ) 51 | 52 | // Blob flags 53 | const ( 54 | flagBlob byte = 1 << iota 55 | flagCompressed 56 | flagParityBlob 57 | flagEOF 58 | ) 59 | 60 | type CompressionAlgorithm byte 61 | 62 | // Compression algorithms flag 63 | const ( 64 | Snappy CompressionAlgorithm = 1 << iota 65 | ) 66 | 67 | var ( 68 | openFdsVar = expvar.NewMap("blobsfile-open-fds") 69 | bytesUploaded = expvar.NewMap("blobsfile-bytes-uploaded") 70 | bytesDownloaded = expvar.NewMap("blobsfile-bytes-downloaded") 71 | blobsUploaded = expvar.NewMap("blobsfile-blobs-uploaded") 72 | blobsDownloaded = expvar.NewMap("blobsfile-blobs-downloaded") 73 | ) 74 | 75 | var ( 76 | // ErrBlobNotFound reports that the blob could not be found 77 | ErrBlobNotFound = errors.New("blob not found") 78 | 79 | // ErrBlobsfileCorrupted reports that one of the BlobsFile is corrupted and could not be repaired 80 | ErrBlobsfileCorrupted = errors.New("blobsfile is corrupted") 81 | 82 | errParityBlobCorrupted = errors.New("a parity blob is corrupted") 83 | ) 84 | 85 | // ErrInterventionNeeded is an error indicating an manual action must be performed before being able to use BobsFile 86 | type ErrInterventionNeeded struct { 87 | msg string 88 | } 89 | 90 | func (ein *ErrInterventionNeeded) Error() string { 91 | return fmt.Sprintf("manual intervention needed: %s", ein.msg) 92 | } 93 | 94 | func checkFlag(f byte) { 95 | if f == flagEOF || f == flagParityBlob { 96 | panic(fmt.Sprintf("Unexpected blob flag %v", f)) 97 | } 98 | } 99 | 100 | // multiError wraps multiple errors in a single one 101 | type multiError struct { 102 | errors []error 103 | } 104 | 105 | func (me *multiError) Error() string { 106 | if me.errors == nil { 107 | return "multiError:" 108 | } 109 | var errs []string 110 | for _, err := range me.errors { 111 | errs = append(errs, err.Error()) 112 | } 113 | return fmt.Sprintf("multiError: %s", strings.Join(errs, ", ")) 114 | } 115 | 116 | func (me *multiError) Append(err error) { 117 | me.errors = append(me.errors, err) 118 | } 119 | 120 | func (me *multiError) Nil() bool { 121 | if me.errors == nil || len(me.errors) == 0 { 122 | return true 123 | } 124 | return false 125 | } 126 | 127 | // corruptedError give more about the corruption of a BlobsFile 128 | type corruptedError struct { 129 | n int 130 | blobs []*blobPos 131 | offset int64 132 | err error 133 | } 134 | 135 | func (ce *corruptedError) Error() string { 136 | if len(ce.blobs) > 0 { 137 | return fmt.Sprintf("%d blobs are corrupt", len(ce.blobs)) 138 | } 139 | return fmt.Sprintf("corrupted at offset %d: %v", ce.offset, ce.err) 140 | } 141 | 142 | func (ce *corruptedError) firstBadOffset() int64 { 143 | if len(ce.blobs) > 0 { 144 | off := int64(ce.blobs[0].offset) 145 | if ce.offset == -1 || off < ce.offset { 146 | return off 147 | } 148 | } 149 | return ce.offset 150 | } 151 | 152 | func firstCorruptedShard(offset int64, shardSize int) int { 153 | i := 0 154 | ioffset := int(offset) 155 | for j := 0; j < dataShards; j++ { 156 | if shardSize+(shardSize*i) > ioffset { 157 | return i 158 | } 159 | i++ 160 | } 161 | return 0 162 | } 163 | 164 | // Stats represents some stats about the DB state 165 | type Stats struct { 166 | // The total number of blobs stored 167 | BlobsCount int 168 | 169 | // The size of all the blobs stored 170 | BlobsSize int64 171 | 172 | // The number of BlobsFile 173 | BlobsFilesCount int 174 | 175 | // The size of all the BlobsFile 176 | BlobsFilesSize int64 177 | } 178 | 179 | // Opts represents the DB options 180 | type Opts struct { 181 | // Compression algorithm 182 | Compression CompressionAlgorithm 183 | 184 | // The max size of a BlobsFile, will be 256MB by default if not set 185 | BlobsFileSize int64 186 | 187 | // Where the data and indexes will be stored 188 | Directory string 189 | 190 | // Allow to catch some events 191 | LogFunc func(msg string) 192 | 193 | // When trying to self-heal in case of recovery, some step need to be performed by the user 194 | AskConfirmationFunc func(msg string) bool 195 | 196 | BlobsFilesSealedFunc func(path string) 197 | 198 | // Not implemented yet, will allow to provide repaired data in case of hard failure 199 | // RepairBlobFunc func(hash string) ([]byte, error) 200 | } 201 | 202 | func (o *Opts) init() { 203 | if o.BlobsFileSize == 0 { 204 | o.BlobsFileSize = defaultMaxBlobsFileSize 205 | } 206 | } 207 | 208 | // BlobsFiles represent the DB 209 | type BlobsFiles struct { 210 | // Directory which holds the blobsfile 211 | directory string 212 | 213 | // Maximum size for a blobsfile (256MB by default) 214 | maxBlobsFileSize int64 215 | 216 | // Backend state 217 | reindexMode bool 218 | 219 | // Compression is disabled by default 220 | compression CompressionAlgorithm 221 | 222 | // The kv index that maintains blob positions 223 | index *blobsIndex 224 | 225 | // Current blobs file opened for write 226 | n int 227 | current *os.File 228 | // Size of the current blobs file 229 | size int64 230 | // All blobs files opened for read 231 | files map[int]*os.File 232 | 233 | lastErr error 234 | lastErrMutex sync.Mutex // mutex for guarding the lastErr 235 | 236 | logFunc func(string) 237 | askConfirmationFunc func(string) bool 238 | blobsFilesSealedFunc func(string) 239 | 240 | // Reed-solomon encoder for the parity blobs 241 | rse reedsolomon.Encoder 242 | 243 | wg sync.WaitGroup 244 | sync.Mutex 245 | } 246 | 247 | // Blob represents a blob hash and size when enumerating the DB. 248 | type Blob struct { 249 | Hash string 250 | Size int 251 | N int 252 | } 253 | 254 | // New intializes a new BlobsFileBackend. 255 | func New(opts *Opts) (*BlobsFiles, error) { 256 | opts.init() 257 | dir := opts.Directory 258 | // Try to create the directory 259 | if err := os.MkdirAll(dir, 0700); err != nil { 260 | return nil, err 261 | } 262 | var reindex bool 263 | // Check if an index file is already present 264 | if _, err := os.Stat(filepath.Join(dir, "blobs-index")); os.IsNotExist(err) { 265 | // No index found 266 | reindex = true 267 | } 268 | index, err := newIndex(dir) 269 | if err != nil { 270 | return nil, err 271 | } 272 | 273 | // Initialize the Reed-Solomon encoder 274 | enc, err := reedsolomon.New(dataShards, parityShards) 275 | if err != nil { 276 | return nil, err 277 | } 278 | backend := &BlobsFiles{ 279 | directory: dir, 280 | compression: opts.Compression, 281 | index: index, 282 | files: make(map[int]*os.File), 283 | maxBlobsFileSize: opts.BlobsFileSize, 284 | blobsFilesSealedFunc: opts.BlobsFilesSealedFunc, 285 | rse: enc, 286 | reindexMode: reindex, 287 | logFunc: opts.LogFunc, 288 | } 289 | if err := backend.load(); err != nil { 290 | panic(fmt.Errorf("error loading %T: %v", backend, err)) 291 | } 292 | return backend, nil 293 | } 294 | 295 | func (backend *BlobsFiles) SetBlobsFilesSealedFunc(f func(string)) { 296 | backend.blobsFilesSealedFunc = f 297 | } 298 | 299 | func (backend *BlobsFiles) getConfirmation(msg string) (bool, error) { 300 | // askConfirmationFunc func(string) bool 301 | if backend.askConfirmationFunc == nil { 302 | return false, &ErrInterventionNeeded{msg} 303 | } 304 | 305 | ok := backend.askConfirmationFunc(msg) 306 | 307 | if !ok { 308 | return false, &ErrInterventionNeeded{msg} 309 | } 310 | 311 | return true, nil 312 | } 313 | 314 | func (backend *BlobsFiles) SealedPacks() []string { 315 | packs := []string{} 316 | for i := 0; i < backend.n; i++ { 317 | packs = append(packs, backend.filename(i)) 318 | } 319 | return packs 320 | } 321 | 322 | func (backend *BlobsFiles) iterOpenFiles() (files []*os.File) { 323 | for _, f := range backend.files { 324 | files = append(files, f) 325 | } 326 | return files 327 | } 328 | 329 | func (backend *BlobsFiles) closeOpenFiles() { 330 | for _, f := range backend.files { 331 | f.Close() 332 | } 333 | } 334 | 335 | func (backend *BlobsFiles) log(msg string, args ...interface{}) { 336 | if backend.logFunc == nil { 337 | return 338 | } 339 | backend.logFunc(fmt.Sprintf(msg, args...)) 340 | } 341 | 342 | // Stats returns some stats about the DB. 343 | func (backend *BlobsFiles) Stats() (*Stats, error) { 344 | // Iterate the index to gather the stats (Enumerate will acquire the lock) 345 | bchan := make(chan *Blob) 346 | errc := make(chan error, 1) 347 | go func() { 348 | errc <- backend.Enumerate(bchan, "", "\xfe", 0) 349 | }() 350 | blobsCount := 0 351 | var blobsSize int64 352 | for ref := range bchan { 353 | blobsCount++ 354 | blobsSize += int64(ref.Size) 355 | } 356 | if err := <-errc; err != nil { 357 | panic(err) 358 | } 359 | 360 | // Now iterate the raw blobsfile for gethering stats 361 | backend.Lock() 362 | defer backend.Unlock() 363 | var bfs int64 364 | for _, f := range backend.iterOpenFiles() { 365 | finfo, err := f.Stat() 366 | if err != nil { 367 | return nil, err 368 | } 369 | bfs += finfo.Size() 370 | } 371 | n, err := backend.getN() 372 | if err != nil { 373 | return nil, err 374 | } 375 | 376 | return &Stats{ 377 | BlobsFilesCount: n + 1, 378 | BlobsFilesSize: bfs, 379 | BlobsCount: blobsCount, 380 | BlobsSize: blobsSize, 381 | }, nil 382 | } 383 | 384 | // setLastError is used by goroutine that can't return an error easily 385 | func (backend *BlobsFiles) setLastError(err error) { 386 | backend.lastErrMutex.Lock() 387 | defer backend.lastErrMutex.Unlock() 388 | backend.lastErr = err 389 | } 390 | 391 | // lastError returns the last error that may have happened in asynchronous way (like the parity blobs writing process). 392 | func (backend *BlobsFiles) lastError() error { 393 | backend.lastErrMutex.Lock() 394 | defer backend.lastErrMutex.Unlock() 395 | if backend.lastErr == nil { 396 | return nil 397 | } 398 | err := backend.lastErr 399 | backend.lastErr = nil 400 | return err 401 | } 402 | 403 | // Close closes all the indexes and data files. 404 | func (backend *BlobsFiles) Close() error { 405 | backend.wg.Wait() 406 | if err := backend.lastError(); err != nil { 407 | return err 408 | } 409 | if err := backend.index.Close(); err != nil { 410 | return err 411 | } 412 | return nil 413 | } 414 | 415 | // RebuildIndex removes the index files and re-build it by re-scanning all the BlobsFiles. 416 | func (backend *BlobsFiles) RebuildIndex() error { 417 | if err := backend.index.remove(); err != nil { 418 | return nil 419 | } 420 | return backend.reindex() 421 | } 422 | 423 | // getN returns the total numbers of BlobsFile. 424 | func (backend *BlobsFiles) getN() (int, error) { 425 | return backend.index.getN() 426 | } 427 | 428 | func (backend *BlobsFiles) saveN() error { 429 | return backend.index.setN(backend.n) 430 | } 431 | 432 | func (backend *BlobsFiles) restoreN() error { 433 | n, err := backend.index.getN() 434 | if err != nil { 435 | return err 436 | } 437 | backend.n = n 438 | return nil 439 | } 440 | 441 | // String implements the Stringer interface. 442 | func (backend *BlobsFiles) String() string { 443 | return fmt.Sprintf("blobsfile-%v", backend.directory) 444 | } 445 | 446 | // scanBlobsFile scan a single BlobsFile (#n), and execute `iterFunc` for each indexed blob. 447 | // `iterFunc` is optional, and without it, this func will check the consistency of each blob, and return 448 | // a `corruptedError` if a blob is corrupted. 449 | func (backend *BlobsFiles) scanBlobsFile(n int, iterFunc func(*blobPos, byte, string, []byte) error) error { 450 | corrupted := []*blobPos{} 451 | 452 | // Ensure this BlosFile is open 453 | err := backend.ropen(n) 454 | if err != nil { 455 | return err 456 | } 457 | 458 | // Seek at the start of data 459 | offset := int64(headerSize) 460 | blobsfile := backend.files[n] 461 | if _, err := blobsfile.Seek(int64(headerSize), os.SEEK_SET); err != nil { 462 | return err 463 | } 464 | 465 | blobsIndexed := 0 466 | 467 | blobHash := make([]byte, hashSize) 468 | blobSizeEncoded := make([]byte, 4) 469 | flags := make([]byte, 2) 470 | 471 | for { 472 | // Read the hash 473 | if _, err := blobsfile.Read(blobHash); err != nil { 474 | if err == io.EOF { 475 | break 476 | } 477 | return &corruptedError{n, nil, offset, fmt.Errorf("failed to read hash: %v", err)} 478 | } 479 | 480 | // Read the 2 byte flags 481 | if _, err := blobsfile.Read(flags); err != nil { 482 | return &corruptedError{n, nil, offset, fmt.Errorf("failed to read flag: %v", err)} 483 | } 484 | 485 | // If we reached the EOF blob, we're done 486 | if flags[0] == flagEOF { 487 | break 488 | } 489 | 490 | // Read the size of the blob 491 | if _, err := blobsfile.Read(blobSizeEncoded); err != nil { 492 | return &corruptedError{n, nil, offset, fmt.Errorf("failed to read blob size: %v", err)} 493 | } 494 | 495 | // Read the actual blob 496 | blobSize := int64(binary.LittleEndian.Uint32(blobSizeEncoded)) 497 | rawBlob := make([]byte, int(blobSize)) 498 | read, err := blobsfile.Read(rawBlob) 499 | if err != nil || read != int(blobSize) { 500 | return &corruptedError{n, nil, offset, fmt.Errorf("error while reading raw blob: %v", err)} 501 | } 502 | 503 | // Build the `blobPos` 504 | blobPos := &blobPos{n: n, offset: offset, size: int(blobSize)} 505 | offset += blobOverhead + blobSize 506 | 507 | // Decompress the blob if needed 508 | var blob []byte 509 | if flags[0] == flagCompressed && flags[1] != 0 { 510 | var err error 511 | var blobDecoded []byte 512 | switch CompressionAlgorithm(flags[1]) { 513 | case Snappy: 514 | blobDecoded, err = snappy.Decode(nil, rawBlob) 515 | } 516 | if err != nil { 517 | return &corruptedError{n, nil, offset, fmt.Errorf("failed to decode blob: %v %v %v", err, blobSize, flags)} 518 | } 519 | blob = blobDecoded 520 | 521 | } else { 522 | blob = rawBlob 523 | } 524 | // Store the real blob size (i.e. the decompressed size if the data is compressed) 525 | blobPos.blobSize = len(blob) 526 | 527 | // Ensure the blob is not corrupted 528 | hash := fmt.Sprintf("%x", blake2b.Sum256(blob)) 529 | if fmt.Sprintf("%x", blobHash) == hash { 530 | if iterFunc != nil { 531 | if err := iterFunc(blobPos, flags[0], hash, blob); err != nil { 532 | return err 533 | } 534 | } 535 | blobsIndexed++ 536 | } else { 537 | // The blobs is corrupted, keep track of it 538 | corrupted = append(corrupted, blobPos) 539 | } 540 | } 541 | 542 | if len(corrupted) > 0 { 543 | return &corruptedError{n, corrupted, -1, nil} 544 | } 545 | 546 | return nil 547 | } 548 | 549 | // scanBlobsFile scan a single BlobsFile (#n), and execute `iterFunc` for each indexed blob. 550 | // `iterFunc` is optional, and without it, this func will check the consistency of each blob, and return 551 | // a `corruptedError` if a blob is corrupted. 552 | func ScanBlobsFile(path string) ([]string, error) { 553 | hashes := []string{} 554 | blobsfile, err := os.Open(path) 555 | if err != nil { 556 | return nil, err 557 | } 558 | defer blobsfile.Close() 559 | 560 | // Seek at the start of data 561 | offset := int64(headerSize) 562 | if _, err := blobsfile.Seek(int64(headerSize), os.SEEK_SET); err != nil { 563 | return nil, err 564 | } 565 | 566 | blobsIndexed := 0 567 | 568 | blobHash := make([]byte, hashSize) 569 | blobSizeEncoded := make([]byte, 4) 570 | flags := make([]byte, 2) 571 | 572 | for { 573 | // Read the hash 574 | if _, err := blobsfile.Read(blobHash); err != nil { 575 | if err == io.EOF { 576 | break 577 | } 578 | return nil, &corruptedError{0, nil, offset, fmt.Errorf("failed to read hash: %v", err)} 579 | } 580 | 581 | // Read the 2 byte flags 582 | if _, err := blobsfile.Read(flags); err != nil { 583 | return nil, &corruptedError{0, nil, offset, fmt.Errorf("failed to read flag: %v", err)} 584 | } 585 | 586 | // If we reached the EOF blob, we're done 587 | if flags[0] == flagEOF { 588 | break 589 | } 590 | 591 | // Read the size of the blob 592 | if _, err := blobsfile.Read(blobSizeEncoded); err != nil { 593 | return nil, &corruptedError{0, nil, offset, fmt.Errorf("failed to read blob size: %v", err)} 594 | } 595 | 596 | // Read the actual blob 597 | blobSize := int64(binary.LittleEndian.Uint32(blobSizeEncoded)) 598 | rawBlob := make([]byte, int(blobSize)) 599 | read, err := blobsfile.Read(rawBlob) 600 | if err != nil || read != int(blobSize) { 601 | return nil, &corruptedError{0, nil, offset, fmt.Errorf("error while reading raw blob: %v", err)} 602 | } 603 | 604 | // Build the `blobPos` 605 | offset += blobOverhead + blobSize 606 | 607 | // Decompress the blob if needed 608 | var blob []byte 609 | if flags[0] == flagCompressed && flags[1] != 0 { 610 | var err error 611 | var blobDecoded []byte 612 | switch CompressionAlgorithm(flags[1]) { 613 | case Snappy: 614 | blobDecoded, err = snappy.Decode(nil, rawBlob) 615 | } 616 | if err != nil { 617 | return nil, &corruptedError{0, nil, offset, fmt.Errorf("failed to decode blob: %v %v %v", err, blobSize, flags)} 618 | } 619 | blob = blobDecoded 620 | 621 | } else { 622 | blob = rawBlob 623 | } 624 | 625 | // Ensure the blob is not corrupted 626 | hash := fmt.Sprintf("%x", blake2b.Sum256(blob)) 627 | if fmt.Sprintf("%x", blobHash) == hash { 628 | hashes = append(hashes, hash) 629 | blobsIndexed++ 630 | } else { 631 | panic("corrupted") 632 | } 633 | } 634 | 635 | return hashes, nil 636 | } 637 | 638 | func copyShards(i [][]byte) (o [][]byte) { 639 | for _, a := range i { 640 | o = append(o, a) 641 | } 642 | return o 643 | } 644 | 645 | // CheckBlobsFiles will check the consistency of all the BlobsFile 646 | func (backend *BlobsFiles) CheckBlobsFiles() error { 647 | err := backend.scan(nil) 648 | if err == nil { 649 | backend.log("all blobs has been verified") 650 | } 651 | return err 652 | } 653 | 654 | func (backend *BlobsFiles) checkBlobsFile(cerr *corruptedError) error { 655 | // TODO(tsileo): provide an exported method to do the check 656 | n := cerr.n 657 | pShards, err := backend.parityShards(n) 658 | if err != nil { 659 | // TODO(tsileo): log the error 660 | fmt.Printf("parity shards err=%v\n", err) 661 | } 662 | parityCnt := len(pShards) 663 | fmt.Printf("scan result=%v %+v\n", cerr, cerr) 664 | // if err == nil && (pShards == nil || len(pShards) != parityShards) { 665 | // // We can rebuild the parity blobs if needed 666 | // // FIXME(tsileo): do it 667 | // var l int 668 | // if pShards != nil { 669 | // l = len(pShards) 670 | // } else { 671 | // pShards = [][]byte{} 672 | // } 673 | 674 | // for i := 0; i < parityShards-l; i++ { 675 | // pShards = append(pShards, nil) 676 | // } 677 | // // TODO(tsileo): save the parity shards 678 | // } 679 | 680 | if pShards == nil || len(pShards) == 0 { 681 | return fmt.Errorf("no parity shards available, can't recover") 682 | } 683 | 684 | dataShardIndex := 0 685 | if cerr != nil { 686 | badOffset := cerr.firstBadOffset() 687 | fmt.Printf("badOffset: %v\n", badOffset) 688 | dataShardIndex = firstCorruptedShard(badOffset, int(backend.maxBlobsFileSize)/dataShards) 689 | fmt.Printf("dataShardIndex=%d\n", dataShardIndex) 690 | } 691 | 692 | // if err != nil { 693 | // if cerr, ok := err.(*corruptedError); ok { 694 | // badOffset := cerr.firstBadOffset() 695 | // fmt.Printf("badOffset: %v\n", badOffset) 696 | // dataShardIndex = firstCorruptedShard(badOffset, int(backend.maxBlobsFileSize)/dataShards) 697 | // fmt.Printf("dataShardIndex=%d\n", dataShardIndex) 698 | // } 699 | // } 700 | 701 | missing := []int{} 702 | for i := dataShardIndex; i < 10; i++ { 703 | missing = append(missing, i) 704 | } 705 | fmt.Printf("missing=%+v\n", missing) 706 | 707 | dShards, err := backend.dataShards(n) 708 | if err != nil { 709 | return err 710 | } 711 | 712 | fmt.Printf("try #1\n") 713 | if len(missing) <= parityCnt { 714 | shards := copyShards(append(dShards, pShards...)) 715 | 716 | for _, idx := range missing { 717 | shards[idx] = nil 718 | } 719 | 720 | if err := backend.rse.Reconstruct(shards); err != nil { 721 | return err 722 | } 723 | 724 | ok, err := backend.rse.Verify(shards) 725 | if err != nil { 726 | return err 727 | } 728 | 729 | if ok { 730 | fmt.Printf("reconstruct successful\n") 731 | if err := backend.rewriteBlobsFile(n, shards); err != nil { 732 | return err 733 | } 734 | 735 | return nil 736 | } 737 | return fmt.Errorf("unrecoverable corruption") 738 | } 739 | 740 | fmt.Printf("try #2\n") 741 | // Try one missing shards 742 | for i := dataShardIndex; i < 10; i++ { 743 | shards := copyShards(append(dShards, pShards...)) 744 | shards[i] = nil 745 | 746 | if err := backend.rse.Reconstruct(shards); err != nil { 747 | return err 748 | } 749 | 750 | ok, err := backend.rse.Verify(shards) 751 | if err != nil { 752 | return err 753 | } 754 | 755 | if ok { 756 | fmt.Printf("reconstruct successful at %d\n", i) 757 | if err := backend.rewriteBlobsFile(n, shards); err != nil { 758 | return err 759 | } 760 | 761 | return nil 762 | } 763 | } 764 | 765 | // TODO(tsileo): only do this check if the two parity blobs are here 766 | fmt.Printf("try #3\n") 767 | if len(pShards) >= 2 { 768 | for i := dataShardIndex; i < 10; i++ { 769 | for j := dataShardIndex; j < 10; j++ { 770 | if j == i { 771 | continue 772 | } 773 | 774 | shards := copyShards(append(dShards, pShards...)) 775 | 776 | shards[i] = nil 777 | shards[j] = nil 778 | 779 | if err := backend.rse.Reconstruct(shards); err != nil { 780 | return err 781 | } 782 | 783 | ok, err := backend.rse.Verify(shards) 784 | if err != nil { 785 | return err 786 | } 787 | 788 | if ok { 789 | if err := backend.rewriteBlobsFile(n, shards); err != nil { 790 | return err 791 | } 792 | 793 | return nil 794 | } 795 | } 796 | } 797 | } 798 | 799 | // XXX(tsileo): support for 4 failed parity shards 800 | return fmt.Errorf("failed to recover") 801 | } 802 | 803 | func (backend *BlobsFiles) rewriteBlobsFile(n int, shards [][]byte) error { 804 | if f, alreadyOpen := backend.files[n]; alreadyOpen { 805 | if err := f.Close(); err != nil { 806 | return err 807 | } 808 | delete(backend.files, n) 809 | } 810 | 811 | // Create a new temporary file 812 | f, err := os.OpenFile(backend.filename(n)+".new", os.O_RDWR|os.O_CREATE, 0666) 813 | if err != nil { 814 | return err 815 | } 816 | 817 | // Re-create the healed Blobsfile 818 | for _, shard := range shards[0:dataShards] { 819 | f.Write(shard) 820 | } 821 | for _, shard := range shards[dataShards:] { 822 | _, parityBlobEncoded := backend.encodeBlob(shard, flagParityBlob) 823 | 824 | n, err := f.Write(parityBlobEncoded) 825 | if err != nil || n != len(parityBlobEncoded) { 826 | return fmt.Errorf("error writing parity blob (%v,%v)", err, n) 827 | } 828 | } 829 | 830 | if err := f.Sync(); err != nil { 831 | return err 832 | } 833 | f.Close() 834 | 835 | // Remove the corrupted BlobsFile 836 | if err := os.Remove(backend.filename(n)); err != nil { 837 | return err 838 | } 839 | 840 | // Rename our newly created BlobsFile to replace the old one 841 | if err := os.Rename(backend.filename(n)+".new", backend.filename(n)); err != nil { 842 | return err 843 | } 844 | 845 | fmt.Printf("reopen\n") 846 | if err := backend.ropen(n); err != nil { 847 | return err 848 | } 849 | fmt.Printf("file rewrite done\n") 850 | // if err := f.Close(); err != nil { 851 | // return err 852 | // } 853 | 854 | // TODO(tsileo): display user info (introduce a new helper) to ask to remove the old blobsfile and rename the 855 | // .restored. 856 | // TODO(tsileo): also use this new helper (which should clean shutdown blobstahs) in case of blbo corruption 857 | // detected. 858 | // TODO(tsileo): also prove a call for corruptions to let wrapper provide a repaired blob from other source. 859 | return nil 860 | } 861 | 862 | func (backend *BlobsFiles) dataShards(n int) ([][]byte, error) { 863 | // Read the whole blobsfile data (except the parity blobs) 864 | data := make([]byte, backend.maxBlobsFileSize) 865 | if _, err := backend.files[n].ReadAt(data, 0); err != nil { 866 | return nil, err 867 | } 868 | 869 | if !bytes.Equal(data[0:len(headerMagic)], []byte(headerMagic)) { 870 | return nil, fmt.Errorf("bad magic when trying to creata data shard") 871 | } 872 | fmt.Printf("data shard magic OK\n") 873 | 874 | // Rebuild the data shards using the data part of the blobsfile 875 | shards, err := backend.rse.Split(data) 876 | if err != nil { 877 | return nil, err 878 | } 879 | 880 | return shards[:10], nil 881 | } 882 | 883 | // parityShards extract the "parity blob" at the end of the BlobsFile 884 | func (backend *BlobsFiles) parityShards(n int) ([][]byte, error) { 885 | blobsfile := backend.files[n] 886 | parityBlobs := [][]byte{} 887 | 888 | merr := &multiError{} 889 | 890 | blobHash := make([]byte, hashSize) 891 | for i := 0; i < parityShards; i++ { 892 | // Seek to the offset where the parity blob should be stored 893 | offset := backend.maxBlobsFileSize + int64(i)*((backend.maxBlobsFileSize/int64(dataShards))+int64(hashSize+6)) 894 | if _, err := backend.files[n].Seek(offset, os.SEEK_SET); err != nil { 895 | merr.Append(fmt.Errorf("failed to seek to parity shards: %v", err)) 896 | parityBlobs = append(parityBlobs, nil) 897 | continue 898 | } 899 | 900 | // Read the hash of the blob 901 | if _, err := blobsfile.Read(blobHash); err != nil { 902 | if err == io.EOF { 903 | merr.Append(fmt.Errorf("missing parity blob %d, only found %d", i, len(parityBlobs)+1)) 904 | parityBlobs = append(parityBlobs, nil) 905 | continue 906 | } 907 | merr.Append(fmt.Errorf("failed to read the hash for parity blob %d: %v", i, err)) 908 | parityBlobs = append(parityBlobs, nil) 909 | continue 910 | } 911 | 912 | // We skip the flags and the blob length as it may be corrupted and we know the length. 913 | if _, err := blobsfile.Seek(offset+6+hashSize, os.SEEK_SET); err != nil { 914 | merr.Append(fmt.Errorf("failed to seek to parity blob %d: %v", i, err)) 915 | parityBlobs = append(parityBlobs, nil) 916 | continue 917 | } 918 | 919 | // Read the blob data 920 | blobSize := int(backend.maxBlobsFileSize / dataShards) 921 | blob := make([]byte, blobSize) 922 | read, err := blobsfile.Read(blob) 923 | if err != nil || read != int(blobSize) { 924 | merr.Append(fmt.Errorf("error while reading raw blob %d: %v", i, err)) 925 | parityBlobs = append(parityBlobs, nil) 926 | continue 927 | } 928 | 929 | // Check the data against the stored hash 930 | hash := fmt.Sprintf("%x", blake2b.Sum256(blob)) 931 | if fmt.Sprintf("%x", blobHash) != hash { 932 | merr.Append(errParityBlobCorrupted) 933 | parityBlobs = append(parityBlobs, nil) 934 | continue 935 | } 936 | 937 | parityBlobs = append(parityBlobs, blob) 938 | } 939 | 940 | if merr.Nil() { 941 | return parityBlobs, nil 942 | } 943 | 944 | return parityBlobs, merr 945 | } 946 | 947 | // checkParityBlobs ensures that the parity blobs and the the data shards can be verified (i.e integrity verification) 948 | func (backend *BlobsFiles) checkParityBlobs(n int) error { 949 | dataShards, err := backend.dataShards(n) 950 | if err != nil { 951 | return fmt.Errorf("failed to build data shards: %v", err) 952 | } 953 | 954 | parityShards, err := backend.parityShards(n) 955 | if err != nil { 956 | // We just log the error 957 | fmt.Printf("failed to build parity shards: %v", err) 958 | } 959 | 960 | shards := append(dataShards, parityShards...) 961 | 962 | // Verify the integrity of the data 963 | ok, err := backend.rse.Verify(shards) 964 | if err != nil { 965 | return fmt.Errorf("failed to verify shards: %v", err) 966 | } 967 | 968 | if !ok { 969 | return ErrBlobsfileCorrupted 970 | } 971 | 972 | return nil 973 | } 974 | 975 | // scan executes the callback func `iterFunc` for each indexed blobs in all the available BlobsFiles. 976 | func (backend *BlobsFiles) scan(iterFunc func(*blobPos, byte, string, []byte) error) error { 977 | n := 0 978 | for { 979 | err := backend.scanBlobsFile(n, iterFunc) 980 | if os.IsNotExist(err) { 981 | break 982 | } 983 | if err != nil { 984 | return err 985 | } 986 | n++ 987 | } 988 | if n == 0 { 989 | return nil 990 | } 991 | return nil 992 | } 993 | 994 | // reindex scans all BlobsFile and reconstruct the index from scratch. 995 | func (backend *BlobsFiles) reindex() error { 996 | backend.wg.Add(1) 997 | defer backend.wg.Done() 998 | 999 | if err := backend.index.remove(); err != nil { 1000 | return err 1001 | } 1002 | 1003 | var err error 1004 | backend.index.db, err = rangedb.New(backend.index.path) 1005 | if err != nil { 1006 | return err 1007 | } 1008 | 1009 | n := 0 1010 | blobsIndexed := 0 1011 | 1012 | iterFunc := func(blobPos *blobPos, flag byte, hash string, _ []byte) error { 1013 | // Skip parity blobs 1014 | if flag == flagParityBlob { 1015 | return nil 1016 | } 1017 | if err := backend.index.setPos(hash, blobPos); err != nil { 1018 | return err 1019 | } 1020 | n = blobPos.n 1021 | blobsIndexed++ 1022 | return nil 1023 | } 1024 | 1025 | if err := backend.scan(iterFunc); err != nil { 1026 | if cerr, ok := err.(*corruptedError); ok { 1027 | if err := backend.checkBlobsFile(cerr); err != nil { 1028 | return err 1029 | } 1030 | 1031 | // If err was nil, then the recontruct was successful, we can try to reindex 1032 | if err := backend.RebuildIndex(); err != nil { 1033 | return err 1034 | } 1035 | return nil 1036 | } 1037 | return err 1038 | } 1039 | 1040 | if n == 0 { 1041 | return nil 1042 | } 1043 | if err := backend.saveN(); err != nil { 1044 | return err 1045 | } 1046 | return nil 1047 | } 1048 | 1049 | // Open all the blobs-XXXXX (read-only) and open the last for write 1050 | func (backend *BlobsFiles) load() error { 1051 | backend.wg.Add(1) 1052 | defer backend.wg.Done() 1053 | 1054 | n := 0 1055 | for { 1056 | err := backend.ropen(n) 1057 | if os.IsNotExist(err) { 1058 | // No more blobsfile 1059 | break 1060 | } 1061 | if err != nil { 1062 | return err 1063 | } 1064 | n++ 1065 | } 1066 | 1067 | if n == 0 { 1068 | // The dir is empty, create a new blobs-XXXXX file, 1069 | // and open it for read 1070 | if err := backend.wopen(n); err != nil { 1071 | return err 1072 | } 1073 | if err := backend.ropen(n); err != nil { 1074 | return err 1075 | } 1076 | if err := backend.saveN(); err != nil { 1077 | return err 1078 | } 1079 | return nil 1080 | } 1081 | 1082 | // Open the last file for write 1083 | if err := backend.wopen(n - 1); err != nil { 1084 | return err 1085 | } 1086 | 1087 | if err := backend.saveN(); err != nil { 1088 | return err 1089 | } 1090 | 1091 | if backend.reindexMode { 1092 | if err := backend.reindex(); err != nil { 1093 | return err 1094 | } 1095 | } 1096 | return nil 1097 | } 1098 | 1099 | // Open a file for writing, will close the previously open file if any. 1100 | func (backend *BlobsFiles) wopen(n int) error { 1101 | // Close the already opened file if any 1102 | if backend.current != nil { 1103 | if err := backend.current.Close(); err != nil { 1104 | openFdsVar.Add(backend.directory, -1) 1105 | return err 1106 | } 1107 | } 1108 | 1109 | // Track if we created the file 1110 | created := false 1111 | if _, err := os.Stat(backend.filename(n)); os.IsNotExist(err) { 1112 | created = true 1113 | } 1114 | 1115 | // Open the file in rw mode 1116 | f, err := os.OpenFile(backend.filename(n), os.O_RDWR|os.O_CREATE, 0666) 1117 | if err != nil { 1118 | return err 1119 | } 1120 | 1121 | backend.current = f 1122 | backend.n = n 1123 | 1124 | if created { 1125 | // Write the header/magic number 1126 | if _, err := backend.current.Write([]byte(headerMagic)); err != nil { 1127 | return err 1128 | } 1129 | // Write the reserved bytes 1130 | reserved := make([]byte, 58) 1131 | binary.LittleEndian.PutUint32(reserved, uint32(Version)) 1132 | if _, err := backend.current.Write(reserved[:]); err != nil { 1133 | return err 1134 | } 1135 | 1136 | // Fsync 1137 | if err = backend.current.Sync(); err != nil { 1138 | panic(err) 1139 | } 1140 | } 1141 | 1142 | backend.size, err = f.Seek(0, os.SEEK_END) 1143 | if err != nil { 1144 | return err 1145 | } 1146 | 1147 | openFdsVar.Add(backend.directory, 1) 1148 | 1149 | return nil 1150 | } 1151 | 1152 | // Open a file for read 1153 | func (backend *BlobsFiles) ropen(n int) error { 1154 | _, alreadyOpen := backend.files[n] 1155 | if alreadyOpen { 1156 | // log.Printf("BlobsFileBackend: blobsfile %v already open", backend.filename(n)) 1157 | return nil 1158 | } 1159 | if n > len(backend.files) { 1160 | return fmt.Errorf("trying to open file %v whereas only %v files currently open", n, len(backend.files)) 1161 | } 1162 | 1163 | filename := backend.filename(n) 1164 | f, err := os.Open(filename) 1165 | if err != nil { 1166 | return err 1167 | } 1168 | 1169 | // Ensure the header's magic is present 1170 | fmagic := make([]byte, len(headerMagic)) 1171 | _, err = f.Read(fmagic) 1172 | if err != nil || headerMagic != string(fmagic) { 1173 | return fmt.Errorf("magic not found in BlobsFile: %v or header not matching", err) 1174 | } 1175 | 1176 | if _, err := f.Seek(int64(headerSize), os.SEEK_SET); err != nil { 1177 | return err 1178 | } 1179 | 1180 | backend.files[n] = f 1181 | openFdsVar.Add(backend.directory, 1) 1182 | 1183 | return nil 1184 | } 1185 | 1186 | func (backend *BlobsFiles) filename(n int) string { 1187 | return filepath.Join(backend.directory, fmt.Sprintf("blobs-%05d", n)) 1188 | } 1189 | 1190 | // writeParityBlobs computes and writes the 4 parity shards using Reed-Solomon 10,4 and write them at 1191 | // end the blobsfile, and write the "data size" (blobsfile size before writing the parity shards). 1192 | func (backend *BlobsFiles) writeParityBlobs(f *os.File, size int) error { 1193 | start := time.Now() 1194 | 1195 | // this will run in a goroutine, add the task in the wait group 1196 | backend.wg.Add(1) 1197 | defer backend.wg.Done() 1198 | 1199 | // First we write the padding blob 1200 | paddingLen := backend.maxBlobsFileSize - (int64(size) + blobOverhead) 1201 | headerEOF := makeHeaderEOF(paddingLen) 1202 | n, err := f.Write(headerEOF) 1203 | if err != nil { 1204 | return fmt.Errorf("failed to write EOF header: %v", err) 1205 | } 1206 | size += n 1207 | 1208 | padding := make([]byte, paddingLen) 1209 | n, err = f.Write(padding) 1210 | if err != nil { 1211 | return fmt.Errorf("failed to write padding 0: %v", err) 1212 | } 1213 | size += n 1214 | 1215 | // We write the data size at the end of the file 1216 | if _, err := f.Seek(0, os.SEEK_END); err != nil { 1217 | return err 1218 | } 1219 | 1220 | // Read the whole blobsfile 1221 | fdata := make([]byte, size) 1222 | if _, err := f.ReadAt(fdata, 0); err != nil { 1223 | return err 1224 | } 1225 | 1226 | // Split into shards 1227 | shards, err := backend.rse.Split(fdata) 1228 | if err != nil { 1229 | return err 1230 | } 1231 | // Create the parity shards 1232 | if err := backend.rse.Encode(shards); err != nil { 1233 | return err 1234 | } 1235 | 1236 | // Save the parity blobs 1237 | parityBlobs := shards[dataShards:] 1238 | for _, parityBlob := range parityBlobs { 1239 | _, parityBlobEncoded := backend.encodeBlob(parityBlob, flagParityBlob) 1240 | 1241 | n, err := f.Write(parityBlobEncoded) 1242 | // backend.size += int64(len(parityBlobEncoded)) 1243 | if err != nil || n != len(parityBlobEncoded) { 1244 | return fmt.Errorf("error writing parity blob (%v,%v)", err, n) 1245 | } 1246 | } 1247 | 1248 | // Fsync 1249 | if err = f.Sync(); err != nil { 1250 | return err 1251 | } 1252 | 1253 | if err := f.Close(); err != nil { 1254 | return err 1255 | } 1256 | 1257 | backend.log("parity blobs created successfully (in %s)", time.Since(start)) 1258 | return nil 1259 | } 1260 | 1261 | // Put save a new blob, hash must be the blake2b hash hex-encoded of the data. 1262 | // 1263 | // If the blob is already stored, then Put will be a no-op. 1264 | // So it's not necessary to make call Exists before saving a new blob. 1265 | func (backend *BlobsFiles) Put(hash string, data []byte) (err error) { 1266 | // Acquire the lock 1267 | backend.Lock() 1268 | defer backend.Unlock() 1269 | 1270 | backend.wg.Add(1) 1271 | defer backend.wg.Done() 1272 | 1273 | // Check if any async error is stored 1274 | if err := backend.lastError(); err != nil { 1275 | return err 1276 | } 1277 | 1278 | // Ensure the data is not already stored 1279 | exists, err := backend.index.checkPos(hash) 1280 | if err != nil { 1281 | return err 1282 | } 1283 | if exists { 1284 | return nil 1285 | } 1286 | 1287 | // Encode the blob 1288 | blobSize, blobEncoded := backend.encodeBlob(data, flagBlob) 1289 | 1290 | var newBlobsFileNeeded bool 1291 | 1292 | // Ensure the blosfile size won't exceed the maxBlobsFileSize 1293 | if backend.size+int64(blobSize+blobOverhead) > backend.maxBlobsFileSize { 1294 | var f *os.File 1295 | f = backend.current 1296 | backend.current = nil 1297 | newBlobsFileNeeded = true 1298 | 1299 | // When restoring, the latest opened blob may already have the parity blobs written 1300 | // TODO(tsileo): make this cleaner 1301 | if backend.size < backend.maxBlobsFileSize { 1302 | 1303 | // This goroutine will write the parity blobs and close the file 1304 | go func(f *os.File, size int, n int) { 1305 | // Write some parity blobs at the end of the blobsfile using Reed-Solomon erasure coding 1306 | if err := backend.writeParityBlobs(f, size); err != nil { 1307 | backend.setLastError(err) 1308 | } 1309 | if backend.blobsFilesSealedFunc != nil { 1310 | backend.blobsFilesSealedFunc(backend.filename(n)) 1311 | } 1312 | }(f, int(backend.size), backend.n) 1313 | } 1314 | } 1315 | 1316 | if newBlobsFileNeeded { 1317 | // Archive this blobsfile, start by creating a new one 1318 | backend.n++ 1319 | if err := backend.wopen(backend.n); err != nil { 1320 | panic(err) 1321 | } 1322 | // Re-open it (since we may need to read blobs from it) 1323 | if err := backend.ropen(backend.n); err != nil { 1324 | panic(err) 1325 | } 1326 | // Update the number of blobsfiles in the index 1327 | if err := backend.saveN(); err != nil { 1328 | panic(err) 1329 | } 1330 | } 1331 | 1332 | // Save the blob in the BlobsFile 1333 | offset := backend.size 1334 | n, err := backend.current.Write(blobEncoded) 1335 | backend.size += int64(len(blobEncoded)) 1336 | if err != nil || n != len(blobEncoded) { 1337 | panic(err) 1338 | } 1339 | 1340 | // Fsync 1341 | if err = backend.current.Sync(); err != nil { 1342 | panic(err) 1343 | } 1344 | 1345 | // Save the blob in the index 1346 | blobPos := &blobPos{n: backend.n, offset: offset, size: blobSize, blobSize: len(data)} 1347 | if err := backend.index.setPos(hash, blobPos); err != nil { 1348 | panic(err) 1349 | } 1350 | 1351 | // Update the expvars 1352 | bytesUploaded.Add(backend.directory, int64(len(blobEncoded))) 1353 | blobsUploaded.Add(backend.directory, 1) 1354 | return 1355 | } 1356 | 1357 | // Exists return true if the blobs is already stored. 1358 | func (backend *BlobsFiles) Exists(hash string) (bool, error) { 1359 | res, err := backend.index.checkPos(hash) 1360 | if err != nil { 1361 | return false, err 1362 | } 1363 | 1364 | return res, nil 1365 | } 1366 | 1367 | func (backend *BlobsFiles) decodeBlob(data []byte) (size int, blob []byte, flag byte) { 1368 | flag = data[hashSize] 1369 | // checkFlag(flag) 1370 | compressionAlgFlag := CompressionAlgorithm(data[hashSize+1]) 1371 | 1372 | size = int(binary.LittleEndian.Uint32(data[hashSize+2 : blobOverhead])) 1373 | 1374 | blob = make([]byte, size) 1375 | copy(blob, data[blobOverhead:]) 1376 | 1377 | var blobDecoded []byte 1378 | var err error 1379 | switch compressionAlgFlag { 1380 | case 0: 1381 | case Snappy: 1382 | blobDecoded, err = snappy.Decode(blobDecoded, blob) 1383 | if err != nil { 1384 | panic(fmt.Errorf("failed to decode blob with Snappy: %v", err)) 1385 | } 1386 | flag = flagBlob 1387 | blob = blobDecoded 1388 | } 1389 | 1390 | h, err := blake2b.New256(nil) 1391 | if err != nil { 1392 | panic(err) 1393 | } 1394 | h.Write(blob) 1395 | 1396 | if !bytes.Equal(h.Sum(nil), data[0:hashSize]) { 1397 | panic(fmt.Errorf("hash doesn't match %x != %x", h.Sum(nil), data[0:hashSize])) 1398 | } 1399 | 1400 | return 1401 | } 1402 | 1403 | func makeHeaderEOF(padSize int64) (h []byte) { 1404 | // Write a hash with only zeroes 1405 | h = make([]byte, blobOverhead) 1406 | // EOF flag, empty second flag 1407 | h[32] = flagEOF 1408 | binary.LittleEndian.PutUint32(h[34:], uint32(padSize)) 1409 | return 1410 | } 1411 | 1412 | func (backend *BlobsFiles) encodeBlob(blob []byte, flag byte) (size int, data []byte) { 1413 | h, err := blake2b.New256(nil) 1414 | if err != nil { 1415 | panic(err) 1416 | } 1417 | h.Write(blob) 1418 | 1419 | var compressionAlgFlag byte 1420 | // Only compress regular blobs 1421 | if flag == flagBlob && backend.compression != 0 { 1422 | var dataEncoded []byte 1423 | switch backend.compression { 1424 | case 0: 1425 | case Snappy: 1426 | dataEncoded = snappy.Encode(nil, blob) 1427 | compressionAlgFlag = byte(Snappy) 1428 | } 1429 | flag = flagCompressed 1430 | blob = dataEncoded 1431 | } 1432 | 1433 | size = len(blob) 1434 | data = make([]byte, len(blob)+blobOverhead) 1435 | 1436 | copy(data[:], h.Sum(nil)) 1437 | 1438 | // set the flag 1439 | data[hashSize] = flag 1440 | data[hashSize+1] = compressionAlgFlag 1441 | 1442 | binary.LittleEndian.PutUint32(data[hashSize+2:], uint32(size)) 1443 | 1444 | copy(data[blobOverhead:], blob) 1445 | 1446 | return 1447 | } 1448 | 1449 | // BlobPos return the index entry for the given hash 1450 | func (backend *BlobsFiles) blobPos(hash string) (*blobPos, error) { 1451 | return backend.index.getPos(hash) 1452 | } 1453 | 1454 | // Size returns the blob size for the given hash. 1455 | func (backend *BlobsFiles) Size(hash string) (int, error) { 1456 | if err := backend.lastError(); err != nil { 1457 | return 0, err 1458 | } 1459 | 1460 | // Fetch the index entry 1461 | blobPos, err := backend.index.getPos(hash) 1462 | if err != nil { 1463 | return 0, fmt.Errorf("error fetching GetPos: %v", err) 1464 | } 1465 | 1466 | // No index entry found, returns an error 1467 | if blobPos == nil { 1468 | if err == nil { 1469 | return 0, ErrBlobNotFound 1470 | } 1471 | return 0, err 1472 | } 1473 | 1474 | return blobPos.blobSize, nil 1475 | } 1476 | 1477 | // Get returns the blob for the given hash. 1478 | func (backend *BlobsFiles) Get(hash string) ([]byte, error) { 1479 | if err := backend.lastError(); err != nil { 1480 | return nil, err 1481 | } 1482 | 1483 | // Fetch the index entry 1484 | blobPos, err := backend.index.getPos(hash) 1485 | if err != nil { 1486 | return nil, fmt.Errorf("error fetching GetPos: %v", err) 1487 | } 1488 | 1489 | // No index entry found, returns an error 1490 | if blobPos == nil { 1491 | if err == nil { 1492 | return nil, ErrBlobNotFound 1493 | } 1494 | return nil, err 1495 | } 1496 | 1497 | // Read the encoded blob from the BlobsFile 1498 | data := make([]byte, blobPos.size+blobOverhead) 1499 | n, err := backend.files[blobPos.n].ReadAt(data, int64(blobPos.offset)) 1500 | if err != nil { 1501 | return nil, fmt.Errorf("error reading blob: %v / blobsfile: %+v", err, backend.files[blobPos.n]) 1502 | } 1503 | 1504 | // Ensure the data length is expcted 1505 | if n != blobPos.size+blobOverhead { 1506 | return nil, fmt.Errorf("error reading blob %v, read %v, expected %v+%v", hash, n, blobPos.size, blobOverhead) 1507 | } 1508 | 1509 | // Decode the blob 1510 | blobSize, blob, _ := backend.decodeBlob(data) 1511 | if blobSize != blobPos.size { 1512 | return nil, fmt.Errorf("bad blob %v encoded size, got %v, expected %v", hash, n, blobSize) 1513 | } 1514 | 1515 | // Update the expvars 1516 | bytesDownloaded.Add(backend.directory, int64(blobSize)) 1517 | blobsUploaded.Add(backend.directory, 1) 1518 | 1519 | return blob, nil 1520 | } 1521 | 1522 | // Enumerate outputs all the blobs into the given chan (ordered lexicographically). 1523 | func (backend *BlobsFiles) Enumerate(blobs chan<- *Blob, start, end string, limit int) error { 1524 | defer close(blobs) 1525 | backend.Lock() 1526 | defer backend.Unlock() 1527 | 1528 | if err := backend.lastError(); err != nil { 1529 | return err 1530 | } 1531 | 1532 | s, err := hex.DecodeString(start) 1533 | if err != nil { 1534 | return err 1535 | } 1536 | 1537 | // Enumerate the raw index directly 1538 | endBytes := []byte(end) 1539 | enum := backend.index.db.Range(formatKey(blobPosKey, s), endBytes, false) 1540 | defer enum.Close() 1541 | k, _, err := enum.Next() 1542 | 1543 | i := 0 1544 | for ; err == nil; k, _, err = enum.Next() { 1545 | 1546 | if limit != 0 && i == limit { 1547 | return nil 1548 | } 1549 | 1550 | hash := hex.EncodeToString(k[1:]) 1551 | blobPos, err := backend.blobPos(hash) 1552 | if err != nil { 1553 | return nil 1554 | } 1555 | 1556 | // Remove the BlobPosKey prefix byte 1557 | blobs <- &Blob{ 1558 | Hash: hash, 1559 | Size: blobPos.blobSize, 1560 | N: blobPos.n, 1561 | } 1562 | 1563 | i++ 1564 | } 1565 | 1566 | return nil 1567 | } 1568 | 1569 | // Enumerate outputs all the blobs into the given chan (ordered lexicographically). 1570 | func (backend *BlobsFiles) EnumeratePrefix(blobs chan<- *Blob, prefix string, limit int) error { 1571 | defer close(blobs) 1572 | backend.Lock() 1573 | defer backend.Unlock() 1574 | 1575 | if err := backend.lastError(); err != nil { 1576 | return err 1577 | } 1578 | 1579 | s, err := hex.DecodeString(prefix) 1580 | if err != nil { 1581 | return err 1582 | } 1583 | 1584 | // Enumerate the raw index directly 1585 | enum := backend.index.db.PrefixRange(formatKey(blobPosKey, s), false) 1586 | defer enum.Close() 1587 | k, _, err := enum.Next() 1588 | 1589 | i := 0 1590 | for ; err == nil; k, _, err = enum.Next() { 1591 | 1592 | if limit != 0 && i == limit { 1593 | return nil 1594 | } 1595 | 1596 | hash := hex.EncodeToString(k[1:]) 1597 | blobPos, err := backend.blobPos(hash) 1598 | if err != nil { 1599 | return nil 1600 | } 1601 | 1602 | // Remove the BlobPosKey prefix byte 1603 | blobs <- &Blob{ 1604 | Hash: hash, 1605 | Size: blobPos.blobSize, 1606 | N: blobPos.n, 1607 | } 1608 | 1609 | i++ 1610 | } 1611 | 1612 | return nil 1613 | } 1614 | -------------------------------------------------------------------------------- /blobsfile_test.go: -------------------------------------------------------------------------------- 1 | package blobsfile 2 | 3 | import ( 4 | "bytes" 5 | "crypto/rand" 6 | "fmt" 7 | "io/ioutil" 8 | mrand "math/rand" 9 | "os" 10 | "reflect" 11 | "sort" 12 | "testing" 13 | 14 | "a4.io/blobstash/pkg/hashutil" 15 | ) 16 | 17 | func check(e error) { 18 | if e != nil { 19 | panic(e) 20 | } 21 | } 22 | 23 | func TestScan(t *testing.T) { 24 | // blobs, err := ScanBlobsFile("/home/thomas/Proj/blobstash/blobstash_yzadat1111111/blobs/blobs-00000") 25 | // check(err) 26 | // t.Logf("blobs=%q", blobs) 27 | } 28 | 29 | func BenchmarkBlobsFilePut512B(b *testing.B) { 30 | back, err := New(&Opts{Directory: "./tmp_blobsfile_test"}) 31 | check(err) 32 | defer back.Close() 33 | defer os.RemoveAll("./tmp_blobsfile_test") 34 | benchmarkBlobsFilePut(back, 512, b) 35 | } 36 | 37 | func BenchmarkBlobsFilePut512KB(b *testing.B) { 38 | back, err := New(&Opts{Directory: "./tmp_blobsfile_test"}) 39 | check(err) 40 | defer back.Close() 41 | defer os.RemoveAll("./tmp_blobsfile_test") 42 | benchmarkBlobsFilePut(back, 512000, b) 43 | } 44 | 45 | func BenchmarkBlobsFilePut2MB(b *testing.B) { 46 | back, err := New(&Opts{Directory: "./tmp_blobsfile_test"}) 47 | check(err) 48 | defer back.Close() 49 | defer os.RemoveAll("./tmp_blobsfile_test") 50 | benchmarkBlobsFilePut(back, 2000000, b) 51 | } 52 | 53 | func BenchmarkBlobsFilePut512BCompressed(b *testing.B) { 54 | back, err := New(&Opts{Directory: "./tmp_blobsfile_test", Compression: Snappy}) 55 | check(err) 56 | defer back.Close() 57 | defer os.RemoveAll("./tmp_blobsfile_test") 58 | benchmarkBlobsFilePut(back, 512, b) 59 | } 60 | 61 | func BenchmarkBlobsFilePut512KBCompressed(b *testing.B) { 62 | back, err := New(&Opts{Directory: "./tmp_blobsfile_test", Compression: Snappy}) 63 | check(err) 64 | defer back.Close() 65 | defer os.RemoveAll("./tmp_blobsfile_test") 66 | benchmarkBlobsFilePut(back, 512000, b) 67 | } 68 | 69 | func BenchmarkBlobsFilePut2MBCompressed(b *testing.B) { 70 | back, err := New(&Opts{Directory: "./tmp_blobsfile_test", Compression: Snappy}) 71 | check(err) 72 | defer back.Close() 73 | defer os.RemoveAll("./tmp_blobsfile_test") 74 | benchmarkBlobsFilePut(back, 2000000, b) 75 | } 76 | 77 | func benchmarkBlobsFilePut(back *BlobsFiles, blobSize int, b *testing.B) { 78 | // b.ResetTimer() 79 | // b.StopTimer() 80 | for i := 0; i < b.N; i++ { 81 | b.StopTimer() 82 | h, blob := randBlob(blobSize) 83 | b.StartTimer() 84 | if err := back.Put(h, blob); err != nil { 85 | panic(err) 86 | } 87 | b.StopTimer() 88 | } 89 | b.SetBytes(int64(blobSize)) 90 | } 91 | 92 | func TestBlobsFileReedSolomon(t *testing.T) { 93 | b, err := New(&Opts{Directory: "./tmp_blobsfile_test", BlobsFileSize: 16000000}) 94 | check(err) 95 | defer os.RemoveAll("./tmp_blobsfile_test") 96 | testParity(t, b, true, nil) 97 | fname := b.filename(0) 98 | b.Close() 99 | // // Corrupt the file 100 | 101 | // f, err := os.OpenFile(fname, os.O_RDWR, 0755) 102 | // if err != nil { 103 | // panic(err) 104 | // } 105 | // FIXME(tsileo): test this 106 | // if _, err := f.Seek(defaultMaxBlobsFileSize/10*3, os.SEEK_SET); err != nil { 107 | // if _, err := f.Seek(defaultMaxBlobsFileSize/10, os.SEEK_SET); err != nil { 108 | // if _, err := f.Seek(16000000/10*2, os.SEEK_SET); err != nil { 109 | data, err := ioutil.ReadFile(fname) 110 | if err != nil { 111 | panic(err) 112 | } 113 | punchOffset := int64(16000000/10*5) - 10 114 | t.Logf("punch at %d\n", punchOffset) 115 | fmt.Printf("punch at %d/%d\n", punchOffset, 16000000) 116 | ndata := []byte("blobsfilelol") 117 | copy(data[punchOffset:punchOffset+int64(len(ndata))], ndata) 118 | if err := ioutil.WriteFile(fname, []byte(data), 0644); err != nil { 119 | panic(err) 120 | } 121 | // Reopen the db 122 | b, err = New(&Opts{Directory: "./tmp_blobsfile_test", BlobsFileSize: 16000000}) 123 | check(err) 124 | defer b.Close() 125 | // Ensure we can recover from this corruption 126 | cb := func(err error) error { 127 | if err != nil { 128 | if err := b.scan(nil); err != nil { 129 | return b.checkBlobsFile(err.(*corruptedError)) 130 | } 131 | panic("should not happen") 132 | } 133 | return nil 134 | } 135 | testParity(t, b, false, cb) 136 | packs := b.SealedPacks() 137 | t.Logf("packs=%+v", packs) 138 | } 139 | 140 | func TestBlobsFileReedSolomonReindex(t *testing.T) { 141 | b, err := New(&Opts{Directory: "./tmp_blobsfile_test", BlobsFileSize: 16000000}) 142 | check(err) 143 | defer os.RemoveAll("./tmp_blobsfile_test") 144 | testParity(t, b, true, nil) 145 | fname := b.filename(0) 146 | b.Close() 147 | // // Corrupt the file 148 | 149 | // f, err := os.OpenFile(fname, os.O_RDWR, 0755) 150 | // if err != nil { 151 | // panic(err) 152 | // } 153 | // FIXME(tsileo): test this 154 | // if _, err := f.Seek(defaultMaxBlobsFileSize/10*3, os.SEEK_SET); err != nil { 155 | // if _, err := f.Seek(defaultMaxBlobsFileSize/10, os.SEEK_SET); err != nil { 156 | // if _, err := f.Seek(16000000/10*2, os.SEEK_SET); err != nil { 157 | data, err := ioutil.ReadFile(fname) 158 | if err != nil { 159 | panic(err) 160 | } 161 | punchOffset := int64(16000000/10*5) - 10 162 | t.Logf("punch at %d\n", punchOffset) 163 | fmt.Printf("punch at %d/%d\n", punchOffset, 16000000) 164 | ndata := []byte("blobsfilelol") 165 | copy(data[punchOffset:punchOffset+int64(len(ndata))], ndata) 166 | if err := ioutil.WriteFile(fname, []byte(data), 0644); err != nil { 167 | panic(err) 168 | } 169 | // Reopen the db 170 | b, err = New(&Opts{Directory: "./tmp_blobsfile_test", BlobsFileSize: 16000000}) 171 | check(err) 172 | defer b.Close() 173 | if err := b.RebuildIndex(); err != nil { 174 | t.Errorf("failed to rebuild index: %v", err) 175 | } 176 | } 177 | 178 | func TestBlobsFileReedSolomonWithCompression(t *testing.T) { 179 | b, err := New(&Opts{Directory: "./tmp_blobsfile_test", BlobsFileSize: 16000000}) 180 | check(err) 181 | defer b.Close() 182 | defer os.RemoveAll("./tmp_blobsfile_test") 183 | testParity(t, b, true, nil) 184 | } 185 | 186 | func testParity(t *testing.T, b *BlobsFiles, insert bool, cb func(error) error) ([]string, [][]byte) { 187 | hashes := []string{} 188 | blobs := [][]byte{} 189 | if insert { 190 | for i := 0; i < 31+10; i++ { 191 | h, blob := randBlob(512000) 192 | hashes = append(hashes, h) 193 | blobs = append(blobs, blob) 194 | if err := b.Put(h, blob); err != nil { 195 | panic(err) 196 | } 197 | } 198 | } 199 | if err := b.checkParityBlobs(0); err != nil { 200 | if cb == nil { 201 | panic(err) 202 | } 203 | if err := cb(err); err != nil { 204 | panic(err) 205 | } 206 | } 207 | return hashes, blobs 208 | } 209 | 210 | func randBlob(size int) (string, []byte) { 211 | blob := make([]byte, size) 212 | if _, err := rand.Read(blob); err != nil { 213 | panic(err) 214 | } 215 | return hashutil.Compute(blob), blob 216 | } 217 | 218 | func TestBlobsFilePutIdempotent(t *testing.T) { 219 | back, err := New(&Opts{Directory: "./tmp_blobsfile_test", Compression: Snappy}) 220 | check(err) 221 | defer back.Close() 222 | defer os.RemoveAll("./tmp_blobsfile_test") 223 | h, blob := randBlob(512) 224 | for i := 0; i < 10; i++ { 225 | if err := back.Put(h, blob); err != nil { 226 | panic(err) 227 | } 228 | } 229 | stats, err := back.Stats() 230 | if err != nil { 231 | panic(err) 232 | } 233 | if stats.BlobsCount != 1 || stats.BlobsSize != 512 { 234 | t.Errorf("bad stats: %+v", stats) 235 | } 236 | } 237 | 238 | func TestBlobsFileBlobPutGetEnumerate(t *testing.T) { 239 | b, err := New(&Opts{Directory: "./tmp_blobsfile_test", Compression: Snappy}) 240 | check(err) 241 | defer os.RemoveAll("./tmp_blobsfile_test") 242 | hashes, blobs := testBackendPutGetEnumerateReindexGetEnumerate(t, b, 100) 243 | b.Close() 244 | // Test we can still read everything when closing/reopening the blobsfile 245 | b, err = New(&Opts{Directory: "./tmp_blobsfile_test"}) 246 | check(err) 247 | prefixes := map[string][]string{} 248 | for _, h := range hashes { 249 | if _, ok := prefixes[h[0:2]]; !ok { 250 | prefixes[h[0:2]] = []string{} 251 | } 252 | prefixes[h[0:2]] = append(prefixes[h[0:2]], h) 253 | } 254 | testBackendEnumerate(t, b, hashes, "", "\xfe") 255 | for prefix, phashes := range prefixes { 256 | testBackendEnumerate2(t, b, phashes, prefix, "") 257 | } 258 | testBackendGet(t, b, hashes, blobs) 259 | if err := b.Close(); err != nil { 260 | panic(err) 261 | } 262 | // Try with the index and removed and test re-indexing 263 | b, err = New(&Opts{Directory: "./tmp_blobsfile_test", Compression: Snappy}) 264 | check(err) 265 | if err := b.RebuildIndex(); err != nil { 266 | panic(err) 267 | } 268 | testBackendEnumerate(t, b, hashes, "", "\xfe") 269 | testBackendGet(t, b, hashes, blobs) 270 | } 271 | 272 | func backendPut(t *testing.T, b *BlobsFiles, blobsCount int) ([]string, [][]byte) { 273 | blobs := [][]byte{} 274 | hashes := []string{} 275 | // TODO(tsileo): 50 blobs if in short mode 276 | for i := 0; i < blobsCount; i++ { 277 | h, blob := randBlob(mrand.Intn(4000000-32) + 32) 278 | hashes = append(hashes, h) 279 | blobs = append(blobs, blob) 280 | if err := b.Put(h, blob); err != nil { 281 | panic(err) 282 | } 283 | } 284 | 285 | stats, err := b.Stats() 286 | if err != nil { 287 | panic(err) 288 | } 289 | fmt.Printf("stats=%+v\n", stats) 290 | 291 | return hashes, blobs 292 | } 293 | 294 | func testBackendPutGetEnumerate(t *testing.T, b *BlobsFiles, blobsCount int) ([]string, [][]byte) { 295 | hashes, blobs := backendPut(t, b, blobsCount) 296 | testBackendGet(t, b, hashes, blobs) 297 | testBackendEnumerate(t, b, hashes, "", "\xfe") 298 | return hashes, blobs 299 | } 300 | 301 | func testBackendPutGetEnumerateReindexGetEnumerate(t *testing.T, b *BlobsFiles, blobsCount int) ([]string, [][]byte) { 302 | hashes, blobs := backendPut(t, b, blobsCount) 303 | testBackendGet(t, b, hashes, blobs) 304 | testBackendEnumerate(t, b, hashes, "", "\xfe") 305 | if err := b.RebuildIndex(); err != nil { 306 | panic(err) 307 | } 308 | testBackendGet(t, b, hashes, blobs) 309 | testBackendEnumerate(t, b, hashes, "", "\xfe") 310 | return hashes, blobs 311 | } 312 | 313 | func testBackendGet(t *testing.T, b *BlobsFiles, hashes []string, blobs [][]byte) { 314 | blobsIndex := map[string]bool{} 315 | for _, blob := range blobs { 316 | blobsIndex[hashutil.Compute(blob)] = true 317 | } 318 | for _, h := range hashes { 319 | if _, err := b.Get(h); err != nil { 320 | panic(err) 321 | } 322 | _, ok := blobsIndex[h] 323 | if !ok { 324 | t.Errorf("blob %s should be index", h) 325 | } 326 | delete(blobsIndex, h) 327 | } 328 | if len(blobsIndex) > 0 { 329 | t.Errorf("index should have been emptied, got len %d", len(blobsIndex)) 330 | } 331 | } 332 | 333 | func testBackendEnumerate2(t *testing.T, b *BlobsFiles, hashes []string, start, end string) []string { 334 | sort.Strings(hashes) 335 | bchan := make(chan *Blob) 336 | errc := make(chan error, 1) 337 | go func() { 338 | errc <- b.EnumeratePrefix(bchan, start, 0) 339 | }() 340 | enumHashes := []string{} 341 | for ref := range bchan { 342 | enumHashes = append(enumHashes, ref.Hash) 343 | } 344 | if err := <-errc; err != nil { 345 | panic(err) 346 | } 347 | if !sort.StringsAreSorted(enumHashes) { 348 | t.Errorf("enum hashes should already be sorted") 349 | } 350 | if !reflect.DeepEqual(hashes, enumHashes) { 351 | t.Errorf("bad enumerate results %q %q", hashes, enumHashes) 352 | } 353 | return enumHashes 354 | } 355 | 356 | func testBackendEnumerate(t *testing.T, b *BlobsFiles, hashes []string, start, end string) []string { 357 | sort.Strings(hashes) 358 | bchan := make(chan *Blob) 359 | errc := make(chan error, 1) 360 | go func() { 361 | errc <- b.Enumerate(bchan, start, end, 0) 362 | }() 363 | enumHashes := []string{} 364 | for ref := range bchan { 365 | enumHashes = append(enumHashes, ref.Hash) 366 | } 367 | if err := <-errc; err != nil { 368 | panic(err) 369 | } 370 | if !sort.StringsAreSorted(enumHashes) { 371 | t.Errorf("enum hashes should already be sorted") 372 | } 373 | if !reflect.DeepEqual(hashes, enumHashes) { 374 | t.Errorf("bad enumerate results %q %q", hashes, enumHashes) 375 | } 376 | return enumHashes 377 | } 378 | 379 | func TestBlobsFileBlobEncodingNoCompression(t *testing.T) { 380 | b, err := New(&Opts{Directory: "./tmp_blobsfile_test"}) 381 | check(err) 382 | defer b.Close() 383 | defer os.RemoveAll("./tmp_blobsfile_test") 384 | _, blob := randBlob(512) 385 | _, data := b.encodeBlob(blob, flagBlob) 386 | size, blob2, f := b.decodeBlob(data) 387 | if f != flagBlob { 388 | t.Errorf("bad flag, got %v, expected %v", f, flagBlob) 389 | } 390 | if size != 512 || !bytes.Equal(blob, blob2) { 391 | t.Errorf("Error blob encoding, got size:%v, expected:512, got blob:%v, expected:%v", size, blob2[:10], blob[:10]) 392 | } 393 | } 394 | 395 | func TestBlobsFileBlobEncoding(t *testing.T) { 396 | b, err := New(&Opts{Directory: "./tmp_blobsfile_test"}) 397 | check(err) 398 | defer b.Close() 399 | defer os.RemoveAll("./tmp_blobsfile_test") 400 | _, blob := randBlob(512) 401 | _, data := b.encodeBlob(blob, flagBlob) 402 | size, blob2, f := b.decodeBlob(data) 403 | if f != flagBlob { 404 | t.Errorf("bad flag, got %v, expected %v", f, flagBlob) 405 | } 406 | // Don't check the size are as the returned size is the size of the compressed blob 407 | if !bytes.Equal(blob, blob2) { 408 | t.Errorf("Error blob encoding, got size:%v, expected:512, got blob:%v, expected:%v", size, blob2[:10], blob[:10]) 409 | } 410 | packs := b.SealedPacks() 411 | t.Logf("packs=%+v", packs) 412 | } 413 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module a4.io/blobsfile 2 | 3 | require ( 4 | a4.io/blobstash v0.0.0-20200419182230-3bd2ac0cc88e 5 | github.com/golang/snappy v0.0.1 6 | github.com/klauspost/reedsolomon v1.9.4 7 | golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 8 | golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f // indirect 9 | ) 10 | 11 | go 1.13 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | a4.io/blobsfile v0.0.0-20181029195936-c742249a3522/go.mod h1:jTrsc9CgnEavpl6Tmowi2bZbGXldVGr5gvkFsS12bKs= 2 | a4.io/blobsfile v0.1.0/go.mod h1:kJFL3M8OxlvHZWzxZ6C9o+ky9iJHmx0jZj59nilWzJM= 3 | a4.io/blobsfile v0.3.8/go.mod h1:ZHwdtHHOeCbaP/dpPnA1FAUrrwXG9GX2ju3OrbufCjQ= 4 | a4.io/blobstash v0.0.0-20181216235946-aa2d4a59f200/go.mod h1:PVI3EM/VmUQAz7pbz/govGO4gHypTF5YWhS56qETj+M= 5 | a4.io/blobstash v0.0.0-20181218201750-765e41187e8a h1:0pDZVLBIYsY8Hak7c8KzleuWppAugOaTyoWadYjiPcw= 6 | a4.io/blobstash v0.0.0-20181218201750-765e41187e8a/go.mod h1:QH1JUxPtdWiC/hCXrfzS03p5tX9mqALuBzUd2yYflso= 7 | a4.io/blobstash v0.0.0-20181225194431-69866d0dc5f5 h1:a4zGyDv924s2g+k34OCh+JDC9stpioU3hPVijBE3mIo= 8 | a4.io/blobstash v0.0.0-20181225194431-69866d0dc5f5/go.mod h1:YtIAw8g6uiD0ODIvwJWSmEoKpx3M1cZLIiqrhh9NjSU= 9 | a4.io/blobstash v0.0.0-20191111100838-e9ad8df67971 h1:xaRQJIfUoJC4OhRIhGEtypAyM54EC/39hcddDEkOBiA= 10 | a4.io/blobstash v0.0.0-20191111100838-e9ad8df67971/go.mod h1:YrXxObZG8guBDGhUolW6/MlbngxiUk7+H5liKBxDd3s= 11 | a4.io/blobstash v0.0.0-20191229152948-bc315003dfb2/go.mod h1:YrXxObZG8guBDGhUolW6/MlbngxiUk7+H5liKBxDd3s= 12 | a4.io/blobstash v0.0.0-20200131212433-e97337e98c5e/go.mod h1:SrGKNSU1yJ6esqF0aqD53kjCrpHoEl28kuQdKHUvXsc= 13 | a4.io/blobstash v0.0.0-20200202192640-d62b4924ec01/go.mod h1:38J1ivXRQHMj5UZasbPi7o/P5nc7snAeaAu7lIXL1gM= 14 | a4.io/blobstash v0.0.0-20200221184026-dea61889318d/go.mod h1:rBeIGLPLaPL47+6gKO4jCld+sBWpC55Va6XvxLrYKzs= 15 | a4.io/blobstash v0.0.0-20200221185322-50bf3c15ebe0/go.mod h1:6zWY2+AG69x/Ijl4B+gcztuKCx30X/dGFAT6Hh/udl4= 16 | a4.io/blobstash v0.0.0-20200311204339-04f83bc3d616/go.mod h1:XwNV/qV4/yuXdjfQgBqDLTCK4i2mlmvfI9UrkCPrt2M= 17 | a4.io/blobstash v0.0.0-20200419182230-3bd2ac0cc88e h1:QkriBQZo5dleGGMAa15A8u99E0HTg/fXzz2TBhO7VKQ= 18 | a4.io/blobstash v0.0.0-20200419182230-3bd2ac0cc88e/go.mod h1:mXp/DXVs6sWxIKCr2mqyZo+A9FdjbR7W693yHx+uCBA= 19 | a4.io/gluapp v0.0.0-20181203183836-c136dc4e9123/go.mod h1:rK/CQwI+tDICKCR1szNtBP0rJdH1LCrO/ZnculcIjWI= 20 | a4.io/gluapp v0.0.0-20181217122610-c6ba9b02f21b/go.mod h1:hDz8O30eiYv+1bAFzssTvbRaLy27xwk7pdR7v2md7Ew= 21 | a4.io/gluapp v0.0.0-20181218195258-2be1706b2908 h1:4X4w3ef5+gyUErHpxdyMoHXSKUCY9naICJGwdwceLc4= 22 | a4.io/gluapp v0.0.0-20181218195258-2be1706b2908/go.mod h1:hDz8O30eiYv+1bAFzssTvbRaLy27xwk7pdR7v2md7Ew= 23 | a4.io/gluapp v0.0.0-20190530193846-2ad05291e3be/go.mod h1:46QpRqVnBeahZFsw+6+/NPhwex2jY7ZrFvPoRdhgu5Y= 24 | a4.io/gluapp v0.0.0-20200131211012-723a51b0e790/go.mod h1:XvZKnPX9E8UAoNcbJ5ESYCr9c6yYTRyv/4iDsDt/Eyo= 25 | a4.io/gluapp v0.0.0-20200202115504-51581a8e4642/go.mod h1:jgLJ6nULqKUJSJsT4W0KuNhh/lqXo6cS/pDuuESx4ko= 26 | a4.io/gluapp v0.0.0-20200214202429-b4a08105811e/go.mod h1:P313jMDJOXJwnAA0qtWCScB6TQPfZ/QwTUah9aO3t4Y= 27 | a4.io/gluapp v0.0.0-20200221184138-44fb2766d27d/go.mod h1:SnPbw2WapvLDMfarcCs/pqYhYUvQqCk5u6TJjni5M3k= 28 | a4.io/gluapp v0.0.0-20200311203905-eb3c48991ada/go.mod h1:A3U6Yc1zqp5Bnsz7PxrlYmbZ4iEN/LBKeuuwaxpLBMY= 29 | a4.io/gluapp v0.0.0-20200404171232-054f285d8e63/go.mod h1:vRUnOCoU0xprQIH3uvQJG2NnyqJGFmnqBTk528JCEwg= 30 | a4.io/gluarequire2 v0.0.0-20170611121149-66e0eb2c6a9f h1:mfEWN0Dd2AfIXU5WO5ZfqbFVk63Qz5M/CANs182pm+U= 31 | a4.io/gluarequire2 v0.0.0-20170611121149-66e0eb2c6a9f/go.mod h1:t7OhwCmPQfuUf8cjm7n8chSbZt5CTILu+dTLu1MQKjQ= 32 | a4.io/gluarequire2 v0.0.0-20200222094423-7528d5a10bc1/go.mod h1:mPtxfgeyyAcPonI669LtVTsmZfNSklo7kq3zSFej+jk= 33 | a4.io/go/indieauth v1.0.0/go.mod h1:yCJuSTw9d22VdPWrZ8frGLwVOdwscJTiXjG4IgVL0Vw= 34 | a4.io/ssse v0.0.0-20181202155639-1949828a8689/go.mod h1:/4k4qDJv4lDmiIcMs9k/5Rs7bU/1FkIvu42oMyf5A7Y= 35 | bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= 36 | bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= 37 | github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= 38 | github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= 39 | github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= 40 | github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= 41 | github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= 42 | github.com/alecthomas/chroma v0.7.0/go.mod h1:1U/PfCsTALWWYHDnsIQkxEBM0+6LLe0v8+RSVMOwxeY= 43 | github.com/alecthomas/chroma v0.7.1/go.mod h1:gHw09mkX1Qp80JlYbmN9L3+4R5o6DJJ3GRShh+AICNc= 44 | github.com/alecthomas/chroma v0.7.2-0.20200305040604-4f3623dce67a/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s= 45 | github.com/alecthomas/chroma v0.7.2/go.mod h1:fv5SzZPFJbwp2NXJWpFIX7DZS4HgV1K4ew4Pc2OZD9s= 46 | github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= 47 | github.com/alecthomas/kong v0.1.17-0.20190424132513-439c674f7ae0/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI= 48 | github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI= 49 | github.com/alecthomas/kong-hcl v0.1.8-0.20190615233001-b21fea9723c8/go.mod h1:MRgZdU3vrFd05IQ89AxUZ0aYdF39BYoNFa324SodPCA= 50 | github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= 51 | github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= 52 | github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= 53 | github.com/aws/aws-sdk-go v1.16.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 54 | github.com/aws/aws-sdk-go v1.16.11 h1:g/c7gJeVyHoXCxM2fddS85bPGVkBF8s2q8t3fyElegc= 55 | github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 56 | github.com/aws/aws-sdk-go v1.25.31/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 57 | github.com/aws/aws-sdk-go v1.28.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 58 | github.com/aws/aws-sdk-go v1.29.4/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= 59 | github.com/aws/aws-sdk-go v1.29.7/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= 60 | github.com/aws/aws-sdk-go v1.29.22/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= 61 | github.com/aws/aws-sdk-go v1.30.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= 62 | github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f h1:kqbi9lqXLLs+zfWlgo1PIiRQ86n33K1JKotjj4rSYOg= 63 | github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f/go.mod h1:IInt5XRvpiGE09KOk9mmCMLjHhydIhNPKPPFLFBB7L8= 64 | github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= 65 | github.com/carbocation/handlers v0.0.0-20140528190747-c939c6d9ef31 h1:SDMgCFII5drFRIyAaihze9ceRMpTt1FW6Q5jjpc2u4c= 66 | github.com/carbocation/handlers v0.0.0-20140528190747-c939c6d9ef31/go.mod h1:iGISoFvZYz358DFlmHvYFlh4CgRdzPLXB2NJE48x6lY= 67 | github.com/carbocation/interpose v0.0.0-20161206215253-723534742ba3 h1:RtCys6GUprNaPOP04Zuo65wS10PMbSPPZNvIb9xYYLE= 68 | github.com/carbocation/interpose v0.0.0-20161206215253-723534742ba3/go.mod h1:4PGcghc3ZjA/uozANO8lCHo/gnHyMsm8iFYppSkVE/M= 69 | github.com/cespare/trie v0.0.0-20150610204604-3fe1a95cbba9/go.mod h1:MCsKum/O9rTzo1Z6ubBQJKJIm76t+3/4A/cD79RMN1Q= 70 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 71 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 72 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 73 | github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q= 74 | github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= 75 | github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0= 76 | github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= 77 | github.com/cznic/fileutil v0.0.0-20181122101858-4d67cfea8c87 h1:94XgeeTZ+3Xi9zsdgBjP1Byx/wywCImjF8FzQ7OaKdU= 78 | github.com/cznic/fileutil v0.0.0-20181122101858-4d67cfea8c87/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg= 79 | github.com/cznic/internal v0.0.0-20181122101858-3279554c546e h1:58AcyflCe84EONph4gkyo3eDOEQcW5HIPfQBrD76W68= 80 | github.com/cznic/internal v0.0.0-20181122101858-3279554c546e/go.mod h1:olo7eAdKwJdXxb55TKGLiJ6xt1H0/tiiRCWKVLmtjY4= 81 | github.com/cznic/kv v0.0.0-20181122101858-e9cdcade440e h1:8ji4rZgRKWMQUJlPNEzfzCkX7yFAZFR829Mrh7PXxLA= 82 | github.com/cznic/kv v0.0.0-20181122101858-e9cdcade440e/go.mod h1:J9vPsG5aOQu5A836WgCTIb9xkiB9w1birknxIQmyWXY= 83 | github.com/cznic/lldb v1.1.0 h1:AIA+ham6TSJ+XkMe8imQ/g8KPzMUVWAwqUQQdtuMsHs= 84 | github.com/cznic/lldb v1.1.0/go.mod h1:FIZVUmYUVhPwRiPzL8nD/mpFcJ/G7SSXjjXYG4uRI3A= 85 | github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= 86 | github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= 87 | github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 h1:LpMLYGyy67BoAFGda1NeOBQwqlv7nUXpm+rIVHGxZZ4= 88 | github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= 89 | github.com/cznic/zappy v0.0.0-20181122101859-ca47d358d4b1 h1:ytLS5Cgkxq6jObotJ+a13nsejdqzLFPliDf8CQ8OkAA= 90 | github.com/cznic/zappy v0.0.0-20181122101859-ca47d358d4b1/go.mod h1:Y1SNZ4dRUOKXshKUbwUapqNncRrho4mkjQebgEHZLj8= 91 | github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= 92 | github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= 93 | github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= 94 | github.com/dave/jennifer v1.3.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= 95 | github.com/dave/jennifer v1.4.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= 96 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 97 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 98 | github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= 99 | github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= 100 | github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= 101 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 102 | github.com/e3b0c442/warp v0.6.1/go.mod h1:pI39WXOdQwVZVP3TYgv6SN995Txh5RyDXebPibH1Rg8= 103 | github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= 104 | github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= 105 | github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= 106 | github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= 107 | github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= 108 | github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= 109 | github.com/evanphx/json-patch v4.1.0+incompatible h1:K1MDoo4AZ4wU0GIU/fPmtZg7VpzLjCxu+UwBD1FvwOc= 110 | github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 111 | github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= 112 | github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= 113 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 114 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 115 | github.com/fxamacker/cbor/v2 v2.2.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= 116 | github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= 117 | github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= 118 | github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab h1:xveKWz2iaueeTaUgdetzel+U7exyigDYBryyVfV/rZk= 119 | github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= 120 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 121 | github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= 122 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 123 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 124 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 125 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 126 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 127 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 128 | github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 129 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 130 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 131 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 132 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 133 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 134 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= 135 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 136 | github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= 137 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 138 | github.com/gomarkdown/markdown v0.0.0-20181104084050-d1d0edeb5d85 h1:C0jjY7t3mKMmf4hXf4tYmc4KOZLx1K0em8kq685+JBM= 139 | github.com/gomarkdown/markdown v0.0.0-20181104084050-d1d0edeb5d85/go.mod h1:gmFANS06wAVmF0B9yi65QKsRmPQ97tze7FRLswua+OY= 140 | github.com/gomarkdown/markdown v0.0.0-20200127000047-1813ea067497/go.mod h1:aii0r/K0ZnHv7G0KF7xy1v0A7s2Ljrb5byB7MO5p6TU= 141 | github.com/goods/httpbuf v0.0.0-20120503183857-5709e9bb814c h1:kES4WSo15F5Rejf0L5d6kJzZhDRs/0SEvb39I8H6H7g= 142 | github.com/goods/httpbuf v0.0.0-20120503183857-5709e9bb814c/go.mod h1:cHMBumiwaaRxRQ6NT8sU3zQSkXbYaPjbBcXa8UgTzAE= 143 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 144 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 145 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 146 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 147 | github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= 148 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 149 | github.com/gorilla/csrf v1.6.0/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= 150 | github.com/gorilla/handlers v1.4.0 h1:XulKRWSQK5uChr4pEgSE4Tc/OcmnU9GJuSwdog/tZsA= 151 | github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= 152 | github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= 153 | github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= 154 | github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= 155 | github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 156 | github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 157 | github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 158 | github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= 159 | github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= 160 | github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= 161 | github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= 162 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 163 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 164 | github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 165 | github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 166 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 167 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 168 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 169 | github.com/inconshreveable/log15 v0.0.0-20180818164646-67afb5ed74ec h1:CGkYB1Q7DSsH/ku+to+foV4agt2F2miquaLUgF6L178= 170 | github.com/inconshreveable/log15 v0.0.0-20180818164646-67afb5ed74ec/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= 171 | github.com/inconshreveable/log15 v0.0.0-20200109203555-b30bc20e4fd1/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= 172 | github.com/interpose/middleware v0.0.0-20150216143757-05ed56ed52fa/go.mod h1:eMb40EJpwUTKSRRKJ3sol3zWoy49dJXNxx7bdciFeYo= 173 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= 174 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= 175 | github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 176 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= 177 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 178 | github.com/jmespath/go-jmespath v0.0.0-20200310193758-2437e8417af5/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= 179 | github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= 180 | github.com/justinas/nosurf v0.0.0-20181122113328-3af30e51c05b h1:fWjiIutptAhQwIoCjCEsyCx6KtaHJ6WyqCLdmFJ3udQ= 181 | github.com/justinas/nosurf v0.0.0-20181122113328-3af30e51c05b/go.mod h1:Aucr5I5chr4OCuuVB4LTuHVrKHBuyRSo7vM2hqrcb7E= 182 | github.com/justinas/nosurf v1.1.0/go.mod h1:ALpWdSbuNGy2lZWtyXdjkYv4edL23oSEgfBT1gPJ5BQ= 183 | github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8= 184 | github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= 185 | github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= 186 | github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= 187 | github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 188 | github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= 189 | github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 190 | github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= 191 | github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 192 | github.com/klauspost/reedsolomon v1.7.0/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= 193 | github.com/klauspost/reedsolomon v1.8.0 h1:lvvOkvk64cE1EGbBIgFk7WSOOsI1GexpuLiT7zjab6g= 194 | github.com/klauspost/reedsolomon v1.8.0/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= 195 | github.com/klauspost/reedsolomon v1.9.3 h1:N/VzgeMfHmLc+KHMD1UL/tNkfXAt8FnUqlgXGIduwAY= 196 | github.com/klauspost/reedsolomon v1.9.3/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= 197 | github.com/klauspost/reedsolomon v1.9.4 h1:FB9jDBGqUNyhUg4Gszz384ulFqVSc61Pdap+HRPgnSo= 198 | github.com/klauspost/reedsolomon v1.9.4/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= 199 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 200 | github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 201 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 202 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 203 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 204 | github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= 205 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 206 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 207 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 208 | github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= 209 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 210 | github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 211 | github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 212 | github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= 213 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 214 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 215 | github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= 216 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 217 | github.com/meatballhat/negroni-logrus v0.0.0-20170801195057-31067281800f h1:V6GHkMOIsnpGDasS1iYiNxEYTY8TmyjQXEF8PqYkKQ8= 218 | github.com/meatballhat/negroni-logrus v0.0.0-20170801195057-31067281800f/go.mod h1:Ylx55XGW4gjY7McWT0pgqU0aQquIOChDnYkOVbSuF/c= 219 | github.com/meatballhat/negroni-logrus v1.1.0/go.mod h1:1yuzU2YqJx1Fh4UJ2nAt2rBa0rZoLxfpXQL/BXpiU0g= 220 | github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= 221 | github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 222 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 223 | github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= 224 | github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= 225 | github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= 226 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 227 | github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU= 228 | github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= 229 | github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= 230 | github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= 231 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 232 | github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= 233 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 234 | github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 235 | github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= 236 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 237 | github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 238 | github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA= 239 | github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= 240 | github.com/peterhellberg/link v1.0.0/go.mod h1:gtSlOT4jmkY8P47hbTc8PTgiDDWpdPbFYl75keYyBB8= 241 | github.com/peterhellberg/link v1.1.0/go.mod h1:gtSlOT4jmkY8P47hbTc8PTgiDDWpdPbFYl75keYyBB8= 242 | github.com/phyber/negroni-gzip v0.0.0-20180113114010-ef6356a5d029 h1:d6HcSW4ZoNlUWrPyZtBwIu8yv4WAWIU3R/jorwVkFtQ= 243 | github.com/phyber/negroni-gzip v0.0.0-20180113114010-ef6356a5d029/go.mod h1:94RTq2fypdZCze25ZEZSjtbAQRT3cL/8EuRUqAZC/+w= 244 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 245 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 246 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 247 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 248 | github.com/reiver/go-porterstemmer v1.0.1 h1:WyERBkASXgoXrTwq/IQ6wyNj/YG7j/ZURvTuMCoud5w= 249 | github.com/reiver/go-porterstemmer v1.0.1/go.mod h1:Z8uL/f/7UEwaeAJNwx1sO8kbqXiEuQieNuD735hLrSU= 250 | github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA= 251 | github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= 252 | github.com/restic/chunker v0.2.0 h1:GjvmvFuv2mx0iekZs+iAlrioo2UtgsGSSplvoXaVHDU= 253 | github.com/restic/chunker v0.2.0/go.mod h1:VdjruEj+7BU1ZZTW8Qqi1exxRx2Omf2JH0NsUEkQ29s= 254 | github.com/restic/chunker v0.3.0/go.mod h1:VdjruEj+7BU1ZZTW8Qqi1exxRx2Omf2JH0NsUEkQ29s= 255 | github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967 h1:x7xEyJDP7Hv3LVgvWhzioQqbC/KtuUhTigKlH/8ehhE= 256 | github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= 257 | github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= 258 | github.com/rwcarlsen/goexif v0.0.0-20180518182100-8d986c03457a h1:ZDZdsnbMuRSoVbq1gR47o005lfn2OwODNCr23zh9gSk= 259 | github.com/rwcarlsen/goexif v0.0.0-20180518182100-8d986c03457a/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= 260 | github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= 261 | github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 h1:7YvPJVmEeFHR1Tj9sZEYsmarJEQfMVYpd/Vyy/A8dqE= 262 | github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= 263 | github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= 264 | github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= 265 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 266 | github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= 267 | github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= 268 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 269 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 270 | github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= 271 | github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= 272 | github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= 273 | github.com/src-d/go-oniguruma v1.1.0/go.mod h1:chVbff8kcVtmrhxtZ3yBVLLquXbzCS6DrxQaAK/CeqM= 274 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 275 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 276 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 277 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 278 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 279 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 280 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 281 | github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2 h1:GnOzE5fEFN3b2zDhJJABEofdb51uMRNb8eqIVtdducs= 282 | github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= 283 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 284 | github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs= 285 | github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= 286 | github.com/toqueteos/trie v0.0.0-20150530104557-56fed4a05683 h1:ej8ns+4aeQO+mm9VIzwnJElkqR0Vs6kTfIcvgyJFoMY= 287 | github.com/toqueteos/trie v0.0.0-20150530104557-56fed4a05683/go.mod h1:Ywk48QhEqhU1+DwhMkJ2x7eeGxDHiGkAdc9+0DYcbsM= 288 | github.com/toqueteos/trie v1.0.0/go.mod h1:Ywk48QhEqhU1+DwhMkJ2x7eeGxDHiGkAdc9+0DYcbsM= 289 | github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= 290 | github.com/unrolled/secure v0.0.0-20181022170031-4b6b7cf51606/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA= 291 | github.com/unrolled/secure v0.0.0-20181221173256-0d6b5bb13069 h1:RKeYksgIwGE8zFJTvXI1WWx09QPrGyaVFMy0vpU7j/o= 292 | github.com/unrolled/secure v0.0.0-20181221173256-0d6b5bb13069/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA= 293 | github.com/unrolled/secure v1.0.4/go.mod h1:R6rugAuzh4TQpbFAq69oqZggyBQxFRFQIewtz5z7Jsc= 294 | github.com/unrolled/secure v1.0.7/go.mod h1:uGc1OcRF8gCVBA+ANksKmvM85Hka6SZtQIbrKc3sHS4= 295 | github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= 296 | github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= 297 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 298 | github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= 299 | github.com/valyala/gozstd v1.2.1 h1:ZZcVQLO6Ff5I3Ca6OMZFmg5SA9lan3C7kIS84YlRjpY= 300 | github.com/valyala/gozstd v1.2.1/go.mod h1:oYOS+oJovjw9ewtrwEYb9+ybolEXd6pHyLMuAWN5zts= 301 | github.com/vmihailenco/msgpack v4.0.1+incompatible h1:RMF1enSPeKTlXrXdOcqjFUElywVZjjC6pqse21bKbEU= 302 | github.com/vmihailenco/msgpack v4.0.1+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= 303 | github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= 304 | github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= 305 | github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro= 306 | github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8= 307 | github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= 308 | github.com/xeonx/timeago v1.0.0-rc3 h1:GOgz7sE0h0c1ed4J/CMgTiur93tUPsNDpnRrxzMN3Wg= 309 | github.com/xeonx/timeago v1.0.0-rc3/go.mod h1:qDLrYEFynLO7y5Ho7w3GwgtYgpy5UfhcXIIQvMKVDkA= 310 | github.com/xeonx/timeago v1.0.0-rc4/go.mod h1:qDLrYEFynLO7y5Ho7w3GwgtYgpy5UfhcXIIQvMKVDkA= 311 | github.com/yuin/goldmark v1.0.5/go.mod h1:GAOXQunDkMxip+WLt/Bb4n4TEwap/Bit20gguI0UhOE= 312 | github.com/yuin/goldmark v1.1.2/go.mod h1:hDgn8A2EV4OniExoeJs1fSrmEc/T7w8+Teyq8YkThxQ= 313 | github.com/yuin/goldmark v1.1.7/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 314 | github.com/yuin/goldmark v1.1.22/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 315 | github.com/yuin/goldmark v1.1.23/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 316 | github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 317 | github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 318 | github.com/yuin/goldmark-highlighting v0.0.0-20191202084645-78f32c8dd6d5/go.mod h1:4QGn5rJFOASBa2uK4Q2h3BRTyJqRfsAucPFIipSTcaM= 319 | github.com/yuin/goldmark-highlighting v0.0.0-20200218065240-d1af22c1126f/go.mod h1:9yW2CHuRSORvHgw7YfybB09PqUZTbzERyW3QFvd8+0Q= 320 | github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691/go.mod h1:YLF3kDffRfUH/bTxOxHhV6lxwIB3Vfj91rEwNMS9MXo= 321 | github.com/yuin/gopher-lua v0.0.0-20181214045814-db9ae37725ec h1:vpF8Kxql6/3OvGH4y2SKtpN3WsB17mvJ8f8H1o2vucQ= 322 | github.com/yuin/gopher-lua v0.0.0-20181214045814-db9ae37725ec/go.mod h1:fFiAh+CowNFr0NK5VASokuwKwkbacRmHsVA7Yb1Tqac= 323 | github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= 324 | github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= 325 | github.com/zpatrick/rbac v0.0.0-20180829190353-d2c4f050cf28 h1:nLE4b8KyHEEirsOy1Dgqw9esMxqRhwfqlZ6GgM2c8lo= 326 | github.com/zpatrick/rbac v0.0.0-20180829190353-d2c4f050cf28/go.mod h1:WBaExyQHBJO9SelgH0SNqmlwYKV62vfnHCX5lXii91c= 327 | golang.org/dl v0.0.0-20190829154251-82a15e2f2ead/go.mod h1:IUMfjQLJQd4UTqG1Z90tenwKoCX93Gn3MAQJMOSBsDQ= 328 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 329 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= 330 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 331 | golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 332 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 333 | golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 334 | golang.org/x/crypto v0.0.0-20191108234033-bd318be0434a h1:R/qVym5WAxsZWQqZCwDY/8sdVKV1m1WgU4/S5IRQAzc= 335 | golang.org/x/crypto v0.0.0-20191108234033-bd318be0434a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 336 | golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 337 | golang.org/x/crypto v0.0.0-20200214034016-1d94cc7ab1c6/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 338 | golang.org/x/crypto v0.0.0-20200221170553-0f24fbd83dfb/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 339 | golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 340 | golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 341 | golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU= 342 | golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 343 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 344 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 345 | golang.org/x/net v0.0.0-20181213202711-891ebc4b82d6/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 346 | golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 347 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis= 348 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 349 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 350 | golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 351 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 352 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 353 | golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 354 | golang.org/x/net v0.0.0-20191109021931-daa7c04131f5/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 355 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 356 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 357 | golang.org/x/net v0.0.0-20200219183655-46282727080f/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 358 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 359 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 360 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 361 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 362 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= 363 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 364 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 365 | golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 366 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 367 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 368 | golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 369 | golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 370 | golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6 h1:IcgEB62HYgAhX0Nd/QrVgZlxlcyxbGQHElLUhW2X4Fo= 371 | golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 372 | golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 373 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 374 | golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 375 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 376 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 377 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 378 | golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 379 | golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 380 | golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4 h1:Hynbrlo6LbYI3H1IqXpkVDOcX/3HiPdhVEuyj5a59RM= 381 | golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 382 | golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 383 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 384 | golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 385 | golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 386 | golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 387 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 388 | golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 389 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 390 | golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 391 | golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= 392 | golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 393 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 394 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 395 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 396 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 397 | golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= 398 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 399 | google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 400 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 401 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 402 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 403 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 404 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 405 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 406 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 407 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 408 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 409 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 410 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 411 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 412 | gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec h1:RlWgLqCMMIYYEVcAR5MDsuHlVkaIPDAF+5Dehzg8L5A= 413 | gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= 414 | gopkg.in/inconshreveable/log15.v2 v2.0.0-20200109203555-b30bc20e4fd1/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= 415 | gopkg.in/src-d/enry.v1 v1.6.7 h1:9989t5TGSGWvtjzG9kPCQUNrZqj4ZkYnClqO5Yi80eE= 416 | gopkg.in/src-d/enry.v1 v1.6.7/go.mod h1:lDDelHa5/fOO+o8klI8JOOoMszXxhqCYOgqFS2mnxQA= 417 | gopkg.in/src-d/enry.v1 v1.7.3/go.mod h1:lDDelHa5/fOO+o8klI8JOOoMszXxhqCYOgqFS2mnxQA= 418 | gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= 419 | gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek= 420 | gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= 421 | gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= 422 | gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= 423 | gopkg.in/src-d/go-git-fixtures.v3 v3.3.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= 424 | gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= 425 | gopkg.in/src-d/go-git.v4 v4.8.1 h1:aAyBmkdE1QUUEHcP4YFCGKmsMQRAuRmUcPEQR7lOAa0= 426 | gopkg.in/src-d/go-git.v4 v4.8.1/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk= 427 | gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= 428 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 429 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 430 | gopkg.in/toqueteos/substring.v1 v1.0.2 h1:urLqCeMm6x/eTuQa1oZerNw8N1KNOIp5hD5kGL7lFsE= 431 | gopkg.in/toqueteos/substring.v1 v1.0.2/go.mod h1:Eb2Z1UYehlVK8LYW2WBVR2rwbujsz3aX8XDrM1vbNew= 432 | gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= 433 | gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= 434 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 435 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 436 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 437 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 438 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 439 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 440 | mvdan.cc/xurls v1.1.0/go.mod h1:TNWuhvo+IqbUCmtUIb/3LJSQdrzel8loVpgFm0HikbI= 441 | willnorris.com/go/microformats v1.0.0/go.mod h1:AXRtimOA0J5fDmM2sxlka4G6PNLWC4bCNJcZjLvFdDw= 442 | -------------------------------------------------------------------------------- /index.go: -------------------------------------------------------------------------------- 1 | package blobsfile 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "encoding/hex" 7 | "fmt" 8 | "os" 9 | "path/filepath" 10 | "strconv" 11 | 12 | "a4.io/blobstash/pkg/rangedb" 13 | ) 14 | 15 | // FIXME(tsileo): optimize the index with the benchmark (not worth it if inserting the blob take longer) 16 | 17 | // MetaKey and BlobPosKey are used to namespace the DB keys. 18 | const ( 19 | metaKey byte = iota 20 | blobPosKey 21 | ) 22 | 23 | // formatKey prepends the prefix byte to the given key. 24 | func formatKey(prefix byte, bkey []byte) []byte { 25 | res := make([]byte, len(bkey)+1) 26 | res[0] = prefix 27 | copy(res[1:], bkey) 28 | return res 29 | } 30 | 31 | // blobsIndex holds the position of blobs in BlobsFile. 32 | type blobsIndex struct { 33 | db *rangedb.RangeDB 34 | path string 35 | } 36 | 37 | // blobPos is a blob entry in the index. 38 | type blobPos struct { 39 | // bobs-n files 40 | n int 41 | // blobs offset/size in the blobs file 42 | offset int64 43 | size int 44 | blobSize int // the actual blob size (will be different from size if compression is enabled) 45 | } 46 | 47 | // Size returns the blob size (as stored in the BlobsFile). 48 | func (blob *blobPos) Size() int { 49 | return blob.size 50 | } 51 | 52 | // Value serialize a BlobsPos as string. 53 | // (value is encoded as uvarint: n + offset + size + blob size) 54 | func (blob *blobPos) Value() []byte { 55 | bufTmp := make([]byte, 10) 56 | var buf bytes.Buffer 57 | w := binary.PutUvarint(bufTmp[:], uint64(blob.n)) 58 | buf.Write(bufTmp[:w]) 59 | w = binary.PutUvarint(bufTmp[:], uint64(blob.offset)) 60 | buf.Write(bufTmp[:w]) 61 | w = binary.PutUvarint(bufTmp[:], uint64(blob.size)) 62 | buf.Write(bufTmp[:w]) 63 | w = binary.PutUvarint(bufTmp[:], uint64(blob.blobSize)) 64 | buf.Write(bufTmp[:w]) 65 | return buf.Bytes() 66 | } 67 | 68 | func decodeBlobPos(data []byte) (blob *blobPos, error error) { 69 | blob = &blobPos{} 70 | r := bytes.NewBuffer(data) 71 | // read blob.n 72 | ures, err := binary.ReadUvarint(r) 73 | if err != nil { 74 | return blob, err 75 | } 76 | blob.n = int(ures) 77 | 78 | // read blob.offset 79 | ures, err = binary.ReadUvarint(r) 80 | if err != nil { 81 | return blob, err 82 | } 83 | blob.offset = int64(ures) 84 | 85 | // read blob.size 86 | ures, err = binary.ReadUvarint(r) 87 | if err != nil { 88 | return blob, err 89 | } 90 | blob.size = int(ures) 91 | 92 | // read blob.blobSize 93 | ures, err = binary.ReadUvarint(r) 94 | if err != nil { 95 | return blob, err 96 | } 97 | blob.blobSize = int(ures) 98 | 99 | return blob, nil 100 | } 101 | 102 | // newIndex initializes a new index. 103 | func newIndex(path string) (*blobsIndex, error) { 104 | dbPath := filepath.Join(path, "blobs-index") 105 | db, err := rangedb.New(dbPath) 106 | return &blobsIndex{db: db, path: dbPath}, err 107 | } 108 | 109 | func (index *blobsIndex) formatBlobPosKey(key string) []byte { 110 | return formatKey(blobPosKey, []byte(key)) 111 | } 112 | 113 | // Close closes all the open file descriptors. 114 | func (index *blobsIndex) Close() error { 115 | return index.db.Close() 116 | } 117 | 118 | // remove removes the kv file. 119 | func (index *blobsIndex) remove() error { 120 | return os.RemoveAll(index.path) 121 | } 122 | 123 | // setPos creates a new blobPos entry in the index for the given hash. 124 | func (index *blobsIndex) setPos(hexHash string, pos *blobPos) error { 125 | hash, err := hex.DecodeString(hexHash) 126 | if err != nil { 127 | return err 128 | } 129 | return index.db.Set(formatKey(blobPosKey, hash), pos.Value()) 130 | } 131 | 132 | // deletePos deletes the stored blobPos for the given hash. 133 | // func (index *blobsIndex) deletePos(hexHash string) error { 134 | // hash, err := hex.DecodeString(hexHash) 135 | // if err != nil { 136 | // return err 137 | // } 138 | // return index.db.Delete(formatKey(blobPosKey, hash)) 139 | //} 140 | 141 | // checkPos checks if a blobPos exists for the given hash (without decoding it). 142 | func (index *blobsIndex) checkPos(hexHash string) (bool, error) { 143 | hash, err := hex.DecodeString(hexHash) 144 | if err != nil { 145 | return false, err 146 | } 147 | data, err := index.db.Get(formatKey(blobPosKey, hash)) 148 | if err != nil { 149 | return false, fmt.Errorf("error getting BlobPos: %v", err) 150 | } 151 | if data == nil || len(data) == 0 { 152 | return false, nil 153 | } 154 | return true, nil 155 | } 156 | 157 | // getPos retrieve the stored blobPos for the given hash. 158 | func (index *blobsIndex) getPos(hexHash string) (*blobPos, error) { 159 | hash, err := hex.DecodeString(hexHash) 160 | if err != nil { 161 | return nil, err 162 | } 163 | data, err := index.db.Get(formatKey(blobPosKey, hash)) 164 | if err != nil { 165 | return nil, fmt.Errorf("error getting BlobPos: %v", err) 166 | } 167 | if data == nil { 168 | return nil, nil 169 | } 170 | bpos, err := decodeBlobPos(data) 171 | return bpos, err 172 | } 173 | 174 | // setN stores the latest N (blobs-N) to remember the latest BlobsFile opened. 175 | func (index *blobsIndex) setN(n int) error { 176 | return index.db.Set(formatKey(metaKey, []byte("n")), []byte(strconv.Itoa(n))) 177 | } 178 | 179 | // getN retrieves the latest N (blobs-N) stored. 180 | func (index *blobsIndex) getN() (int, error) { 181 | data, err := index.db.Get(formatKey(metaKey, []byte("n"))) 182 | if err != nil || string(data) == "" { 183 | return 0, nil 184 | } 185 | return strconv.Atoi(string(data)) 186 | } 187 | -------------------------------------------------------------------------------- /index_test.go: -------------------------------------------------------------------------------- 1 | package blobsfile 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "sync" 7 | "testing" 8 | 9 | "golang.org/x/crypto/blake2b" 10 | ) 11 | 12 | func TestBlobsIndexBasic(t *testing.T) { 13 | index, err := newIndex("tmp_test_index") 14 | check(err) 15 | defer index.Close() 16 | defer os.RemoveAll("tmp_test_index") 17 | 18 | bp := &blobPos{n: 1, offset: 5, size: 10, blobSize: 10} 19 | h := fmt.Sprintf("%x", blake2b.Sum256([]byte("fakehash"))) 20 | err = index.setPos(h, bp) 21 | check(err) 22 | bp3, err := index.getPos(h) 23 | if bp.n != bp3.n || bp.offset != bp3.offset || bp.size != bp3.size || bp.blobSize != bp3.blobSize { 24 | t.Errorf("index.GetPos error, expected:%q, got:%q", bp, bp3) 25 | } 26 | 27 | err = index.setN(5) 28 | check(err) 29 | n2, err := index.getN() 30 | check(err) 31 | if n2 != 5 { 32 | t.Errorf("Error GetN, got %v, expected 5", n2) 33 | } 34 | err = index.setN(100) 35 | check(err) 36 | n2, err = index.getN() 37 | check(err) 38 | if n2 != 100 { 39 | t.Errorf("Error GetN, got %v, expected 100", n2) 40 | } 41 | } 42 | 43 | func TestBlobsIndex(t *testing.T) { 44 | index, err := newIndex("tmp_test_index") 45 | check(err) 46 | defer index.Close() 47 | defer os.RemoveAll("tmp_test_index") 48 | var wg sync.WaitGroup 49 | var mu sync.Mutex 50 | expected := map[string]*blobPos{} 51 | for i := 0; i < 50000; i++ { 52 | wg.Add(1) 53 | go func(i int) { 54 | defer wg.Done() 55 | data := fmt.Sprintf("fakehash %d", i) 56 | h := fmt.Sprintf("%x", blake2b.Sum256([]byte(data))) 57 | bp := &blobPos{n: 1, offset: 100, size: len(data), blobSize: len(data)} 58 | if err := index.setPos(h, bp); err != nil { 59 | panic(fmt.Errorf("failed to index.setPos a i=%d: %v", i, err)) 60 | } 61 | mu.Lock() 62 | expected[h] = bp 63 | mu.Unlock() 64 | }(i) 65 | } 66 | 67 | wg.Wait() 68 | 69 | for h, ebp := range expected { 70 | bp, err := index.getPos(h) 71 | if err != nil { 72 | panic(fmt.Errorf("failed to index.getPos(\"%s\"): %v", h, err)) 73 | } 74 | if bp.n != ebp.n || bp.offset != ebp.offset || bp.size != ebp.size || bp.blobSize != ebp.blobSize { 75 | t.Errorf("index.getPos error, expected:%q, got:%q", bp, ebp) 76 | } 77 | } 78 | } 79 | 80 | func BenchmarkBlobsIndex(b *testing.B) { 81 | index, err := newIndex("tmp_test_index") 82 | check(err) 83 | defer index.Close() 84 | defer os.RemoveAll("tmp_test_index") 85 | b.ResetTimer() 86 | b.StopTimer() 87 | for i := 0; i < b.N; i++ { 88 | b.StopTimer() 89 | data := fmt.Sprintf("fakehash %d", i) 90 | h := fmt.Sprintf("%x", blake2b.Sum256([]byte(data))) 91 | bp := &blobPos{n: 1, offset: int64(100 + (i * len(data))), size: len(data), blobSize: len(data)} 92 | b.StartTimer() 93 | if err := index.setPos(h, bp); err != nil { 94 | panic(fmt.Errorf("failed to index.setPos a i=%d: %v", i, err)) 95 | } 96 | b.StopTimer() 97 | } 98 | } 99 | --------------------------------------------------------------------------------