3 | //
4 | // This program is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // This program is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU General Public License
15 | // along with this program. If not, see .
16 |
17 | // This package implements a content-addressable file storage that keeps its
18 | // data in RAM.
19 | package ram
20 |
21 | import (
22 | "bytes"
23 | "crypto/sha256"
24 | "fmt"
25 | . "github.com/indyjo/cafs"
26 | "github.com/indyjo/cafs/chunking"
27 | "hash"
28 | "io"
29 | "log"
30 | "sync"
31 | )
32 |
33 | type ramStorage struct {
34 | mutex sync.Mutex
35 | entries map[SKey]*ramEntry
36 | bytesUsed, bytesMax int64
37 | bytesLocked int64
38 | youngest, oldest SKey
39 | }
40 |
41 | type ramFile struct {
42 | storage *ramStorage
43 | key SKey
44 | entry *ramEntry
45 | disposed bool
46 | }
47 |
48 | type chunkRef struct {
49 | key SKey
50 | // Points to the byte position within the file immediately after this chunk
51 | nextPos int64
52 | }
53 |
54 | type ramEntry struct {
55 | // Keys to the next older and next younger entry
56 | younger, older SKey
57 | info string
58 | // Holds data if entry is of simple kind
59 | data []byte
60 | // Holds a list of chunk positions if entry is of chunk list type
61 | chunks []chunkRef
62 | refs int
63 | }
64 |
65 | type ramDataReader struct {
66 | data []byte
67 | index int
68 | }
69 |
70 | type ramChunkReader struct {
71 | storage *ramStorage // Storage to read from
72 | entry *ramEntry // Entry containing the chunks
73 | key SKey // SKey of that entry
74 | chunksTail []chunkRef // Remaining chunks
75 | closed bool // Whether Close() has been called
76 | dataReader io.ReadCloser
77 | }
78 |
79 | type ramTemporary struct {
80 | storage *ramStorage
81 | info string // Info text given by user identifying the current file
82 | buffer bytes.Buffer // Stores bytes since beginning of current chunk
83 | fileHash hash.Hash // hash since the beginning of the file
84 | chunkHash hash.Hash // hash since the beginning of the current chunk
85 | valid bool // If false, something has gone wrong
86 | open bool // Set to false on Close()
87 | chunker chunking.Chunker // Determines chunk boundaries
88 | chunks []chunkRef // Grows every time a chunk boundary is encountered
89 | }
90 |
91 | func NewRamStorage(maxBytes int64) BoundedStorage {
92 | return &ramStorage{
93 | entries: make(map[SKey]*ramEntry),
94 | bytesMax: maxBytes,
95 | }
96 | }
97 |
98 | func (s *ramStorage) GetUsageInfo() UsageInfo {
99 | s.mutex.Lock()
100 | defer s.mutex.Unlock()
101 | return UsageInfo{Used: s.bytesUsed, Capacity: s.bytesMax, Locked: s.bytesLocked}
102 | }
103 |
104 | func (s *ramStorage) FreeCache() int64 {
105 | s.mutex.Lock()
106 | defer s.mutex.Unlock()
107 | oldBytesUsed := s.bytesUsed
108 | s.reserveBytes("FreeCache", s.bytesMax)
109 | return oldBytesUsed - s.bytesUsed
110 | }
111 |
112 | func (s *ramStorage) Get(key *SKey) (File, error) {
113 | s.mutex.Lock()
114 | entry, ok := s.entries[*key]
115 | if ok {
116 | if entry.refs == 0 {
117 | s.removeFromChain(key, entry)
118 | s.bytesLocked += entry.storageSize()
119 | }
120 | entry.refs++
121 | }
122 | s.mutex.Unlock()
123 | if ok {
124 | return &ramFile{s, *key, entry, false}, nil
125 | } else {
126 | return nil, ErrNotFound
127 | }
128 | return nil, nil // never reached
129 | }
130 |
131 | func (s *ramStorage) Create(info string) Temporary {
132 | return &ramTemporary{
133 | storage: s,
134 | info: info,
135 | fileHash: sha256.New(),
136 | chunkHash: sha256.New(),
137 | valid: true,
138 | open: true,
139 | chunker: chunking.New(),
140 | chunks: make([]chunkRef, 0, 16),
141 | }
142 | }
143 |
144 | func (s *ramStorage) DumpStatistics(log Printer) {
145 | s.mutex.Lock()
146 | defer s.mutex.Unlock()
147 |
148 | link := func(k SKey, n int, local bool) string {
149 | zero := SKey{}
150 | if k == zero {
151 | return fmt.Sprintf("%x", k[:n])
152 | } else if local {
153 | return fmt.Sprintf(`%x`, k, k[:n])
154 | } else {
155 | return fmt.Sprintf(`%x`, k, k[:n])
156 | }
157 | }
158 |
159 | log.Printf("CAFS Statistics")
160 | log.Printf("Bytes used: %d, locked: %d, oldest: %x, youngest: %x", s.bytesUsed, s.bytesLocked, s.oldest[:4], s.youngest[:4])
161 | for key, entry := range s.entries {
162 | log.Printf(" [%v] refs=%d size=%v [%v] %v (older) %v (younger)",
163 | key, link(key, 4, false), entry.refs, entry.storageSize(), entry.info,
164 | link(entry.older, 4, true), link(entry.younger, 4, true))
165 |
166 | prevPos := int64(0)
167 | for i, chunk := range entry.chunks {
168 | log.Printf(" chunk %4d: %v (length %6d, ends at %7d)", i,
169 | link(chunk.key, 4, true), chunk.nextPos-prevPos, chunk.nextPos)
170 | prevPos = chunk.nextPos
171 | }
172 | }
173 | log.Printf("")
174 | }
175 |
176 | func (s *ramStorage) reserveBytes(info string, numBytes int64) error {
177 | if numBytes > s.bytesMax {
178 | return ErrNotEnoughSpace
179 | }
180 | bytesFree := s.bytesMax - s.bytesUsed
181 | if bytesFree < numBytes && LoggingEnabled {
182 | log.Printf("[%v] Need to free %v (currently unlocked %v) more bytes of CAFS space to store object of size %v",
183 | info, numBytes-bytesFree, s.bytesUsed-s.bytesLocked, numBytes)
184 | }
185 | for bytesFree < numBytes {
186 | oldestKey := s.oldest
187 | oldestEntry := s.entries[oldestKey]
188 | if oldestEntry == nil {
189 | return ErrNotEnoughSpace
190 | }
191 | s.removeFromChain(&s.oldest, oldestEntry)
192 | delete(s.entries, oldestKey)
193 |
194 | oldLocked := s.bytesLocked
195 | // Dereference all referenced chunks
196 | for _, chunk := range oldestEntry.chunks {
197 | s.release(&chunk.key, s.entries[chunk.key])
198 | }
199 | oldestSize := oldestEntry.storageSize()
200 | s.bytesUsed -= oldestSize
201 | bytesFree += oldestSize
202 | if LoggingEnabled {
203 | log.Printf("[%v] Deleted object of size %v bytes: [%v] %v", info, oldestSize, oldestEntry.info, oldestKey)
204 | if oldLocked != s.bytesLocked {
205 | log.Printf(" -> unlocked %d bytes", oldLocked-s.bytesLocked)
206 | }
207 | }
208 | }
209 | return nil
210 | }
211 |
212 | // Puts an entry into the store. If an entry already exists, it must be identical to the old one.
213 | // The newly-created or recycled entry has been lock'ed once and must be release'd properly.
214 | func (s *ramStorage) storeEntry(key *SKey, data []byte, chunks []chunkRef, info string) error {
215 | if len(data) > 0 && len(chunks) > 0 {
216 | panic("Illegal entry")
217 | }
218 | s.mutex.Lock()
219 | defer s.mutex.Unlock()
220 |
221 | // Detect if we're re-writing the same data (or even handle a hash collision)
222 | var newEntry *ramEntry
223 | if oldEntry := s.entries[*key]; oldEntry != nil {
224 | if len(oldEntry.data) != len(data) || len(oldEntry.chunks) != len(chunks) {
225 | panic(fmt.Sprintf("[%v] Key collision: %v [%v]", info, key, oldEntry.info))
226 | }
227 | if LoggingEnabled {
228 | log.Printf("[%v] Recycling key: %v [%v] (data: %d bytes, chunks: %d)", info, key, oldEntry.info, len(data), len(chunks))
229 | }
230 |
231 | // Ref the reused entry.
232 | s.lock(key, oldEntry)
233 |
234 | // Unref all referenced chunks
235 | for _, chunk := range chunks {
236 | chunkEntry := s.entries[chunk.key]
237 | s.release(&chunk.key, chunkEntry)
238 | }
239 |
240 | // re-use old entry
241 | newEntry = oldEntry
242 | } else {
243 | newEntry = &ramEntry{
244 | info: info,
245 | data: data,
246 | chunks: chunks,
247 | refs: 1,
248 | }
249 | // Reserve the necessary space for storing the object
250 | if err := s.reserveBytes(info, newEntry.storageSize()); err != nil {
251 | return err
252 | }
253 |
254 | s.entries[*key] = newEntry
255 | s.bytesUsed += newEntry.storageSize()
256 | s.bytesLocked += newEntry.storageSize()
257 | if LoggingEnabled {
258 | log.Printf("[%v] Stored key: %v (data: %d bytes, chunks: %d)", info, key, len(data), len(chunks))
259 | }
260 | }
261 |
262 | return nil
263 | }
264 |
265 | func (s *ramStorage) removeFromChain(key *SKey, entry *ramEntry) {
266 | if youngerEntry := s.entries[entry.younger]; youngerEntry != nil {
267 | youngerEntry.older = entry.older
268 | } else if s.youngest == *key {
269 | s.youngest = entry.older
270 | }
271 | if olderEntry := s.entries[entry.older]; olderEntry != nil {
272 | olderEntry.younger = entry.younger
273 | } else if s.oldest == *key {
274 | s.oldest = entry.younger
275 | }
276 | // clear outgoing links
277 | entry.younger, entry.older = SKey{}, SKey{}
278 | }
279 |
280 | func (s *ramStorage) insertIntoChain(key *SKey, entry *ramEntry) {
281 | entry.older = s.youngest
282 | if youngestEntry := s.entries[s.youngest]; youngestEntry != nil {
283 | // chain former youngest entry to new one
284 | youngestEntry.younger = *key
285 | } else {
286 | // empty map, new entry will also be oldest
287 | s.oldest = *key
288 | }
289 | s.youngest = *key
290 | }
291 |
292 | // Mutex lock-protected version of lock()
293 | func (s *ramStorage) lockL(key *SKey, entry *ramEntry) {
294 | s.mutex.Lock()
295 | defer s.mutex.Unlock()
296 | s.lock(key, entry)
297 | }
298 |
299 | func (s *ramStorage) lock(key *SKey, entry *ramEntry) {
300 | if entry.refs == 0 {
301 | s.removeFromChain(key, entry)
302 | s.bytesLocked += entry.storageSize()
303 | }
304 | entry.refs++
305 | }
306 |
307 | // Mutex lock-protected version of release()
308 | func (s *ramStorage) releaseL(key *SKey, entry *ramEntry) {
309 | s.mutex.Lock()
310 | defer s.mutex.Unlock()
311 | s.release(key, entry)
312 | }
313 |
314 | // Dereferences a single entry. Must happen while mutex is held.
315 | func (s *ramStorage) release(key *SKey, entry *ramEntry) {
316 | if entry.refs == 0 {
317 | panic(fmt.Sprintf("Can't release entry %v with 0 references", key))
318 | }
319 | entry.refs--
320 | if entry.refs == 0 {
321 | s.bytesLocked -= entry.storageSize()
322 | s.insertIntoChain(key, entry)
323 | }
324 | }
325 |
326 | // These are only estimates. Even an empty file consumes storage.
327 | const entrySize = 112
328 | const chunkSize = 40
329 |
330 | func (e *ramEntry) storageSize() int64 {
331 | return int64(entrySize + len(e.data) + chunkSize*len(e.chunks))
332 | }
333 |
334 | func (f *ramFile) Key() SKey {
335 | return f.key
336 | }
337 |
338 | func (f *ramFile) Open() io.ReadCloser {
339 | if len(f.entry.chunks) > 0 {
340 | f.storage.lockL(&f.key, f.entry)
341 | return &ramChunkReader{
342 | storage: f.storage,
343 | entry: f.entry,
344 | key: f.key,
345 | chunksTail: f.entry.chunks,
346 | closed: false,
347 | }
348 | } else {
349 | return &ramDataReader{f.entry.data, 0}
350 | }
351 | }
352 |
353 | func (f *ramFile) Size() int64 {
354 | if f.entry.data != nil {
355 | return int64(len(f.entry.data))
356 | } else {
357 | return f.entry.chunks[len(f.entry.chunks)-1].nextPos
358 | }
359 | }
360 |
361 | func (f *ramFile) Dispose() {
362 | if !f.disposed {
363 | f.disposed = true
364 | f.storage.releaseL(&f.key, f.entry)
365 | }
366 | }
367 |
368 | func (f *ramFile) checkValid() {
369 | if f.disposed {
370 | panic("Already disposed")
371 | }
372 | }
373 |
374 | func (f *ramFile) Duplicate() File {
375 | f.checkValid()
376 | file, err := f.storage.Get(&f.key)
377 | if err != nil {
378 | panic("Couldn't duplicate file")
379 | }
380 | return file
381 | }
382 |
383 | func (f *ramFile) IsChunked() bool {
384 | f.checkValid()
385 | return len(f.entry.chunks) > 0
386 | }
387 |
388 | func (f *ramFile) Chunks() FileIterator {
389 | var chunks []chunkRef
390 | if len(f.entry.chunks) > 0 {
391 | chunks = f.entry.chunks
392 | } else {
393 | chunks = make([]chunkRef, 1)
394 | chunks[0] = chunkRef{f.key, f.Size()}
395 | }
396 | f.storage.lockL(&f.key, f.entry)
397 | return &ramChunksIter{
398 | storage: f.storage,
399 | entry: f.entry,
400 | key: f.key,
401 | chunks: chunks,
402 | chunkIdx: 0,
403 | lastChunkIdx: -1,
404 | disposed: false,
405 | }
406 | }
407 |
408 | func (f *ramFile) NumChunks() int64 {
409 | if len(f.entry.chunks) > 0 {
410 | return int64(len(f.entry.chunks))
411 | } else {
412 | return 1
413 | }
414 | }
415 |
416 | func (ci *ramChunksIter) checkValid() {
417 | if ci.disposed {
418 | panic("Already disposed")
419 | }
420 | }
421 |
422 | type ramChunksIter struct {
423 | storage *ramStorage
424 | key SKey
425 | entry *ramEntry
426 | chunks []chunkRef
427 | chunkIdx int
428 | lastChunkIdx int
429 | disposed bool
430 | }
431 |
432 | func (ci *ramChunksIter) Dispose() {
433 | if !ci.disposed {
434 | ci.disposed = true
435 | ci.storage.releaseL(&ci.key, ci.entry)
436 | }
437 | }
438 |
439 | func (ci *ramChunksIter) Duplicate() FileIterator {
440 | ci.checkValid()
441 | ci.storage.lockL(&ci.key, ci.entry)
442 | return &ramChunksIter{
443 | storage: ci.storage,
444 | key: ci.key,
445 | entry: ci.entry,
446 | chunks: ci.chunks,
447 | chunkIdx: ci.chunkIdx,
448 | disposed: false,
449 | }
450 | }
451 |
452 | func (ci *ramChunksIter) Next() bool {
453 | ci.checkValid()
454 | if ci.chunkIdx == len(ci.chunks) {
455 | ci.Dispose()
456 | return false
457 | } else {
458 | ci.lastChunkIdx = ci.chunkIdx
459 | ci.chunkIdx++
460 | return true
461 | }
462 | }
463 |
464 | func (ci *ramChunksIter) Key() SKey {
465 | ci.checkValid()
466 | return ci.chunks[ci.lastChunkIdx].key
467 | }
468 |
469 | func (ci *ramChunksIter) Size() int64 {
470 | ci.checkValid()
471 | startPos := int64(0)
472 | if ci.lastChunkIdx > 0 {
473 | startPos = ci.chunks[ci.lastChunkIdx-1].nextPos
474 | }
475 | return ci.chunks[ci.lastChunkIdx].nextPos - startPos
476 | }
477 |
478 | func (ci *ramChunksIter) File() File {
479 | ci.checkValid()
480 | if f, err := ci.storage.Get(&ci.chunks[ci.lastChunkIdx].key); err != nil {
481 | panic(err)
482 | } else {
483 | return f
484 | }
485 | }
486 |
487 | func (r *ramDataReader) Read(b []byte) (n int, err error) {
488 | if len(b) == 0 {
489 | return 0, nil
490 | }
491 | if r.index >= len(r.data) {
492 | return 0, io.EOF
493 | }
494 | n = copy(b, r.data[r.index:])
495 | r.index += n
496 | return
497 | }
498 |
499 | func (r *ramDataReader) Close() error {
500 | return nil
501 | }
502 |
503 | func (r *ramChunkReader) Read(b []byte) (n int, err error) {
504 | if r.closed {
505 | err = ErrInvalidState
506 | return
507 | }
508 | for n == 0 && err == nil {
509 | if r.dataReader == nil {
510 | if len(r.chunksTail) > 0 {
511 | if f, e := r.storage.Get(&r.chunksTail[0].key); e != nil {
512 | panic(e)
513 | } else {
514 | defer f.Dispose()
515 | r.dataReader = f.Open()
516 | r.chunksTail = r.chunksTail[1:]
517 | }
518 | } else {
519 | return 0, io.EOF
520 | }
521 | }
522 |
523 | n, err = r.dataReader.Read(b)
524 | if err == io.EOF {
525 | // never pass through delegate EOF
526 | err = r.dataReader.Close()
527 | r.dataReader = nil
528 | }
529 | }
530 | return
531 | }
532 |
533 | func (r *ramChunkReader) Close() (err error) {
534 | if r.closed {
535 | return nil
536 | }
537 | r.closed = true
538 |
539 | r.storage.releaseL(&r.key, r.entry)
540 |
541 | if r.dataReader != nil {
542 | err = r.dataReader.Close()
543 | r.dataReader = nil
544 | }
545 |
546 | return
547 | }
548 |
549 | // Writes the current buffer into a new chunk and resets the buffer.
550 | // Assumes that chunkHash has already been updated.
551 | func (t *ramTemporary) flushBufferIntoChunk() error {
552 | if t.buffer.Len() == 0 {
553 | return nil
554 | }
555 |
556 | // Copy the chunk's data
557 | chunkInfo := fmt.Sprintf("%v #%d", t.info, len(t.chunks))
558 | chunkData := make([]byte, t.buffer.Len())
559 | copy(chunkData, t.buffer.Bytes())
560 |
561 | // Get the chunk hash
562 | var key SKey
563 | t.chunkHash.Sum(key[:0])
564 | t.chunkHash.Reset()
565 |
566 | if err := t.storage.storeEntry(&key, chunkData, nil, chunkInfo); err != nil {
567 | return err
568 | }
569 |
570 | chunk := chunkRef{
571 | key: key,
572 | nextPos: int64(t.buffer.Len()),
573 | }
574 | if len(t.chunks) > 0 {
575 | chunk.nextPos += t.chunks[len(t.chunks)-1].nextPos
576 | }
577 | t.chunks = append(t.chunks, chunk)
578 |
579 | t.buffer.Reset()
580 | return nil
581 | }
582 |
583 | func (t *ramTemporary) Write(b []byte) (int, error) {
584 | if !t.valid || !t.open {
585 | return 0, ErrInvalidState
586 | }
587 | t.valid = false // only temporary -> set to true on successful end of function
588 |
589 | nBytes := len(b)
590 |
591 | for len(b) > 0 {
592 | nBoundary := t.chunker.Scan(b)
593 | if _, err := t.buffer.Write(b[:nBoundary]); err != nil {
594 | return 0, err
595 | }
596 | t.chunkHash.Write(b[:nBoundary])
597 | t.fileHash.Write(b[:nBoundary])
598 | if nBoundary < len(b) {
599 | // a chunk boundary was detected
600 | if err := t.flushBufferIntoChunk(); err != nil {
601 | return 0, err
602 | }
603 | b = b[nBoundary:]
604 | } else {
605 | b = nil
606 | }
607 | }
608 |
609 | t.valid = true
610 | return nBytes, nil
611 | }
612 |
613 | func (t *ramTemporary) Close() error {
614 | if !t.valid || !t.open {
615 | return ErrInvalidState
616 | }
617 | t.open = false
618 | t.valid = false // only temporary -> set to true on successful end of function
619 | var key SKey
620 | t.fileHash.Sum(key[:0])
621 |
622 | if len(t.chunks) == 0 {
623 | // File is single-chunk
624 | data := make([]byte, t.buffer.Len())
625 | copy(data, t.buffer.Bytes())
626 | if err := t.storage.storeEntry(&key, data, nil, t.info); err != nil {
627 | return err
628 | }
629 | } else {
630 | // Flush buffer contents into one last chunk
631 | if err := t.flushBufferIntoChunk(); err != nil {
632 | return err
633 | }
634 | finalChunks := make([]chunkRef, len(t.chunks))
635 | copy(finalChunks, t.chunks)
636 | if err := t.storage.storeEntry(&key, nil, finalChunks, t.info); err != nil {
637 | return err
638 | }
639 | }
640 | t.valid = true
641 | return nil
642 | }
643 |
644 | func (t *ramTemporary) File() File {
645 | if !t.valid {
646 | panic(ErrInvalidState)
647 | }
648 | if t.open {
649 | panic(ErrStillOpen)
650 | }
651 |
652 | var key SKey
653 | t.fileHash.Sum(key[:0])
654 |
655 | file, err := t.storage.Get(&key)
656 | if err != nil {
657 | // Shouldn't happen
658 | panic(err)
659 | }
660 | return file
661 | }
662 |
663 | func (t *ramTemporary) Dispose() {
664 | if t.chunks == nil {
665 | // temporary was already disposed, we allow this
666 | return
667 | }
668 |
669 | t.releaseFromStorage()
670 |
671 | t.valid = false
672 | wasOpen := t.open
673 | t.open = false
674 | t.buffer = bytes.Buffer{}
675 | t.chunker = nil
676 | t.chunks = nil
677 | if LoggingEnabled {
678 | if wasOpen {
679 | log.Printf("[%v] Temporary canceled", t.info)
680 | } else {
681 | log.Printf("[%v] Temporary disposed", t.info)
682 | }
683 | }
684 | }
685 |
686 | // Calls release() on all chunks locked by this temporary.
687 | func (t *ramTemporary) releaseFromStorage() {
688 | t.storage.mutex.Lock()
689 | defer t.storage.mutex.Unlock()
690 |
691 | // dereference single-chunk entry if successfully closed
692 | if !t.open && t.valid {
693 | var key SKey
694 | t.fileHash.Sum(key[:0])
695 | t.storage.release(&key, t.storage.entries[key])
696 | } else {
697 | // dereference all locked chunks otherwise
698 | // (they have been locked once just by storing them)
699 | for _, chunk := range t.chunks {
700 | t.storage.release(&chunk.key, t.storage.entries[chunk.key])
701 | }
702 | }
703 | }
704 |
--------------------------------------------------------------------------------
/ram/ramstorage_test.go:
--------------------------------------------------------------------------------
1 | package ram
2 |
3 | import (
4 | "fmt"
5 | . "github.com/indyjo/cafs"
6 | "io"
7 | "math/rand"
8 | "testing"
9 | )
10 |
11 | func TestSimple(t *testing.T) {
12 | s := NewRamStorage(1000)
13 | _ = addData(t, s, 128)
14 | }
15 |
16 | func TestTwo(t *testing.T) {
17 | s := NewRamStorage(1000)
18 | f1 := addData(t, s, 128)
19 | f2 := addData(t, s, 256)
20 | if f1.Key() == f2.Key() {
21 | t.FailNow()
22 | }
23 | }
24 |
25 | func TestSame(t *testing.T) {
26 | s := NewRamStorage(1000)
27 | f1 := addData(t, s, 128)
28 | f2 := addData(t, s, 128)
29 | if f1.Key() != f2.Key() {
30 | t.FailNow()
31 | }
32 | }
33 |
34 | func TestEmptyFile(t *testing.T) {
35 | s := NewRamStorage(1000)
36 | f := addData(t, s, 0)
37 | if f.Size() != 0 {
38 | t.FailNow()
39 | }
40 | iter := f.Chunks()
41 | if !iter.Next() {
42 | t.Fatal("Expected empty file to have at least one chunk")
43 | }
44 | if iter.Key().String() != "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" {
45 | t.Fatalf("Unexpected key of empty chunk: %v", iter.Key())
46 | }
47 | if iter.Next() {
48 | t.Fatal("Expected empty file to not have any further chunks")
49 | }
50 | }
51 |
52 | type logPrinter struct {
53 | }
54 |
55 | func (p logPrinter) Printf(format string, v ...interface{}) {
56 | fmt.Printf(format+"\n", v...)
57 | }
58 |
59 | func TestLRU(t *testing.T) {
60 | s := NewRamStorage(1000)
61 | f1 := addData(t, s, 400)
62 | f1.Dispose()
63 | //s.DumpStatistics(logPrinter{})
64 | f2 := addData(t, s, 350)
65 | f2.Dispose()
66 | //s.DumpStatistics(logPrinter{})
67 | f3 := addData(t, s, 250)
68 | f3.Dispose()
69 | //s.DumpStatistics(logPrinter{})
70 | f4 := addData(t, s, 450)
71 | f4.Dispose()
72 | //s.DumpStatistics(logPrinter{})
73 | var key SKey
74 | key = f1.Key()
75 | if _, err := s.Get(&key); err != ErrNotFound {
76 | t.Fatalf("f1 should have been removed. err:%v", err)
77 | }
78 | key = f2.Key()
79 | if _, err := s.Get(&key); err != ErrNotFound {
80 | t.Fatalf("f2 should have been removed. err:%v", err)
81 | }
82 | key = f4.Key()
83 | if f, err := s.Get(&key); err != nil {
84 | t.Fatalf("f4 should be stored. err:%v", err)
85 | } else {
86 | f.Dispose()
87 | }
88 | key = f3.Key()
89 | if f, err := s.Get(&key); err != nil {
90 | t.Fatalf("f3 should not have been removed. err:%v", err)
91 | } else {
92 | f.Dispose()
93 | }
94 |
95 | //s.DumpStatistics(logPrinter{})
96 |
97 | // Now f3 is youngest, then f4 (f1 and f2 are gone)
98 | addData(t, s, 500).Dispose()
99 |
100 | key = f4.Key()
101 | if _, err := s.Get(&key); err != ErrNotFound {
102 | t.Fatalf("f4 should have been removed. err:%v", err)
103 | }
104 | key = f3.Key()
105 | if _, err := s.Get(&key); err != nil {
106 | t.Fatalf("f3 should be stored. err:%v", err)
107 | }
108 |
109 | {
110 | defer func() {
111 | if v := recover(); v == ErrNotEnoughSpace {
112 | t.Logf("Expectedly recovered from: %v", v)
113 | } else {
114 | t.Fatalf("Expected to recover from something other than: %v", v)
115 | }
116 | }()
117 | addData(t, s, 1010)
118 | }
119 | }
120 |
121 | func TestCompression(t *testing.T) {
122 | s := NewRamStorage(1000000)
123 | f1 := addData(t, s, 1000001)
124 | defer f1.Dispose()
125 | iter := f1.Chunks()
126 | defer iter.Dispose()
127 | t.Log("Iterating over chunks...")
128 | for iter.Next() {
129 | t.Logf("Chunk: Key %v, size %v", iter.Key(), iter.Size())
130 | }
131 | }
132 |
133 | func TestCompression2(t *testing.T) {
134 | s := NewRamStorage(1000000)
135 | temp := s.Create("Adding cyclic random data")
136 | defer temp.Dispose()
137 | cycle := 65536
138 | times := 24
139 | r := rand.New(rand.NewSource(0))
140 | data := make([]byte, cycle)
141 | for i := 0; i < cycle; i++ {
142 | data[i] = byte(r.Int())
143 | }
144 | t.Logf("data=%016x...", data[:8])
145 | for i := 0; i < times; i++ {
146 | if _, err := temp.Write(data); err != nil {
147 | t.Errorf("Error on Write: %v", err)
148 | }
149 | }
150 | if err := temp.Close(); err != nil {
151 | t.Errorf("Error on Close: %v", err)
152 | }
153 |
154 | f := temp.File()
155 | defer f.Dispose()
156 | w := f.Open()
157 | data2 := make([]byte, 1)
158 | for i := 0; i < times*cycle; i++ {
159 | if n, err := io.ReadFull(w, data2); err != nil || n != 1 {
160 | t.Fatalf("Error on Read: %v (n=%d)", err, n)
161 | }
162 | if data2[0] != data[i%cycle] {
163 | t.Fatalf("Data read != data written on byte %d: %02x != %02x", i, data2[0], data[i%cycle])
164 | }
165 | }
166 | }
167 |
168 | func TestRefCounting(t *testing.T) {
169 | _s := NewRamStorage(80 * 1024)
170 | //s := _s.(*ramStorage)
171 | _f := addRandomData(t, _s, 60*1024)
172 | f := _f.(*ramFile)
173 | //defer s.DumpStatistics(logPrinter{})
174 | if f.entry.refs != 1 {
175 | t.Fatalf("Refs != 1 before dispose: %v", f.entry.refs)
176 | }
177 | _f.Dispose()
178 | if f.entry.refs != 0 {
179 | t.Fatalf("Refs != 0 after dispose: %v", f.entry.refs)
180 | }
181 | // This has to push out many chunks of first file
182 | addRandomData(t, _s, 70*1024)
183 | }
184 |
185 | func addData(t *testing.T, s FileStorage, size int) File {
186 | temp := s.Create(fmt.Sprintf("Adding %v bytes object", size))
187 | defer temp.Dispose()
188 | for size > 0 {
189 | if _, err := temp.Write([]byte{byte(size)}); err != nil {
190 | panic(err)
191 | }
192 | size--
193 | }
194 | if err := temp.Close(); err != nil {
195 | panic(err)
196 | }
197 | return temp.File()
198 | }
199 |
200 | func addRandomData(t *testing.T, s FileStorage, size int) File {
201 | temp := s.Create(fmt.Sprintf("%v random bytes", size))
202 | defer temp.Dispose()
203 | buf := make([]byte, size)
204 | for i, _ := range buf {
205 | buf[i] = byte(rand.Int())
206 | }
207 | if _, err := temp.Write(buf); err != nil {
208 | panic(err)
209 | }
210 | if err := temp.Close(); err != nil {
211 | panic(err)
212 | }
213 | return temp.File()
214 | }
215 |
--------------------------------------------------------------------------------
/remotesync/httpsync/cmd/synctest/synctest.go:
--------------------------------------------------------------------------------
1 | // BitWrk - A Bitcoin-friendly, anonymous marketplace for computing power
2 | // Copyright (C) 2013-2019 Jonas Eschenburg
3 | //
4 | // This program is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // This program is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU General Public License
15 | // along with this program. If not, see .package main
16 |
17 | package main
18 |
19 | import (
20 | "context"
21 | "flag"
22 | "fmt"
23 | "github.com/indyjo/cafs"
24 | "github.com/indyjo/cafs/ram"
25 | "github.com/indyjo/cafs/remotesync"
26 | "github.com/indyjo/cafs/remotesync/httpsync"
27 | "io"
28 | "log"
29 | "math/rand"
30 | "net/http"
31 | "os"
32 | "runtime/pprof"
33 | )
34 |
35 | var storage cafs.FileStorage = ram.NewRamStorage(1 << 30)
36 | var fileHandlers = make(map[string]*httpsync.FileHandler)
37 |
38 | func main() {
39 | addr := ":8080"
40 | flag.StringVar(&addr, "l", addr, "which port to listen to")
41 |
42 | preload := ""
43 | flag.StringVar(&preload, "i", preload, "input file to load")
44 |
45 | flag.BoolVar(&remotesync.LoggingEnabled, "enable-remotesync-logging", remotesync.LoggingEnabled,
46 | "enables detailed logging from the remotesync algorithm")
47 |
48 | flag.Parse()
49 |
50 | if preload != "" {
51 | if err := loadFile(storage, preload); err != nil {
52 | log.Fatalf("Error loading '[%v]: %v", preload, err)
53 | }
54 | }
55 |
56 | http.HandleFunc("/load", handleLoad)
57 | http.HandleFunc("/sync", handleSyncFrom)
58 | http.HandleFunc("/stackdump", func(w http.ResponseWriter, r *http.Request) {
59 | name := r.FormValue("name")
60 | if len(name) == 0 {
61 | name = "goroutine"
62 | }
63 | profile := pprof.Lookup(name)
64 | if profile == nil {
65 | _, _ = w.Write([]byte("No such profile"))
66 | return
67 | }
68 | err := profile.WriteTo(w, 1)
69 | if err != nil {
70 | log.Printf("Error in profile.WriteTo: %v\n", err)
71 | }
72 | })
73 |
74 | err := http.ListenAndServe(addr, nil)
75 | if err != nil {
76 | log.Fatalf("Error in ListenAndServe: %v", err)
77 | }
78 | }
79 |
80 | func loadFile(storage cafs.FileStorage, path string) (err error) {
81 | f, err := os.Open(path)
82 | if err != nil {
83 | return
84 | }
85 |
86 | tmp := storage.Create(path)
87 | defer tmp.Dispose()
88 | n, err := io.Copy(tmp, f)
89 | if err != nil {
90 | return fmt.Errorf("error after copying %v bytes: %v", n, err)
91 | }
92 |
93 | err = tmp.Close()
94 | if err != nil {
95 | return
96 | }
97 |
98 | file := tmp.File()
99 | defer file.Dispose()
100 | log.Printf("Read file: %v (%v bytes, chunked: %v, %v chunks)", path, n, file.IsChunked(), file.NumChunks())
101 |
102 | printer := log.New(os.Stderr, "", log.LstdFlags)
103 | handler := httpsync.NewFileHandlerFromFile(file, rand.Perm(256)).WithPrinter(printer)
104 | fileHandlers[file.Key().String()] = handler
105 |
106 | path = fmt.Sprintf("/file/%v", file.Key().String()[:16])
107 | http.Handle(path, handler)
108 | log.Printf(" serving under %v", path)
109 | return
110 | }
111 |
112 | func handleLoad(w http.ResponseWriter, r *http.Request) {
113 | if r.Method != "POST" {
114 | http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
115 | return
116 | }
117 | path := r.FormValue("path")
118 | if err := loadFile(storage, path); err != nil {
119 | http.Error(w, err.Error(), http.StatusInternalServerError)
120 | }
121 | }
122 |
123 | func handleSyncFrom(w http.ResponseWriter, r *http.Request) {
124 | if r.Method != "POST" {
125 | http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
126 | return
127 | }
128 | source := r.FormValue("source")
129 | if err := syncFile(storage, source); err != nil {
130 | http.Error(w, err.Error(), http.StatusInternalServerError)
131 | }
132 | }
133 |
134 | func syncFile(fileStorage cafs.FileStorage, source string) error {
135 | log.Printf("Sync from %v", source)
136 | if file, err := httpsync.SyncFrom(context.Background(), fileStorage, http.DefaultClient, source, "synced from "+source); err != nil {
137 | return err
138 | } else {
139 | log.Printf("Successfully received %v (%v bytes)", file.Key(), file.Size())
140 | file.Dispose()
141 | }
142 | return nil
143 | }
144 |
--------------------------------------------------------------------------------
/remotesync/httpsync/httpsync.go:
--------------------------------------------------------------------------------
1 | // BitWrk - A Bitcoin-friendly, anonymous marketplace for computing power
2 | // Copyright (C) 2013-2019 Jonas Eschenburg
3 | //
4 | // This program is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // This program is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU General Public License
15 | // along with this program. If not, see .
16 |
17 | // Package httpsync implements methods for requesting and serving files via CAFS
18 | package httpsync
19 |
20 | import (
21 | "bufio"
22 | "context"
23 | "encoding/json"
24 | "fmt"
25 | "github.com/indyjo/cafs"
26 | "github.com/indyjo/cafs/remotesync"
27 | "github.com/indyjo/cafs/remotesync/shuffle"
28 | "io"
29 | "io/ioutil"
30 | "net/http"
31 | "sync"
32 | "time"
33 | )
34 |
35 | // Struct FileHandler implements the http.Handler interface and serves a file over HTTP.
36 | // The protocol used matches with function SyncFrom.
37 | // Create using the New... functions.
38 | type FileHandler struct {
39 | m sync.Mutex
40 | source chunksSource
41 | syncinfo *remotesync.SyncInfo
42 | log cafs.Printer
43 | }
44 |
45 | // It is the owner's responsibility to correctly dispose of FileHandler instances.
46 | func (handler *FileHandler) Dispose() {
47 | handler.m.Lock()
48 | s := handler.source
49 | handler.source = nil
50 | handler.syncinfo = nil
51 | handler.m.Unlock()
52 | if s != nil {
53 | s.Dispose()
54 | }
55 | }
56 |
57 | // Function NewFileHandlerFromFile creates a FileHandler that serves chunks of a File.
58 | func NewFileHandlerFromFile(file cafs.File, perm shuffle.Permutation) *FileHandler {
59 | result := &FileHandler{
60 | m: sync.Mutex{},
61 | source: fileBasedChunksSource{file: file.Duplicate()},
62 | syncinfo: &remotesync.SyncInfo{Perm: perm},
63 | log: cafs.NewWriterPrinter(ioutil.Discard),
64 | }
65 | result.syncinfo.SetChunksFromFile(file)
66 | return result
67 | }
68 |
69 | // Function NewFileHandlerFromSyncInfo creates a FileHandler that serves chunks as
70 | // specified in a FileInfo. It doesn't necessarily require all of the chunks to be present
71 | // and will block waiting for a missing chunk to become available.
72 | // As a specialty, a FileHander created using this function needs not be disposed.
73 | func NewFileHandlerFromSyncInfo(syncinfo *remotesync.SyncInfo, storage cafs.FileStorage) *FileHandler {
74 | result := &FileHandler{
75 | m: sync.Mutex{},
76 | source: syncInfoChunksSource{
77 | syncinfo: syncinfo,
78 | storage: storage,
79 | },
80 | syncinfo: syncinfo,
81 | log: cafs.NewWriterPrinter(ioutil.Discard),
82 | }
83 | return result
84 | }
85 |
86 | // Sets the FileHandler's log Printer.
87 | func (handler *FileHandler) WithPrinter(printer cafs.Printer) *FileHandler {
88 | handler.log = printer
89 | return handler
90 | }
91 |
92 | func (handler *FileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
93 | if r.Method == http.MethodGet {
94 | if err := json.NewEncoder(w).Encode(handler.syncinfo); err != nil {
95 | handler.log.Printf("Error serving SyncInfo: R%v", err)
96 | }
97 | return
98 | } else if r.Method != http.MethodPost {
99 | http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
100 | return
101 | }
102 |
103 | // Require a Connection: close header that will trick Go's HTTP server into allowing bi-directional streams.
104 | if r.Header.Get("Connection") != "close" {
105 | http.Error(w, "Connection: close required", http.StatusBadRequest)
106 | return
107 | }
108 |
109 | chunks, err := handler.source.GetChunks()
110 | if err != nil {
111 | handler.log.Printf("GetChunks() failed: %v", err)
112 | http.Error(w, err.Error(), http.StatusInternalServerError)
113 | return
114 | }
115 | defer chunks.Dispose()
116 |
117 | w.WriteHeader(http.StatusOK)
118 | w.(http.Flusher).Flush()
119 |
120 | var bytesSkipped, bytesTransferred int64
121 | cb := func(toTransfer, transferred int64) {
122 | bytesSkipped = -toTransfer
123 | bytesTransferred = transferred
124 | }
125 | handler.log.Printf("Calling WriteChunkData")
126 | start := time.Now()
127 | err = remotesync.WriteChunkData(chunks, 0, bufio.NewReader(r.Body), handler.syncinfo.Perm,
128 | remotesync.SimpleFlushWriter{w, w.(http.Flusher)}, cb)
129 | duration := time.Since(start)
130 | speed := float64(bytesTransferred) / duration.Seconds()
131 | handler.log.Printf("WriteChunkData took %v. KBytes transferred: %v (%.2f/s) skipped: %v",
132 | duration, bytesTransferred>>10, speed/1024, bytesSkipped>>10)
133 | if err != nil {
134 | handler.log.Printf("Error in WriteChunkData: %v", err)
135 | return
136 | }
137 | }
138 |
139 | // Function SyncFrom uses an HTTP client to connect to some URL and download a fie into the
140 | // given FileStorage.
141 | func SyncFrom(ctx context.Context, storage cafs.FileStorage, client *http.Client, url, info string) (file cafs.File, err error) {
142 | // Fetch SyncInfo from remote
143 | resp, err := client.Get(url)
144 | if err != nil {
145 | return
146 | }
147 | if resp.StatusCode != http.StatusOK {
148 | return nil, fmt.Errorf("GET returned status %v", resp.Status)
149 | }
150 | var syncinfo remotesync.SyncInfo
151 | err = json.NewDecoder(resp.Body).Decode(&syncinfo)
152 | if err != nil {
153 | return
154 | }
155 |
156 | // Create Builder and establish a bidirectional POST connection
157 | builder := remotesync.NewBuilder(storage, &syncinfo, 32, info)
158 | defer builder.Dispose()
159 |
160 | pr, pw := io.Pipe()
161 | req, err := http.NewRequest(http.MethodPost, url, pr)
162 | if err != nil {
163 | return
164 | }
165 |
166 | // Enable cancelation
167 | req = req.WithContext(ctx)
168 |
169 | // Trick Go's HTTP server implementation into allowing bi-directional data flow
170 | req.Header.Set("Connection", "close")
171 |
172 | go func() {
173 | if err := builder.WriteWishList(remotesync.NopFlushWriter{pw}); err != nil {
174 | _ = pw.CloseWithError(fmt.Errorf("error in WriteWishList: %v", err))
175 | return
176 | }
177 | _ = pw.Close()
178 | }()
179 |
180 | res, err := client.Do(req)
181 | if err != nil {
182 | return
183 | }
184 | file, err = builder.ReconstructFileFromRequestedChunks(res.Body)
185 | return
186 | }
187 |
--------------------------------------------------------------------------------
/remotesync/httpsync/util.go:
--------------------------------------------------------------------------------
1 | // BitWrk - A Bitcoin-friendly, anonymous marketplace for computing power
2 | // Copyright (C) 2013-2019 Jonas Eschenburg
3 | //
4 | // This program is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // This program is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU General Public License
15 | // along with this program. If not, see .package main
16 |
17 | package httpsync
18 |
19 | import (
20 | "github.com/indyjo/cafs"
21 | "github.com/indyjo/cafs/remotesync"
22 | "io"
23 | "sync"
24 | "time"
25 | )
26 |
27 | // Interface chunksSource specifies a factory for Chunks
28 | type chunksSource interface {
29 | GetChunks() (remotesync.Chunks, error)
30 | Dispose()
31 | }
32 |
33 | // struct fileBasedChunksSource implements ChunksSource using a File.
34 | type fileBasedChunksSource struct {
35 | m sync.Mutex
36 | file cafs.File
37 | }
38 |
39 | func (f fileBasedChunksSource) GetChunks() (remotesync.Chunks, error) {
40 | f.m.Lock()
41 | file := f.file
42 | f.m.Unlock()
43 | if file == nil {
44 | return nil, remotesync.ErrDisposed
45 | }
46 | return remotesync.ChunksOfFile(file), nil
47 | }
48 |
49 | func (f fileBasedChunksSource) Dispose() {
50 | f.m.Lock()
51 | file := f.file
52 | f.file = nil
53 | f.m.Unlock()
54 | if file != nil {
55 | file.Dispose()
56 | }
57 | }
58 |
59 | // Struct syncInfoChunksSource implements ChunksSource using only a SyncInfo object.
60 | // It requests chunks from a FileStore and waits until that chunk becomes available.
61 | // There is no guarantee that chunks are kept or will actually become available at some
62 | // time.
63 | type syncInfoChunksSource struct {
64 | syncinfo *remotesync.SyncInfo
65 | storage cafs.FileStorage
66 | }
67 |
68 | func (s syncInfoChunksSource) GetChunks() (remotesync.Chunks, error) {
69 | return &syncInfoChunks{
70 | chunks: s.syncinfo.Chunks,
71 | storage: s.storage,
72 | done: make(chan struct{}),
73 | }, nil
74 | }
75 |
76 | func (s syncInfoChunksSource) Dispose() {
77 | }
78 |
79 | // Struct syncInfoChunks implements the Chunks interface and does the actual waiting.
80 | type syncInfoChunks struct {
81 | chunks []remotesync.ChunkInfo
82 | storage cafs.FileStorage
83 | done chan struct{}
84 | }
85 |
86 | func (s *syncInfoChunks) NextChunk() (cafs.File, error) {
87 | if len(s.chunks) == 0 {
88 | return nil, io.EOF
89 | }
90 | key := s.chunks[0].Key
91 | s.chunks = s.chunks[1:]
92 | ticker := time.NewTicker(100 * time.Millisecond)
93 | defer func() {
94 | ticker.Stop()
95 | }()
96 | for {
97 | if f, err := s.storage.Get(&key); err == nil {
98 | return f, nil
99 | } else if err != cafs.ErrNotFound {
100 | return nil, err
101 | }
102 |
103 | select {
104 | case <-s.done:
105 | return nil, remotesync.ErrDisposed
106 | case <-ticker.C:
107 | // next try
108 | }
109 | }
110 | }
111 |
112 | func (s *syncInfoChunks) Dispose() {
113 | close(s.done)
114 | }
115 |
--------------------------------------------------------------------------------
/remotesync/receive.go:
--------------------------------------------------------------------------------
1 | // BitWrk - A Bitcoin-friendly, anonymous marketplace for computing power
2 | // Copyright (C) 2013-2019 Jonas Eschenburg
3 | //
4 | // This program is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // This program is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU General Public License
15 | // along with this program. If not, see .
16 |
17 | package remotesync
18 |
19 | import (
20 | "bufio"
21 | "errors"
22 | "fmt"
23 | "github.com/indyjo/cafs"
24 | "github.com/indyjo/cafs/remotesync/shuffle"
25 | "io"
26 | "log"
27 | "sync"
28 | )
29 |
30 | var ErrDisposed = errors.New("disposed")
31 | var ErrUnexpectedChunk = errors.New("unexpected chunk")
32 |
33 | // Used by receiver to memorize information about a chunk in the time window between
34 | // putting it into the wishlist and receiving the actual chunk data.
35 | type memo struct {
36 | ci ChunkInfo // key and length
37 | file cafs.File // A File if the chunk existed already, nil otherwise
38 | requested bool // Whether the chunk was requested from the sender
39 | }
40 |
41 | // Type Builder contains state needed for the duration of a file transmission.
42 | type Builder struct {
43 | done chan struct{}
44 | storage cafs.FileStorage
45 | memos chan memo
46 | info string
47 | syncinf *SyncInfo
48 |
49 | mutex sync.Mutex // Guards subsequent variables
50 | disposed bool // Set in Dispose
51 | started bool // Set in WriteWishList. Signals that chunks channel will be used.
52 | }
53 |
54 | // Returns a new Builder for reconstructing a file. Must eventually be disposed.
55 | // The builder can then proceed sending a "wishlist" of chunks that are missing
56 | // in the local storage for complete reconstruction of the file.
57 | func NewBuilder(storage cafs.FileStorage, syncinf *SyncInfo, windowSize int, info string) *Builder {
58 | return &Builder{
59 | done: make(chan struct{}),
60 | storage: storage,
61 | memos: make(chan memo, windowSize),
62 | info: info,
63 | syncinf: syncinf,
64 | }
65 | }
66 |
67 | // Disposes the Builder. Must be called exactly once per Builder. May cause the goroutines running
68 | // WriteWishList and ReconstructFileFromRequestedChunks to terminate with error ErrDisposed.
69 | func (b *Builder) Dispose() {
70 | b.mutex.Lock()
71 | if b.disposed {
72 | panic("Builder must be disposed exactly once")
73 | }
74 | b.disposed = true
75 | started := b.started
76 | b.mutex.Unlock()
77 |
78 | close(b.done)
79 |
80 | if started {
81 | for chunk := range b.memos {
82 | if chunk.file != nil {
83 | chunk.file.Dispose()
84 | }
85 | }
86 | }
87 | }
88 |
89 | // Outputs a bit stream with '1' for each missing chunk, and
90 | // '0' for each chunk that is already available or already requested.
91 | func (b *Builder) WriteWishList(w FlushWriter) error {
92 | if LoggingEnabled {
93 | log.Printf("Receiver: Begin WriteWishList")
94 | defer log.Printf("Receiver: End WriteWishList")
95 | }
96 |
97 | if err := b.start(); err != nil {
98 | return err
99 | }
100 |
101 | defer close(b.memos)
102 |
103 | requested := make(map[cafs.SKey]bool)
104 | bitWriter := newBitWriter(w)
105 |
106 | consumeFunc := func(v interface{}) error {
107 | ci := v.(ChunkInfo)
108 | key := ci.Key
109 |
110 | mem := memo{
111 | ci: ci,
112 | }
113 |
114 | if key == emptyKey || requested[key] {
115 | // This key was already requested. Also, the empty key is never requested.
116 | mem.requested = false
117 | } else if file, err := b.storage.Get(&key); err != nil {
118 | // File was not found in storage -> request and remember
119 | mem.requested = true
120 | requested[key] = true
121 | } else {
122 | // File was already in storage -> prevent it from being collected until it is needed
123 | mem.file = file
124 | mem.requested = false
125 | requested[key] = true
126 | }
127 |
128 | // Write memo into channel. This might block if channel buffer is full.
129 | // Only wait until disposed.
130 | select {
131 | case b.memos <- mem:
132 | // Responsibility for disposing chunk.file is passed to the channel
133 | case <-b.done:
134 | if mem.file != nil {
135 | mem.file.Dispose()
136 | }
137 | return ErrDisposed
138 | }
139 |
140 | if err := bitWriter.WriteBit(mem.requested); err != nil {
141 | return err
142 | }
143 |
144 | return nil // success
145 | }
146 |
147 | // Create a shuffler using the above consumeFunc and push the SyncInfo's chunk infos through it.
148 | // For every ChunkInfo leaving the shuffler (in shuffled order), the consumeFunc
149 | // writes a bit into the wishlist.
150 | shuffler := shuffle.NewStreamShuffler(b.syncinf.Perm, emptyChunkInfo, consumeFunc)
151 | nChunks := len(b.syncinf.Chunks)
152 | for idx := 0; idx < nChunks; idx++ {
153 | if err := shuffler.Put(b.syncinf.Chunks[idx]); err != nil {
154 | return fmt.Errorf("error from shuffler.Put: %v", err)
155 | }
156 | }
157 | if err := shuffler.End(); err != nil {
158 | return fmt.Errorf("error from shuffler.End: %v", err)
159 | }
160 | return bitWriter.Flush()
161 | }
162 |
163 | // Function start is called by WriteWishList to mark the Builder as started.
164 | // This has consequences for the Dispose method.
165 | func (b *Builder) start() error {
166 | b.mutex.Lock()
167 | defer b.mutex.Unlock()
168 | if b.disposed {
169 | return ErrDisposed
170 | }
171 | if b.started {
172 | panic("WriteWishList called twice")
173 | }
174 | b.started = true
175 | return nil
176 | }
177 |
178 | var placeholder interface{} = struct{}{}
179 | var zeroMemo = memo{}
180 |
181 | // Reads a sequence of length-prefixed data chunks and tries to reconstruct a file from that
182 | // information.
183 | func (b *Builder) ReconstructFileFromRequestedChunks(_r io.Reader) (cafs.File, error) {
184 | if LoggingEnabled {
185 | log.Printf("Receiver: Begin ReconstructFileFromRequestedChunks")
186 | defer log.Printf("Receiver: End ReconstructFileFromRequestedChunks")
187 | }
188 |
189 | temp := b.storage.Create(b.info)
190 | defer temp.Dispose()
191 |
192 | r := bufio.NewReader(_r)
193 |
194 | errDone := errors.New("done")
195 |
196 | unshuffler := shuffle.NewInverseStreamShuffler(b.syncinf.Perm, placeholder, func(v interface{}) error {
197 | chunk := v.(cafs.File)
198 | // Write a chunk of the work file
199 | err := appendChunk(temp, chunk)
200 | chunk.Dispose()
201 | return err
202 | })
203 |
204 | // Make sure all chunks in the unshuffler are disposed in the end
205 | defer unshuffler.WithFunc(func(v interface{}) error {
206 | v.(cafs.File).Dispose()
207 | return nil
208 | }).End()
209 |
210 | idx := 0
211 | iteration := func() error {
212 | var mem memo
213 |
214 | // Wait until either a chunk info can be read from the channel, or the builder
215 | // has been disposed.
216 | select {
217 | case <-b.done:
218 | return ErrDisposed
219 | case mem = <-b.memos:
220 | // successfully read, continue...
221 | }
222 |
223 | // It is our responsibility to dispose the file.
224 | if mem.file != nil {
225 | defer mem.file.Dispose()
226 | }
227 |
228 | if mem.ci == emptyChunkInfo {
229 | return unshuffler.Put(placeholder)
230 | }
231 |
232 | // Under the following circumstances, read chunk data from the stream.
233 | // - chunk data was requested
234 | // - the chunk memo stream has ended (to check whether the chunk data stream also ends).
235 | // If there was a real error, abort.
236 | if mem.requested || mem == zeroMemo {
237 | chunkFile, err := readChunk(b.storage, r, fmt.Sprintf("%v #%d", b.info, idx))
238 | if chunkFile != nil {
239 | defer chunkFile.Dispose()
240 | }
241 | if err == io.EOF && mem == zeroMemo {
242 | return errDone
243 | } else if err == io.EOF {
244 | return io.ErrUnexpectedEOF
245 | } else if err != nil {
246 | return err
247 | } else if mem == zeroMemo {
248 | return fmt.Errorf("unsolicited chunk data")
249 | } else if chunkFile.Key() != mem.ci.Key {
250 | return ErrUnexpectedChunk
251 | } else if chunkFile.Size() != int64(mem.ci.Size) {
252 | return ErrUnexpectedChunk
253 | }
254 | }
255 |
256 | // Retrieve the chunk from CAFS (we can expect to find it)
257 | chunk, _ := b.storage.Get(&mem.ci.Key)
258 | // ... and dispatch it to the unshuffler, where it will be buffered for a while.
259 | // Disposing is done by the unshuffler's ConsumeFunc.
260 | if LoggingEnabled {
261 | log.Printf("Receiver: unshuffler.Put(total:%v, %v)", chunk.Size(), chunk.Key())
262 | }
263 | return unshuffler.Put(chunk)
264 | }
265 |
266 | for {
267 | if err := iteration(); err == errDone {
268 | break
269 | } else if err != nil {
270 | return nil, err
271 | }
272 | idx++
273 | }
274 |
275 | if err := unshuffler.End(); err != nil {
276 | return nil, err
277 | }
278 |
279 | if err := temp.Close(); err != nil {
280 | return nil, err
281 | }
282 |
283 | return temp.File(), nil
284 | }
285 |
286 | // Function appendChunk appends data of `chunk` to `temp`.
287 | func appendChunk(temp io.Writer, chunk cafs.File) error {
288 | if LoggingEnabled {
289 | log.Printf("Receiver: appendChunk(total:%v, %v)", chunk.Size(), chunk.Key())
290 | }
291 | r := chunk.Open()
292 | //noinspection GoUnhandledErrorResult
293 | defer r.Close()
294 | if _, err := io.Copy(temp, r); err != nil {
295 | return err
296 | }
297 | return nil
298 | }
299 |
--------------------------------------------------------------------------------
/remotesync/remotesync.go:
--------------------------------------------------------------------------------
1 | // BitWrk - A Bitcoin-friendly, anonymous marketplace for computing power
2 | // Copyright (C) 2013-2018 Jonas Eschenburg
3 | //
4 | // This program is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // This program is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU General Public License
15 | // along with this program. If not, see .
16 |
17 | // Package remotesync implements a differential file synching mechanism based on the content-based chunking
18 | // that is used by CAFS internally.
19 | // Step 1: Sender and receiver agree on hashes of the file's chunks
20 | // Step 2: Receiver streams missing chunks (one bit per chunk)
21 | // Step 3: Sender responds by sending content of requested chunks
22 | package remotesync
23 |
24 | var LoggingEnabled = false
25 |
--------------------------------------------------------------------------------
/remotesync/remotesync_test.go:
--------------------------------------------------------------------------------
1 | package remotesync
2 |
3 | import (
4 | "bufio"
5 | "fmt"
6 | "github.com/indyjo/cafs"
7 | . "github.com/indyjo/cafs/ram"
8 | "github.com/indyjo/cafs/remotesync/shuffle"
9 | "io"
10 | "math/rand"
11 | "testing"
12 | )
13 |
14 | // This is a regression test that deadlocks as long as indyjo/bitwrk#152 isn't solved.
15 | // https://github.com/indyjo/bitwrk/issues/152
16 | func TestDispose(t *testing.T) {
17 | store := NewRamStorage(256 * 1024)
18 | syncinfo := &SyncInfo{}
19 | syncinfo.SetPermutation(rand.Perm(10))
20 | builder := NewBuilder(store, syncinfo, 8, "Test file")
21 | // Dispose builder before call to WriteWishList
22 | builder.Dispose()
23 | }
24 |
25 | func TestRemoteSync(t *testing.T) {
26 | // Re-use stores to test for leaks on the fly
27 | storeA := NewRamStorage(8 * 1024 * 1024)
28 | storeB := NewRamStorage(8 * 1024 * 1024)
29 | // LoggingEnabled = true
30 |
31 | // Test for different amounts of overlapping data
32 | for _, p := range []float64{0, 0.01, 0.25, 0.5, 0.75, 0.99, 1} {
33 | // Test for different number of blocks, so that storeB will _almost_ be filled up.
34 | // We can't test up to 512 because we don't know how much overhead data was produced
35 | // by the chunking algorithm (yes, RAM storage counts that overhead!)
36 | for _, nBlocks := range []int{0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 400} {
37 | sigma := 0.25
38 | if nBlocks > 256 {
39 | sigma = 0
40 | }
41 | // Test for different permutation sizes
42 | for _, permSize := range []int{1, 2, 3, 5, 10, 100, 1000} {
43 | perm := shuffle.Permutation(rand.Perm(permSize))
44 | func() {
45 | defer reportUsage(t, "B", storeB)
46 | defer reportUsage(t, "A", storeA)
47 | testWithParams(t, storeA, storeB, p, sigma, nBlocks, perm)
48 | }()
49 | }
50 | }
51 | }
52 | }
53 |
54 | func check(t *testing.T, msg string, err error) {
55 | if err != nil {
56 | t.Fatalf("Error %v: %v", msg, err)
57 | }
58 | }
59 |
60 | func testWithParams(t *testing.T, storeA, storeB cafs.BoundedStorage, p, sigma float64, nBlocks int, perm shuffle.Permutation) {
61 | t.Logf("Testing with params: p=%f, nBlocks=%d, permSize=%d", p, nBlocks, len(perm))
62 | tempA := storeA.Create(fmt.Sprintf("Data A(%.2f,%d)", p, nBlocks))
63 | defer tempA.Dispose()
64 | tempB := storeB.Create(fmt.Sprintf("Data B(%.2f,%d)", p, nBlocks))
65 | defer tempB.Dispose()
66 |
67 | check(t, "creating similar data", createSimilarData(tempA, tempB, p, sigma, 8192, nBlocks))
68 |
69 | check(t, "closing tempA", tempA.Close())
70 | check(t, "closing tempB", tempB.Close())
71 |
72 | fileA := tempA.File()
73 | defer fileA.Dispose()
74 |
75 | syncinf := &SyncInfo{}
76 | syncinf.SetPermutation(perm)
77 | syncinf.SetChunksFromFile(fileA)
78 | builder := NewBuilder(storeB, syncinf, 8, fmt.Sprintf("Recovered A(%.2f,%d)", p, nBlocks))
79 | defer builder.Dispose()
80 |
81 | // task: transfer file A to storage B
82 | // Pipe 1 is used to transfer the wishlist bit-stream from the receiver to the sender
83 | pipeReader1, pipeWriter1 := io.Pipe()
84 | // Pipe 2 is used to transfer the actual requested chunk data to the receiver
85 | pipeReader2, pipeWriter2 := io.Pipe()
86 |
87 | go func() {
88 | if err := builder.WriteWishList(NopFlushWriter{pipeWriter1}); err != nil {
89 | _ = pipeWriter1.CloseWithError(fmt.Errorf("Error generating wishlist: %v", err))
90 | } else {
91 | _ = pipeWriter1.Close()
92 | }
93 | }()
94 |
95 | go func() {
96 | chunks := ChunksOfFile(fileA)
97 | defer chunks.Dispose()
98 | if err := WriteChunkData(chunks, fileA.Size(), bufio.NewReader(pipeReader1), perm, NopFlushWriter{pipeWriter2}, nil); err != nil {
99 | _ = pipeWriter2.CloseWithError(fmt.Errorf("Error sending requested chunk data: %v", err))
100 | } else {
101 | _ = pipeWriter2.Close()
102 | }
103 | }()
104 |
105 | var fileB cafs.File
106 | if f, err := builder.ReconstructFileFromRequestedChunks(pipeReader2); err != nil {
107 | t.Fatalf("Error reconstructing: %v", err)
108 | } else {
109 | fileB = f
110 | defer f.Dispose()
111 | }
112 |
113 | _ = fileB
114 | assertEqual(t, fileA.Open(), fileB.Open())
115 | }
116 |
117 | func assertEqual(t *testing.T, a, b io.ReadCloser) {
118 | bufA := make([]byte, 1)
119 | bufB := make([]byte, 1)
120 | count := 0
121 | for {
122 | nA, errA := a.Read(bufA)
123 | nB, errB := b.Read(bufB)
124 | if nA != nB {
125 | t.Fatal("Chunks differ in total")
126 | }
127 | if errA != errB {
128 | t.Fatalf("Error a:%v b:%v", errA, errB)
129 | }
130 | if bufA[0] != bufB[0] {
131 | t.Fatalf("Chunks differ in content at position %v: %02x vs %02x", count, bufA[0], bufB[0])
132 | }
133 | if errA == io.EOF && errB == io.EOF {
134 | break
135 | }
136 | count++
137 | }
138 | check(t, "closing file a in assertEqual", a.Close())
139 | check(t, "closing file b in assertEqual", b.Close())
140 | }
141 |
142 | func createSimilarData(tempA, tempB io.Writer, p, sigma, avgchunk float64, numchunks int) error {
143 | for numchunks > 0 {
144 | numchunks--
145 | lengthA := int(avgchunk*sigma*rand.NormFloat64() + avgchunk)
146 | if lengthA < 16 {
147 | lengthA = 16
148 | }
149 | data := randomBytes(lengthA)
150 | if _, err := tempA.Write(data); err != nil {
151 | return err
152 | }
153 | same := rand.Float64() <= p
154 | if same {
155 | if _, err := tempB.Write(data); err != nil {
156 | return err
157 | }
158 | } else {
159 | lengthB := int(avgchunk*sigma*rand.NormFloat64() + avgchunk)
160 | if lengthB < 16 {
161 | lengthB = 16
162 | }
163 | data = randomBytes(lengthB)
164 | if _, err := tempB.Write(data); err != nil {
165 | return err
166 | }
167 | }
168 | }
169 | return nil
170 | }
171 |
172 | func randomBytes(length int) []byte {
173 | result := make([]byte, 0, length)
174 | for len(result) < length {
175 | result = append(result, byte(rand.Int()))
176 | }
177 | return result
178 | }
179 |
180 | type testPrinter struct {
181 | t *testing.T
182 | }
183 |
184 | func (t *testPrinter) Printf(format string, v ...interface{}) {
185 | t.t.Logf(format, v...)
186 | }
187 |
188 | func reportUsage(t *testing.T, name string, store cafs.BoundedStorage) {
189 | store.FreeCache()
190 | ui := store.GetUsageInfo()
191 | if ui.Locked != 0 {
192 | t.Errorf(" Store %v: %v", name, ui)
193 | store.DumpStatistics(&testPrinter{t})
194 | t.FailNow()
195 | }
196 | }
197 |
--------------------------------------------------------------------------------
/remotesync/send.go:
--------------------------------------------------------------------------------
1 | // BitWrk - A Bitcoin-friendly, anonymous marketplace for computing power
2 | // Copyright (C) 2013-2019 Jonas Eschenburg
3 | //
4 | // This program is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // This program is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU General Public License
15 | // along with this program. If not, see .
16 |
17 | package remotesync
18 |
19 | import (
20 | "errors"
21 | "fmt"
22 | "github.com/indyjo/cafs"
23 | "github.com/indyjo/cafs/remotesync/shuffle"
24 | "io"
25 | "log"
26 | )
27 |
28 | // By passing a callback function to some of the transmissions functions,
29 | // the caller may subscribe to the current transmission status.
30 | type TransferStatusCallback func(bytesToTransfer, bytesTransferred int64)
31 |
32 | // Interface Chunks allows iterating over any sequence of chunks.
33 | type Chunks interface {
34 | // Function NextChunk returns either of three cases:
35 | // - A File, nil (good case)
36 | // - nil, io.EOF (terminal case: end of stream)
37 | // - nil, an error (terminal case: an error occurred)
38 | // It is the caller's duty to call Dispose() on the file returned.
39 | NextChunk() (cafs.File, error)
40 |
41 | // Function Dispose must be called when this object is no longer used.
42 | Dispose()
43 | }
44 |
45 | // Function ChunksOfFiles returns the chunks of a File as an implementation of the Chunks
46 | // interface. It's the caller's responsibility to call Dispose() on the returned object.
47 | func ChunksOfFile(file cafs.File) Chunks {
48 | return chunksOfFile{iter: file.Chunks()}
49 | }
50 |
51 | // Struct chunksOfFile is a minimal wrapper around a FileIterator that implements
52 | // the Chunks interface.
53 | type chunksOfFile struct {
54 | iter cafs.FileIterator
55 | }
56 |
57 | func (c chunksOfFile) NextChunk() (cafs.File, error) {
58 | if c.iter.Next() {
59 | return c.iter.File(), nil
60 | }
61 | return nil, io.EOF
62 | }
63 |
64 | func (c chunksOfFile) Dispose() {
65 | c.iter.Dispose()
66 | }
67 |
68 | // Iterates over a wishlist (read from `r` and pertaining to a permuted order of hashes),
69 | // and calls `f` for each chunk of `file`, requested or not.
70 | // If `f` returns an error, aborts the iteration and also returns the error.
71 | func forEachChunk(chunks Chunks, r io.ByteReader, perm shuffle.Permutation, f func(chunk cafs.File, requested bool) error) error {
72 | bits := newBitReader(r)
73 |
74 | // Prepare shuffler for iterating the file's chunks in shuffled order, matching them with
75 | // whishlist bits and calling `f` for each chunk, requested or not.
76 | shuffler := shuffle.NewStreamShuffler(perm, nil, func(v interface{}) error {
77 | var requested bool
78 | if b, err := bits.ReadBit(); err != nil {
79 | return fmt.Errorf("error reading from wishlist bitstream: %v", err)
80 | } else {
81 | requested = b
82 | }
83 |
84 | if v == nil {
85 | // This is a placeholder key generated by the shuffler. Require that the receiver
86 | // signalled not to request the corresponding chunk.
87 | if requested {
88 | return errors.New("receiver requested the empty chunk")
89 | }
90 | // otherwise, there's nothing to do
91 | return nil
92 | }
93 |
94 | // We have a chunk with a corresponding wishlist bit. Dispatch to delegate function.
95 | chunk := v.(cafs.File)
96 | err := f(chunk, requested)
97 | chunk.Dispose()
98 | return err
99 | })
100 |
101 | // At the end of this function, we must make sure that all chunks still stored
102 | // in the shuffler are disposed of.
103 | defer func() {
104 | s := shuffler.WithFunc(func(v interface{}) error {
105 | if v != nil {
106 | v.(cafs.File).Dispose()
107 | }
108 | return nil
109 | })
110 | _ = s.End()
111 | }()
112 |
113 | // Iterate through the chunks and put their keys into the shuffler.
114 | for {
115 | if chunk, err := chunks.NextChunk(); err == nil {
116 | if err := shuffler.Put(chunk); err != nil {
117 | return err
118 | }
119 | } else if err == io.EOF {
120 | break
121 | } else {
122 | return err
123 | }
124 |
125 | }
126 | if err := shuffler.End(); err != nil {
127 | return err
128 | }
129 |
130 | // Expect whishlist byte stream to be read completely
131 | if _, err := r.ReadByte(); err != io.EOF {
132 | return errors.New("wishlist too long")
133 | }
134 | return nil
135 | }
136 |
137 | // Writes a stream of chunk length / data pairs, permuted by a shuffler corresponding to `perm`,
138 | // into an io.Writer, based on the chunks of a file and a matching permuted wishlist of requested chunks,
139 | // read from `r`.
140 | func WriteChunkData(chunks Chunks, bytesToTransfer int64, r io.ByteReader, perm shuffle.Permutation, w FlushWriter, cb TransferStatusCallback) error {
141 | if LoggingEnabled {
142 | log.Printf("Sender: Begin WriteChunkData")
143 | defer log.Printf("Sender: End WriteChunkData")
144 | }
145 |
146 | // Determine the number of bytes to transmit by starting at the maximum and subtracting chunk
147 | // total whenever we read a 0 (chunk not requested)
148 | if cb != nil {
149 | cb(bytesToTransfer, 0)
150 | }
151 |
152 | // Iterate requested chunks. Write the chunk's length (as varint) and the chunk data
153 | // into the output writer. Update the number of bytes transferred on the go.
154 | var bytesTransferred int64
155 | return forEachChunk(chunks, r, perm, func(chunk cafs.File, requested bool) error {
156 | if requested {
157 | if err := writeVarint(w, chunk.Size()); err != nil {
158 | return err
159 | }
160 | r := chunk.Open()
161 | if n, err := io.Copy(w, r); err != nil {
162 | _ = r.Close()
163 | return err
164 | } else {
165 | w.Flush()
166 | bytesTransferred += n
167 | }
168 | if err := r.Close(); err != nil {
169 | return err
170 | }
171 | } else {
172 | bytesToTransfer -= chunk.Size()
173 | }
174 | if cb != nil {
175 | // Notify callback of status
176 | cb(bytesToTransfer, bytesTransferred)
177 | }
178 | return nil
179 | })
180 | }
181 |
--------------------------------------------------------------------------------
/remotesync/shuffle/shuffle.go:
--------------------------------------------------------------------------------
1 | // BitWrk - A Bitcoin-friendly, anonymous marketplace for computing power
2 | // Copyright (C) 2013-2017 Jonas Eschenburg
3 | //
4 | // This program is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // This program is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU General Public License
15 | // along with this program. If not, see .
16 |
17 | // Package shuffle implements an efficient algorithm for performing a
18 | // cyclic permutation on a possibly infinite stream of data elements.
19 | package shuffle
20 |
21 | import "math/rand"
22 |
23 | // Type Permutation contains a permutation of integer numbers 0..k-1,
24 | // where k is the length of the permutation cycle.
25 | type Permutation []int
26 |
27 | // Type Shuffler implements a buffer for permuting a stream of
28 | // data elements.
29 | //
30 | // Data elements are put into a Shuffler and retrieved from it
31 | // in a different order. Internally using a buffer of size k, each
32 | // data element is retrieved from the buffer up to k-1 steps after
33 | // it is put in. A stream of data elemnts shuffled this way is
34 | // reversible to its original order.
35 | type Shuffler struct {
36 | perm Permutation
37 | buffer []interface{}
38 | idx int
39 | }
40 |
41 | // Interface StreamShuffler is common for shufflers and unshufflers working on a
42 | // stream with a well-defined beginning and end.
43 | type StreamShuffler interface {
44 | // Puts one data element into the StreamShuffler. Calls the ConsumeFunc exactly once.
45 | Put(interface{}) error
46 | // Feeds remaining data from the buffer into the ConsumeFunc, calling it k-1 times.
47 | End() error
48 | // Returns a shallow copy of this StreamShuffler with a different ConsumeFunc.
49 | WithFunc(consume ConsumeFunc) StreamShuffler
50 | }
51 |
52 | // Type ConsumeFunc defines a function that accepts one parameter of
53 | // arbitrary type and returns an error.
54 | type ConsumeFunc func(interface{}) error
55 |
56 | // Functions of type applyFunc are used internally to apply a value returned from
57 | // the shuffler to a ConsumeFunc. This is necessary because shuffling and unshuffling
58 | // are not fully symmetric and require the substitution or removal of placeholder values.
59 | type applyFunc func(ConsumeFunc, interface{}) error
60 |
61 | // Type streamShuffler uses a Shuffler to permute a sequence of arbitrary length.
62 | type streamShuffler struct {
63 | consume ConsumeFunc
64 | shuffler *Shuffler
65 | apply applyFunc
66 | }
67 |
68 | // Creates a random permutation of given length.
69 | func Random(size int, r *rand.Rand) Permutation {
70 | return r.Perm(size)
71 | }
72 |
73 | // Given a permutation p, creates a complimentary permutation p'
74 | // such that using the output of a Shuffler based on p as the input
75 | // of a Shuffler based on p' restores the original stream order
76 | // delayed by len(p) - 1 steps.
77 | func (p Permutation) Inverse() Permutation {
78 | inv := make(Permutation, len(p))
79 | for i, j := range p {
80 | inv[j] = (len(p) - 1 + i) % len(p)
81 | }
82 | return inv
83 | }
84 |
85 | // Creates a new Shuffler based on permutation p.
86 | func NewShuffler(p Permutation) *Shuffler {
87 | result := new(Shuffler)
88 | result.perm = p
89 | result.buffer = make([]interface{}, len(p))
90 | return result
91 | }
92 |
93 | // Inputs a data element v into the shuffler and simultaneously
94 | // retrieves another (or, every k invocations, the same) data element.
95 | // May return nil while the buffer hasn't been completely filled.
96 | func (s *Shuffler) Put(v interface{}) interface{} {
97 | i := s.idx
98 | s.idx++
99 | if s.idx == len(s.buffer) {
100 | s.idx = 0
101 | }
102 | s.buffer[s.perm[i]] = v
103 | return s.buffer[i]
104 | }
105 |
106 | // Returns a complimentary shuffler that reverses the permutation (except
107 | // for a delay of k-1 steps).
108 | func (s *Shuffler) Inverse() *Shuffler {
109 | return NewShuffler(s.perm.Inverse())
110 | }
111 |
112 | // Returns the length k of the permutation buffer used by the shuffler.
113 | func (s *Shuffler) Length() int {
114 | return len(s.buffer)
115 | }
116 |
117 | // Creates a StreamShuffler applying a permutation to a stream. Argument `placeholder`
118 | // specifies a value that is inserted into the permuted stream in order to symbolize blank space.
119 | func NewStreamShuffler(p Permutation, placeholder interface{}, consume ConsumeFunc) StreamShuffler {
120 | return &streamShuffler{
121 | consume: consume,
122 | shuffler: NewShuffler(p),
123 | apply: func(f ConsumeFunc, v interface{}) error {
124 | if v == nil {
125 | v = placeholder
126 | }
127 | return f(v)
128 | },
129 | }
130 | }
131 |
132 | func (e *streamShuffler) Put(v interface{}) error {
133 | r := e.shuffler.Put(v)
134 | return e.apply(e.consume, r)
135 | }
136 |
137 | func (e *streamShuffler) End() error {
138 | for i := 0; i < e.shuffler.Length()-1; i++ {
139 | r := e.shuffler.Put(nil)
140 | if err := e.apply(e.consume, r); err != nil {
141 | return err
142 | }
143 | }
144 | return nil
145 | }
146 |
147 | func (e *streamShuffler) WithFunc(consume ConsumeFunc) StreamShuffler {
148 | var s streamShuffler
149 | s = *e
150 | s.consume = consume
151 | return &s
152 | }
153 |
154 | // Creates a StreamShuffler applying the inverse permutation and thereby restoring
155 | // the original stream order. Argument `placeholder` specifies blank space inserted
156 | // into the stream by the original shuffler. Values equal to `placeholder` will not
157 | // be forwarded to `consume`.
158 | func NewInverseStreamShuffler(p Permutation, placeholder interface{}, consume ConsumeFunc) StreamShuffler {
159 | return &streamShuffler{
160 | consume: consume,
161 | shuffler: NewShuffler(p.Inverse()),
162 | apply: func(f ConsumeFunc, v interface{}) error {
163 | if v == nil || v == placeholder {
164 | return nil
165 | }
166 | return f(v)
167 | },
168 | }
169 | }
170 |
--------------------------------------------------------------------------------
/remotesync/shuffle/shuffle_test.go:
--------------------------------------------------------------------------------
1 | package shuffle
2 |
3 | import (
4 | "math/rand"
5 | "testing"
6 | )
7 |
8 | func TestShuffler(t *testing.T) {
9 | rgen := rand.New(rand.NewSource(1))
10 | for _, permSize := range []int{1, 2, 3, 4, 5, 7, 10, 31, 57, 127, 512} {
11 | perm := Random(permSize, rgen)
12 | inv := perm.Inverse()
13 | for _, dataSize := range []int{1, 2, 3, 4, 5, 7, 10, 31, 57, 127, 512, 1024} {
14 | delay := len(perm) - 1
15 |
16 | for repeat := 0; repeat < 100; repeat++ {
17 | data := Random(dataSize, rgen)
18 |
19 | forward := NewShuffler(perm)
20 | inverse := NewShuffler(inv)
21 |
22 | for i := 0; i < len(data)+delay; i++ {
23 | var v interface{}
24 | if i < len(data) {
25 | v = data[i]
26 | } else {
27 | v = nil
28 | }
29 | w := inverse.Put(forward.Put(v))
30 | if i < delay {
31 | continue
32 | }
33 | if w.(int) != data[i-delay] {
34 | t.Errorf("When testing with permutations of size %v", permSize)
35 | t.Errorf(" and data of size %v,", dataSize)
36 | t.Fatalf(" a mismatch was detected on %vth iteration: w:%v != data[%v]:%v", i, w, i-delay, data[i-delay])
37 | }
38 | }
39 | }
40 | }
41 | }
42 | }
43 |
44 | func TestStreamShuffler(t *testing.T) {
45 | permutations := []Permutation{
46 | {0},
47 | {0, 1},
48 | {1, 0},
49 | {3, 4, 2, 1, 0},
50 | {4, 6, 3, 1, 5, 2, 0},
51 | {6, 7, 5, 3, 2, 1, 4, 0},
52 | {
53 | 24, 40, 90, 31, 11, 6, 26, 54, 76, 43, 79, 92, 7, 49, 17, 32, 80,
54 | 95, 15, 86, 20, 48, 94, 5, 27, 50, 65, 58, 38, 33, 60, 87, 36, 59,
55 | 85, 55, 23, 72, 47, 53, 39, 71, 96, 74, 82, 83, 28, 97, 62, 3, 45, 21,
56 | 2, 44, 70, 1, 25, 4, 68, 10, 19, 67, 77, 81, 51, 61, 35, 91, 84, 57,
57 | 16, 64, 78, 73, 93, 34, 29, 8, 30, 9, 66, 89, 52, 22, 18, 56, 13, 46,
58 | 69, 75, 88, 41, 42, 63, 12, 37, 14, 0,
59 | },
60 | }
61 | strings := []string{
62 | "",
63 | "x",
64 | "xy",
65 | "xyz",
66 | "0123456789abcde",
67 | "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
68 | }
69 | for _, perm := range permutations {
70 | for _, str := range strings {
71 | testWith(t, perm, str)
72 | }
73 | }
74 | }
75 |
76 | func testWith(t *testing.T, perm Permutation, original string) {
77 | shuffled := shuffleString(t, original, NewStreamShuffler(perm, '_', nil))
78 | unshuffled := shuffleString(t, shuffled, NewInverseStreamShuffler(perm, '_', nil))
79 |
80 | if original != unshuffled {
81 | t.Errorf("When testing with %#v (inverse: %#v):", perm, perm.Inverse())
82 | t.Fatalf(" Shuffling and unshuffling returned %#v. Expected: %#v. Shuffled: %#v",
83 | unshuffled, original, shuffled)
84 | }
85 | }
86 |
87 | func shuffleString(t *testing.T, in string, s StreamShuffler) (out string) {
88 | f := func(v interface{}) error {
89 | out = out + string(v.(rune))
90 | return nil
91 | }
92 | s = s.WithFunc(f)
93 | for _, c := range in {
94 | s.Put(c)
95 | }
96 | s.End()
97 | return
98 | }
99 |
100 | // Test that doing multiple simultaneous transmissions via a buffered connection to a simultated
101 | // caching receiver actually reduces the amount of data per transmission to the expected degree.
102 | func TestTransmission(t *testing.T) {
103 | const NTRANSMISSIONS = 3
104 | const BUFFER_SIZE = 1000
105 | const PERMUTATION_SIZE = 8000
106 |
107 | // Simulate a cache of already received data. The value expresses at which time step the data is received.
108 | // That's how we can simulate buffering behavior.
109 | received := make(map[int]int)
110 | // Counter of data that was actually sent
111 | sent := 0
112 |
113 | // Time step.
114 | time := 0
115 |
116 | // Create a number of shufflers that simulate parallel transmissions.
117 | transmissions := make([]StreamShuffler, NTRANSMISSIONS)
118 |
119 | r := rand.New(rand.NewSource(0))
120 | for i := range transmissions {
121 | transmissions[i] = NewStreamShuffler(Random(PERMUTATION_SIZE, r), -1, func(v interface{}) error {
122 | if v.(int) >= 0 {
123 | // Test if data has arrived at cache yet. If not, put it in and trigger retransmission
124 | if t_received, ok := received[v.(int)]; !ok || time < t_received {
125 | if !ok {
126 | received[v.(int)] = time + BUFFER_SIZE
127 | }
128 | sent++
129 | }
130 | }
131 | return nil
132 | })
133 | }
134 |
135 | for i := 0; i < PERMUTATION_SIZE; i++ {
136 | for _, transmission := range transmissions {
137 | if err := transmission.Put(i); err != nil {
138 | t.Fatalf("Unexpected error: %v", err)
139 | }
140 | }
141 | time++
142 | }
143 |
144 | // Purposefully don't call End() on transmissions because that would not simulate parallel behavior.
145 | // Instead, flood the shufflers with a dummy value in parallel. By putting PERMUTATION_SIZE
146 | // dummy values per transmission, we can guarantee the buffer was flushed.
147 | for i := 0; i < PERMUTATION_SIZE; i++ {
148 | for _, transmission := range transmissions {
149 | if err := transmission.Put(-1); err != nil {
150 | t.Fatalf("Unexpected error: %v", err)
151 | }
152 | }
153 | time++
154 | }
155 |
156 | transmission_avg := float64(sent) / float64(PERMUTATION_SIZE)
157 | t.Logf("Transmissions/data: % 5.2f", transmission_avg)
158 |
159 | // This model is probably wrong because it assumes stochastic independence of data chunk transmissions
160 | t.Logf("Expected: % 5.2f", 1+float64(NTRANSMISSIONS-1)*float64(BUFFER_SIZE)/float64(PERMUTATION_SIZE))
161 | // TODO: Add actual test here
162 | }
163 |
--------------------------------------------------------------------------------
/remotesync/syncinfo.go:
--------------------------------------------------------------------------------
1 | // BitWrk - A Bitcoin-friendly, anonymous marketplace for computing power
2 | // Copyright (C) 2013-2019 Jonas Eschenburg
3 | //
4 | // This program is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // This program is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU General Public License
15 | // along with this program. If not, see .
16 |
17 | package remotesync
18 |
19 | import (
20 | "bufio"
21 | "fmt"
22 | "github.com/indyjo/cafs"
23 | "github.com/indyjo/cafs/chunking"
24 | "github.com/indyjo/cafs/remotesync/shuffle"
25 | "io"
26 | )
27 |
28 | // Struct SyncInfo contains information which two CAFS instances have to agree on before
29 | // transmitting a file.
30 | type SyncInfo struct {
31 | Chunks []ChunkInfo // hashes and sizes of chunks
32 | Perm shuffle.Permutation // the permutation of chunks to use when transferring
33 | }
34 |
35 | // Func SetNoPermutation sets the prmutation to the trivial permutation (the one that doesn't permute).
36 | func (s *SyncInfo) SetTrivialPermutation() {
37 | s.Perm = []int{0}
38 | }
39 |
40 | // Func SetPermutation sets the permutation to use when transferring chunks.
41 | func (s *SyncInfo) SetPermutation(perm shuffle.Permutation) {
42 | s.Perm = append(s.Perm[:0], perm...)
43 | }
44 |
45 | // Func SetChunksFromFile prepares sync information for a CAFS file.
46 | func (s *SyncInfo) SetChunksFromFile(file cafs.File) {
47 | if !file.IsChunked() {
48 | s.Chunks = append(s.Chunks[:0], ChunkInfo{
49 | Key: file.Key(),
50 | Size: intsize(file.Size()),
51 | })
52 | return
53 | }
54 |
55 | iter := file.Chunks()
56 | s.Chunks = s.Chunks[:0]
57 | for iter.Next() {
58 | s.addChunk(iter.Key(), iter.Size())
59 | }
60 | iter.Dispose()
61 | }
62 |
63 | // func ReadFromLegacyStream reads chunk hashes from a stream encoded in the format previously used. No permutation
64 | // data is sent and it is expected that permutation remain the trivial permutation {0}.
65 | func (s *SyncInfo) ReadFromLegacyStream(stream io.Reader) error {
66 | // We need ReadByte
67 | r := bufio.NewReader(stream)
68 |
69 | for {
70 | // Read a chunk hash and its size
71 | var key cafs.SKey
72 | if _, err := io.ReadFull(r, key[:]); err == io.EOF {
73 | break
74 | } else if err != nil {
75 | return fmt.Errorf("error reading chunk hash: %v", err)
76 | }
77 | var size int64
78 | if l, err := readChunkLength(r); err != nil {
79 | return fmt.Errorf("error reading size of chunk: %v", err)
80 | } else {
81 | size = l
82 | }
83 |
84 | s.addChunk(key, size)
85 | }
86 | return nil
87 | }
88 |
89 | // Func WriteToLegacyStream writes chunk hashes to a stream encoded in the format previously used.
90 | func (s *SyncInfo) WriteToLegacyStream(stream io.Writer) error {
91 | for _, ci := range s.Chunks {
92 | if _, err := stream.Write(ci.Key[:]); err != nil {
93 | return err
94 | }
95 | if err := writeVarint(stream, int64(ci.Size)); err != nil {
96 | return err
97 | }
98 | }
99 | return nil
100 | }
101 |
102 | func (s *SyncInfo) addChunk(key cafs.SKey, size int64) {
103 | s.Chunks = append(s.Chunks, ChunkInfo{key, intsize(size)})
104 | }
105 |
106 | func intsize(size int64) int {
107 | if size < 0 || size > chunking.MaxChunkSize {
108 | panic("invalid chunk total")
109 | }
110 | return int(size)
111 | }
112 |
113 | // Applies the permutation contained within the receiver to it's own list of chunks, returning
114 | // a SyncInfo with a permuted list of chunks and the trivial permutation.
115 | // This is interesting for assistive transfers where a file is offered for retrieval while it is
116 | // still being retrieved from a different source. In that case, forwarding file chunks in a
117 | // different shuffle order than the one retrieved would lead to unnecessary delays waiting for
118 | // a certain chunk while others are already available.
119 | func (s *SyncInfo) Shuffle() *SyncInfo {
120 | newChunks := make([]ChunkInfo, 0, len(s.Chunks))
121 | shuffler := shuffle.NewStreamShuffler(s.Perm, nil, func(v interface{}) error {
122 | if v != nil {
123 | newChunks = append(newChunks, v.(ChunkInfo))
124 | }
125 | return nil
126 | })
127 | for _, c := range s.Chunks {
128 | _ = shuffler.Put(c)
129 | }
130 | _ = shuffler.End()
131 | return &SyncInfo{
132 | Chunks: newChunks,
133 | Perm: shuffle.Permutation{0},
134 | }
135 | }
136 |
--------------------------------------------------------------------------------
/remotesync/syncinfo_test.go:
--------------------------------------------------------------------------------
1 | package remotesync
2 |
3 | import (
4 | "bytes"
5 | "encoding/json"
6 | "github.com/indyjo/cafs"
7 | "testing"
8 | )
9 |
10 | func TestSyncInfoJSON(t *testing.T) {
11 | s := SyncInfo{}
12 | s.addChunk(cafs.SKey{11, 22, 33, 44, 55, 66, 77, 88}, 1337)
13 | s.addChunk(cafs.SKey{11, 22, 33, 44, 55, 66, 77, 88}, 1337)
14 | b, err := json.Marshal(s)
15 | if err != nil {
16 | t.Fatalf("Error encoding: %v", err)
17 | }
18 | // t.Logf("%v", string(b))
19 |
20 | s2 := SyncInfo{}
21 | err = json.Unmarshal(b, &s2)
22 | if err != nil {
23 | t.Fatalf("Error decoding: %v", err)
24 | }
25 |
26 | b2, err := json.Marshal(s2)
27 | if err != nil {
28 | t.Fatalf("Error encoding: %v", err)
29 | }
30 |
31 | if !bytes.Equal(b, b2) {
32 | t.Fatalf("Encoding differs")
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/remotesync/util.go:
--------------------------------------------------------------------------------
1 | // BitWrk - A Bitcoin-friendly, anonymous marketplace for computing power
2 | // Copyright (C) 2013-2018 Jonas Eschenburg
3 | //
4 | // This program is free software: you can redistribute it and/or modify
5 | // it under the terms of the GNU General Public License as published by
6 | // the Free Software Foundation, either version 3 of the License, or
7 | // (at your option) any later version.
8 | //
9 | // This program is distributed in the hope that it will be useful,
10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 | // GNU General Public License for more details.
13 | //
14 | // You should have received a copy of the GNU General Public License
15 | // along with this program. If not, see .
16 |
17 | package remotesync
18 |
19 | import (
20 | "bufio"
21 | "encoding/binary"
22 | "fmt"
23 | "github.com/indyjo/cafs"
24 | "github.com/indyjo/cafs/chunking"
25 | "io"
26 | "net/http"
27 | )
28 |
29 | // Interface FlushWriter acts like an io.Writer with an additional Flush method.
30 | type FlushWriter interface {
31 | io.Writer
32 | Flush()
33 | }
34 |
35 | // Struct SimpleFlushWriter implements FlushWriter using a Writer and a Flusher.
36 | type SimpleFlushWriter struct {
37 | W io.Writer
38 | F http.Flusher
39 | }
40 |
41 | func (s SimpleFlushWriter) Write(p []byte) (n int, err error) {
42 | return s.W.Write(p)
43 | }
44 |
45 | func (s SimpleFlushWriter) Flush() {
46 | s.F.Flush()
47 | }
48 |
49 | // An implementation of FlushWriter whose Flush() function is a nop.
50 | type NopFlushWriter struct {
51 | W io.Writer
52 | }
53 |
54 | func (f NopFlushWriter) Write(p []byte) (n int, err error) {
55 | return f.W.Write(p)
56 | }
57 |
58 | func (f NopFlushWriter) Flush() {
59 | }
60 |
61 | // The key pertaining to the SHA256 of an empty string is used to represent placeholders
62 | // for empty slots generated by shuffled transmissions.
63 | var emptyKey = *cafs.MustParseKey("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
64 |
65 | // Type ChunkInfo contains a chunk's hash and size.
66 | type ChunkInfo struct {
67 | Key cafs.SKey
68 | Size int
69 | }
70 |
71 | var emptyChunkInfo = ChunkInfo{emptyKey, 0}
72 |
73 | func readChunkLength(r *bufio.Reader) (int64, error) {
74 | if l, err := binary.ReadVarint(r); err != nil {
75 | return 0, err
76 | } else if l < 0 || l > chunking.MaxChunkSize {
77 | return 0, fmt.Errorf("Illegal chunk length: %v", l)
78 | } else {
79 | return l, nil
80 | }
81 | }
82 |
83 | func writeVarint(w io.Writer, value int64) error {
84 | var buf [binary.MaxVarintLen64]byte
85 | _, err := w.Write(buf[:binary.PutVarint(buf[:], value)])
86 | return err
87 | }
88 |
89 | type bitWriter struct {
90 | w FlushWriter
91 | n int
92 | buf [1]byte
93 | }
94 |
95 | func newBitWriter(writer FlushWriter) *bitWriter {
96 | return &bitWriter{w: writer}
97 | }
98 |
99 | func (w *bitWriter) WriteBit(b bool) (err error) {
100 | if b {
101 | w.buf[0] = (w.buf[0] << 1) | 1
102 | } else {
103 | w.buf[0] = w.buf[0] << 1
104 | }
105 | w.n++
106 | if w.n == 8 {
107 | _, err = w.w.Write(w.buf[:])
108 | if err == nil {
109 | w.w.Flush()
110 | }
111 | w.n = 0
112 | }
113 | return
114 | }
115 |
116 | func (w *bitWriter) Flush() (err error) {
117 | for err == nil && w.n != 0 {
118 | err = w.WriteBit(false)
119 | }
120 | return
121 | }
122 |
123 | type bitReader struct {
124 | r io.ByteReader
125 | n uint
126 | b byte
127 | }
128 |
129 | func newBitReader(r io.ByteReader) *bitReader {
130 | return &bitReader{r: r, n: 0, b: 0}
131 | }
132 |
133 | func (r *bitReader) ReadBit() (bit bool, err error) {
134 | if r.n == 8 {
135 | r.n = 0
136 | }
137 | if r.n == 0 {
138 | r.b, err = r.r.ReadByte()
139 | if err != nil {
140 | return
141 | }
142 | }
143 | n := r.n
144 | r.n++
145 | bit = 0 != (0x80 & (r.b << n))
146 | return
147 | }
148 |
149 | // Function readChunk reads a single chunk worth of data from stream `r` into a new
150 | // file on FileStorage `s`.
151 | // The expected encoding is (varint, data...).
152 | func readChunk(s cafs.FileStorage, r *bufio.Reader, info string) (cafs.File, error) {
153 | var length int64
154 | if n, err := readChunkLength(r); err != nil {
155 | return nil, err
156 | } else {
157 | length = n
158 | }
159 | tempChunk := s.Create(info)
160 | defer tempChunk.Dispose()
161 | if _, err := io.CopyN(tempChunk, r, length); err != nil {
162 | return nil, err
163 | }
164 | if err := tempChunk.Close(); err != nil {
165 | return nil, err
166 | }
167 | return tempChunk.File(), nil
168 | }
169 |
--------------------------------------------------------------------------------