├── go.sum.license ├── AUTHORS ├── .gitignore ├── test ├── all │ ├── all.go │ └── all_test.go ├── codec.go ├── registry.go ├── main.go ├── get.go ├── concurrent.go └── simple.go ├── .builds └── test.yml ├── multilog ├── test │ ├── all │ │ ├── all.go │ │ └── all_test.go │ ├── helpers_test.go │ ├── main.go │ ├── registry.go │ ├── multilog_live.go │ ├── sublog.go │ └── sink.go ├── roaring │ ├── fs │ │ └── fs.go │ ├── test │ │ ├── roaring_test.go │ │ └── test.go │ ├── mkv │ │ └── mkv.go │ ├── sqlite │ │ └── sqlite.go │ ├── badger │ │ ├── badger.go │ │ └── cmd │ │ │ └── mbdump │ │ │ └── main.go │ ├── sublog.go │ ├── qry.go │ └── multilog.go ├── multilog.go └── sink.go ├── offset2 ├── test │ ├── pump_test.go │ ├── test.go │ ├── pumplive.go │ └── pump.go ├── journal.go ├── offset.go ├── data.go ├── alter_test.go ├── alter_replace_test.go ├── rw_test.go ├── qry.go └── log.go ├── internal ├── persist │ ├── badger │ │ ├── opts.go │ │ ├── opts_lite.go │ │ ├── badger.go │ │ ├── shared_test.go │ │ └── saver.go │ ├── interface.go │ ├── mkv │ │ ├── modernkv.go │ │ └── saver.go │ ├── sqlite │ │ ├── sqlite.go │ │ └── saver.go │ ├── fs │ │ └── fs.go │ └── test │ │ └── general_test.go ├── tools │ └── tools.go └── seqobsv │ ├── simple_test.go │ └── seqobsv.go ├── indexes ├── mapidx │ ├── test │ │ ├── map_test.go │ │ └── test.go │ └── map.go ├── mkv │ ├── test │ │ ├── mkv_test.go │ │ └── test.go │ └── index.go ├── badger │ ├── test │ │ ├── badger_test.go │ │ └── test.go │ └── index.go ├── test │ ├── all │ │ ├── all.go │ │ └── all_test.go │ ├── registry.go │ ├── seqsetidx.go │ ├── setidx.go │ └── sinkindex.go ├── obv.go ├── sinkindex.go └── index.go ├── mem ├── test │ └── test.go ├── qry.go └── log.go ├── log_test.go ├── seq.go ├── seqwrap.go ├── .github └── workflows │ └── go.yml ├── codec.go ├── LICENSES ├── MIT.txt └── CC0-1.0.txt ├── LICENSE ├── log.go ├── go.mod ├── codec ├── json │ └── codec.go ├── cbor │ └── cbor.go └── msgpack │ └── msgpack.go ├── README.md └── qry.go /go.sum.license: -------------------------------------------------------------------------------- 1 | SPDX-FileCopyrightText: 2021 The margaret Authors 2 | 3 | SPDX-License-Identifier: CC0-1.0 -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2021 The margaret Authors 2 | # SPDX-License-Identifier: MIT 3 | 4 | Henry Bubert 5 | cblgh 6 | keks 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2021 The margaret Authors 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | *.swp 6 | *.swo 7 | 8 | c.out 9 | trace.out 10 | 11 | /test/all/TestLog* 12 | indexes/test/all/TestLog* 13 | testrun -------------------------------------------------------------------------------- /test/all/all.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package all 6 | 7 | import ( 8 | // import to register testing helpers 9 | _ "github.com/ssbc/margaret/mem/test" 10 | 11 | _ "github.com/ssbc/margaret/offset2/test" 12 | ) 13 | -------------------------------------------------------------------------------- /test/all/all_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package all 6 | 7 | import ( 8 | "testing" 9 | 10 | mtest "github.com/ssbc/margaret/test" 11 | ) 12 | 13 | func TestLog(t *testing.T) { 14 | mtest.RunTests(t) 15 | } 16 | -------------------------------------------------------------------------------- /.builds/test.yml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2021 The margaret Authors 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | image: alpine/latest 6 | packages: 7 | - go 8 | sources: 9 | - https://git.sr.ht/~cryptix/go-margaret 10 | tasks: 11 | - test: | 12 | cd go-margaret 13 | go test ./... 14 | -------------------------------------------------------------------------------- /test/codec.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "github.com/ssbc/margaret" 9 | ) 10 | 11 | // NewCodecFunc is a function that returns a codec 12 | type NewCodecFunc func(tipe interface{}) margaret.Codec 13 | -------------------------------------------------------------------------------- /multilog/test/all/all.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package all 6 | 7 | import ( 8 | // import to register testing helpers 9 | _ "github.com/ssbc/margaret/mem/test" 10 | _ "github.com/ssbc/margaret/offset2/test" 11 | 12 | _ "github.com/ssbc/margaret/multilog/roaring/test" 13 | ) 14 | -------------------------------------------------------------------------------- /offset2/test/pump_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "testing" 9 | ) 10 | 11 | func TestPump(t *testing.T) { 12 | for k, v := range newLogFuncs { 13 | t.Run(k+"/pump", LogTestPump(v)) 14 | t.Run(k+"/pumplive", LogTestPumpLive(v)) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /internal/persist/badger/opts.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | //go:build !lite 6 | // +build !lite 7 | 8 | package badger 9 | 10 | import ( 11 | "github.com/dgraph-io/badger/v3" 12 | ) 13 | 14 | func BadgerOpts(dbPath string) badger.Options { 15 | opts := badger.DefaultOptions(dbPath) 16 | return opts 17 | } 18 | -------------------------------------------------------------------------------- /multilog/roaring/fs/fs.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package fs 6 | 7 | import ( 8 | "github.com/ssbc/margaret/internal/persist/fs" 9 | "github.com/ssbc/margaret/multilog/roaring" 10 | ) 11 | 12 | func NewMultiLog(base string) (*roaring.MultiLog, error) { 13 | return roaring.NewStore(fs.New(base)), nil 14 | } 15 | -------------------------------------------------------------------------------- /indexes/mapidx/test/map_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/ssbc/margaret/indexes/test" 11 | ) 12 | 13 | func TestMap(t *testing.T) { 14 | t.Run("SetterIndex", test.RunSetterIndexTests) 15 | t.Run("SeqSetterIndex", test.RunSeqSetterIndexTests) 16 | } 17 | -------------------------------------------------------------------------------- /indexes/mkv/test/mkv_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/ssbc/margaret/indexes/test" 11 | ) 12 | 13 | func TestMKV(t *testing.T) { 14 | t.Run("SetterIndex", test.RunSetterIndexTests) 15 | t.Run("SeqSetterIndex", test.RunSeqSetterIndexTests) 16 | } 17 | -------------------------------------------------------------------------------- /indexes/badger/test/badger_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/ssbc/margaret/indexes/test" 11 | ) 12 | 13 | func TestBadger(t *testing.T) { 14 | t.Run("SetterIndex", test.RunSetterIndexTests) 15 | t.Run("SeqSetterIndex", test.RunSeqSetterIndexTests) 16 | } 17 | -------------------------------------------------------------------------------- /internal/tools/tools.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | // +build tools 6 | 7 | package tools 8 | 9 | import ( 10 | _ "github.com/maxbrunsfeld/counterfeiter/v6" 11 | ) 12 | 13 | // This file imports packages that are used when running go generate, or used 14 | // during the development process but not otherwise depended on by built code. 15 | -------------------------------------------------------------------------------- /mem/test/test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "github.com/ssbc/margaret" 9 | "github.com/ssbc/margaret/mem" 10 | mtest "github.com/ssbc/margaret/test" 11 | ) 12 | 13 | func init() { 14 | mtest.Register("mem", func(string, interface{}) (margaret.Log, error) { 15 | return mem.New(), nil 16 | }) 17 | } 18 | -------------------------------------------------------------------------------- /multilog/roaring/test/roaring_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/ssbc/margaret/multilog/test" 11 | ) 12 | 13 | func TestRoaringFiles(t *testing.T) { 14 | t.Run("SubLog", test.RunSubLogTests) 15 | t.Run("MultiLog", test.RunMultiLogTests) 16 | t.Run("Sink", test.RunSinkTests) 17 | } 18 | -------------------------------------------------------------------------------- /indexes/test/all/all.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package all 6 | 7 | import ( 8 | // imported only for side effects / registring testing helpers 9 | _ "github.com/ssbc/margaret/test/all" 10 | 11 | _ "github.com/ssbc/margaret/indexes/badger/test" 12 | _ "github.com/ssbc/margaret/indexes/mapidx/test" 13 | _ "github.com/ssbc/margaret/indexes/mkv/test" 14 | ) 15 | -------------------------------------------------------------------------------- /indexes/test/all/all_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package all 6 | 7 | import ( 8 | "testing" 9 | 10 | ltest "github.com/ssbc/margaret/indexes/test" 11 | ) 12 | 13 | func Test(t *testing.T) { 14 | t.Run("SeqSetterIndex", ltest.RunSeqSetterIndexTests) 15 | t.Run("SetterIndex", ltest.RunSetterIndexTests) 16 | t.Run("SinkIndex", ltest.RunSinkIndexTests) 17 | } 18 | -------------------------------------------------------------------------------- /multilog/roaring/mkv/mkv.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package mkv 6 | 7 | import ( 8 | "github.com/ssbc/margaret/internal/persist/mkv" 9 | "github.com/ssbc/margaret/multilog/roaring" 10 | ) 11 | 12 | func NewMultiLog(base string) (*roaring.MultiLog, error) { 13 | s, err := mkv.New(base) 14 | if err != nil { 15 | return nil, err 16 | } 17 | return roaring.NewStore(s), nil 18 | } 19 | -------------------------------------------------------------------------------- /multilog/roaring/sqlite/sqlite.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package sqlite 6 | 7 | import ( 8 | "github.com/ssbc/margaret/internal/persist/sqlite" 9 | "github.com/ssbc/margaret/multilog/roaring" 10 | ) 11 | 12 | func NewMultiLog(base string) (*roaring.MultiLog, error) { 13 | s, err := sqlite.New(base) 14 | if err != nil { 15 | return nil, err 16 | } 17 | return roaring.NewStore(s), nil 18 | } 19 | -------------------------------------------------------------------------------- /multilog/test/all/all_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package all 6 | 7 | import ( 8 | "testing" 9 | 10 | mltest "github.com/ssbc/margaret/multilog/test" 11 | ) 12 | 13 | func TestSink(t *testing.T) { 14 | mltest.RunSinkTests(t) 15 | } 16 | 17 | func TestMultiLog(t *testing.T) { 18 | mltest.RunMultiLogTests(t) 19 | } 20 | 21 | func TestSubLog(t *testing.T) { 22 | mltest.RunSubLogTests(t) 23 | } 24 | -------------------------------------------------------------------------------- /log_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package margaret 6 | 7 | import "testing" 8 | 9 | var _ error = ErrNulled 10 | 11 | func TestNulledErr(t *testing.T) { 12 | 13 | var e = ErrNulled 14 | 15 | var hidden interface{} 16 | 17 | hidden = e 18 | 19 | err, ok := hidden.(error) 20 | if !ok { 21 | t.Fatal("not an error") 22 | } 23 | 24 | if !IsErrNulled(err) { 25 | t.Fatal("not a nulled err") 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /test/registry.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import "testing" 8 | 9 | var NewLogFuncs map[string]NewLogFunc 10 | 11 | func init() { 12 | NewLogFuncs = map[string]NewLogFunc{} 13 | } 14 | 15 | func Register(name string, f NewLogFunc) { 16 | NewLogFuncs[name] = f 17 | } 18 | 19 | func RunTests(t *testing.T) { 20 | for name, newLog := range NewLogFuncs { 21 | t.Run(name, LogTest(newLog)) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /seq.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package margaret // import "github.com/ssbc/margaret" 6 | 7 | const ( 8 | // SeqEmpty is the current sequence number of an empty log 9 | SeqEmpty int64 = -1 10 | 11 | // SeqErrored is returned if an operation (like Append) fails 12 | SeqErrored int64 = -2 13 | 14 | SeqSublogDeleted int64 = -255 15 | ) 16 | 17 | // Seqer returns the current sequence of a log 18 | type Seqer interface { 19 | Seq() int64 20 | } 21 | -------------------------------------------------------------------------------- /test/main.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test // import "github.com/ssbc/margaret/test" 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/ssbc/margaret" 11 | ) 12 | 13 | type NewLogFunc func(string, interface{}) (margaret.Log, error) 14 | 15 | func LogTest(f NewLogFunc) func(*testing.T) { 16 | return func(t *testing.T) { 17 | t.Run("Get", LogTestGet(f)) 18 | t.Run("Simple", LogTestSimple(f)) 19 | t.Run("Concurrent", LogTestConcurrent(f)) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /internal/persist/badger/opts_lite.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | //go:build lite 6 | // +build lite 7 | 8 | package badger 9 | 10 | import ( 11 | "github.com/dgraph-io/badger/v3" 12 | ) 13 | 14 | func BadgerOpts(dbPath string) badger.Options { 15 | return badger.DefaultOptions(dbPath). 16 | WithMemTableSize(1 << 25). 17 | WithValueLogFileSize(1 << 25). 18 | WithNumMemtables(10). 19 | WithNumLevelZeroTables(3). 20 | WithNumLevelZeroTablesStall(7). 21 | WithNumCompactors(2). 22 | WithIndexCacheSize(1 << 27). 23 | WithBlockCacheSize(1 << 27). 24 | WithLogger(nil) 25 | } 26 | -------------------------------------------------------------------------------- /internal/persist/interface.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package persist 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "io" 11 | ) 12 | 13 | type Key []byte 14 | 15 | func (k Key) String() string { 16 | return fmt.Sprintf("key:%x", []byte(k)) 17 | } 18 | 19 | var ErrNotFound = errors.New("persist: item not found") 20 | 21 | type Saver interface { 22 | io.Closer 23 | Put(Key, []byte) error 24 | PutMultiple([]KeyValuePair) error 25 | 26 | Get(Key) ([]byte, error) 27 | 28 | List() ([]Key, error) 29 | 30 | Delete(Key) error 31 | } 32 | 33 | type KeyValuePair struct { 34 | Key Key 35 | Value []byte 36 | } 37 | -------------------------------------------------------------------------------- /indexes/mapidx/test/test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "github.com/ssbc/margaret/indexes" 9 | "github.com/ssbc/margaret/indexes/mapidx" 10 | "github.com/ssbc/margaret/indexes/test" 11 | ) 12 | 13 | func init() { 14 | newSeqSetterIdx := func(name string, tipe interface{}) (indexes.SeqSetterIndex, error) { 15 | return mapidx.New(), nil 16 | } 17 | 18 | newSetterIdx := func(name string, tipe interface{}) (indexes.SetterIndex, error) { 19 | return mapidx.New(), nil 20 | } 21 | 22 | test.RegisterSeqSetterIndex("mapidx", newSeqSetterIdx) 23 | test.RegisterSetterIndex("mapidx", newSetterIdx) 24 | } 25 | -------------------------------------------------------------------------------- /seqwrap.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package margaret 6 | 7 | // SeqWrapper wraps a value to attach a sequence number to it. 8 | type SeqWrapper interface { 9 | Seqer 10 | 11 | // Value returns the item itself. 12 | Value() interface{} 13 | } 14 | 15 | type seqWrapper struct { 16 | seq int64 17 | v interface{} 18 | } 19 | 20 | func (sw *seqWrapper) Seq() int64 { 21 | return sw.seq 22 | } 23 | 24 | func (sw *seqWrapper) Value() interface{} { 25 | return sw.v 26 | } 27 | 28 | // WrapWithSeq wraps the value v to attach a sequence number to it. 29 | func WrapWithSeq(v interface{}, seq int64) SeqWrapper { 30 | return &seqWrapper{ 31 | seq: seq, 32 | v: v, 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /multilog/roaring/badger/badger.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package badger 6 | 7 | import ( 8 | "github.com/dgraph-io/badger/v3" 9 | 10 | pbadger "github.com/ssbc/margaret/internal/persist/badger" 11 | "github.com/ssbc/margaret/multilog/roaring" 12 | ) 13 | 14 | func NewStandalone(base string) (*roaring.MultiLog, error) { 15 | s, err := pbadger.NewStandalone(base) 16 | if err != nil { 17 | return nil, err 18 | } 19 | return roaring.NewStore(s), nil 20 | } 21 | 22 | func NewShared(db *badger.DB, keyPrefix []byte) (*roaring.MultiLog, error) { 23 | s, err := pbadger.NewShared(db, keyPrefix) 24 | if err != nil { 25 | return nil, err 26 | } 27 | return roaring.NewStore(s), nil 28 | } 29 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2021 The margaret Authors 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | name: Go 6 | 7 | on: 8 | push: 9 | branches: [ master ] 10 | pull_request: 11 | branches: [ master ] 12 | 13 | jobs: 14 | 15 | build: 16 | name: Build 17 | runs-on: ubuntu-latest 18 | steps: 19 | 20 | - name: Set up Go 1.x 21 | uses: actions/setup-go@v2 22 | with: 23 | go-version: ^1.16 24 | id: go 25 | 26 | - name: Check out code into the Go module directory 27 | uses: actions/checkout@v2 28 | 29 | - name: Get dependencies 30 | run: go get -v -t -d ./... 31 | 32 | - name: Build smoke test 33 | run: go build ./... 34 | 35 | - name: Test 36 | run: go test ./... 37 | -------------------------------------------------------------------------------- /codec.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package margaret // import "github.com/ssbc/margaret" 6 | 7 | import ( 8 | "io" 9 | ) 10 | 11 | // Codec marshals and unmarshals values and creates encoders and decoders 12 | type Codec interface { 13 | // Marshal encodes a single value and returns the serialized byte slice. 14 | Marshal(value interface{}) ([]byte, error) 15 | 16 | // Unmarshal decodes and returns the value stored in data. 17 | Unmarshal(data []byte) (interface{}, error) 18 | 19 | NewDecoder(io.Reader) Decoder 20 | NewEncoder(io.Writer) Encoder 21 | } 22 | 23 | // Decoder decodes values 24 | type Decoder interface { 25 | Decode() (interface{}, error) 26 | } 27 | 28 | // Encoder encodes values 29 | type Encoder interface { 30 | Encode(v interface{}) error 31 | } 32 | -------------------------------------------------------------------------------- /multilog/test/helpers_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestFactor(t *testing.T) { 15 | type testcase struct { 16 | n int 17 | factors []int 18 | } 19 | 20 | test := func(tc testcase) func(*testing.T) { 21 | return func(t *testing.T) { 22 | a := assert.New(t) 23 | out := factorize(tc.n) 24 | a.Equal(tc.factors, out, "factor mismatch") 25 | } 26 | } 27 | 28 | tcs := []testcase{ 29 | { 30 | n: 30, 31 | factors: []int{2, 3, 5}, 32 | }, 33 | { 34 | n: 50, 35 | factors: []int{2, 5, 5}, 36 | }, 37 | { 38 | n: 1024, 39 | factors: []int{2, 2, 2, 2, 2, 2, 2, 2, 2, 2}, 40 | }, 41 | } 42 | 43 | for i, tc := range tcs { 44 | t.Run(fmt.Sprint(i), test(tc)) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /multilog/multilog.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package multilog 6 | 7 | import ( 8 | "errors" 9 | "io" 10 | 11 | "github.com/ssbc/margaret" 12 | "github.com/ssbc/margaret/indexes" 13 | ) 14 | 15 | var ( 16 | ErrSublogNotFound = errors.New("multilog: requested sublog not found") 17 | ErrSublogDeleted = errors.New("multilog: stored sublog was deleted. please re-open") 18 | ) 19 | 20 | // MultiLog is a collection of logs, keyed by a indexes.Addr 21 | type MultiLog interface { 22 | Get(indexes.Addr) (margaret.Log, error) 23 | List() ([]indexes.Addr, error) 24 | 25 | io.Closer 26 | 27 | Flush() error 28 | 29 | // Delete removes all entries related to that log 30 | Delete(indexes.Addr) error 31 | } 32 | 33 | func Has(mlog MultiLog, addr indexes.Addr) (bool, error) { 34 | slog, err := mlog.Get(addr) 35 | if err != nil { 36 | return false, err 37 | } 38 | 39 | return slog.Seq() != margaret.SeqEmpty, nil 40 | } 41 | -------------------------------------------------------------------------------- /multilog/test/main.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test // import "github.com/ssbc/margaret/multilog/test" 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/ssbc/margaret/multilog" 11 | ) 12 | 13 | type NewLogFunc func(name string, tipe interface{}, testdir string) (multilog.MultiLog, string, error) 14 | 15 | func SinkTest(f NewLogFunc) func(*testing.T) { 16 | return func(t *testing.T) { 17 | t.Run("Simple", SinkTestSimple(f)) 18 | } 19 | } 20 | 21 | func MultiLogTest(f NewLogFunc) func(*testing.T) { 22 | return func(t *testing.T) { 23 | // makes sure local fork reproduction doesn't make a reappearance 24 | t.Run("GetFreshThenReopenAndLogSomeMore", MultilogTestGetFreshLogCloseThenOpenAgain(f)) 25 | t.Run("MultiSimple", MultiLogTestSimple(f)) 26 | t.Run("Live", MultilogLiveQueryCheck(f)) 27 | } 28 | } 29 | 30 | func SubLogTest(f NewLogFunc) func(*testing.T) { 31 | return func(t *testing.T) { 32 | t.Run("Get", SubLogTestGet(f)) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /internal/persist/mkv/modernkv.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package mkv 6 | 7 | import ( 8 | "os" 9 | 10 | "github.com/pkg/errors" 11 | "github.com/ssbc/margaret/internal/persist" 12 | "modernc.org/kv" 13 | ) 14 | 15 | type ModernSaver struct { 16 | db *kv.DB 17 | } 18 | 19 | var _ persist.Saver = (*ModernSaver)(nil) 20 | 21 | func (sl ModernSaver) Close() error { 22 | return sl.db.Close() 23 | } 24 | 25 | func New(path string) (*ModernSaver, error) { 26 | var ms ModernSaver 27 | 28 | opts := &kv.Options{} 29 | _, err := os.Stat(path) 30 | if os.IsNotExist(err) { 31 | ms.db, err = kv.Create(path, opts) 32 | if err != nil { 33 | return nil, errors.Wrap(err, "failed to create KV") 34 | } 35 | } else if err != nil { 36 | return nil, errors.Wrap(err, "failed to stat path location") 37 | } else { 38 | ms.db, err = kv.Open(path, opts) 39 | if err != nil { 40 | return nil, errors.Wrap(err, "failed to open KV") 41 | } 42 | } 43 | 44 | return &ms, nil 45 | } 46 | -------------------------------------------------------------------------------- /LICENSES/MIT.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /multilog/test/registry.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "testing" 9 | ) 10 | 11 | var NewLogFuncs map[string]NewLogFunc 12 | 13 | func init() { 14 | NewLogFuncs = map[string]NewLogFunc{} 15 | } 16 | 17 | func Register(name string, f NewLogFunc) { 18 | NewLogFuncs[name] = f 19 | } 20 | 21 | func RunSinkTests(t *testing.T) { 22 | if len(NewLogFuncs) == 0 { 23 | t.Fatal("found no multilogs") 24 | } 25 | for name, newLog := range NewLogFuncs { 26 | t.Run(name, SinkTest(newLog)) 27 | } 28 | } 29 | 30 | func RunMultiLogTests(t *testing.T) { 31 | if len(NewLogFuncs) == 0 { 32 | t.Fatal("found no multilogs") 33 | } 34 | for name, newLog := range NewLogFuncs { 35 | t.Run(name+"-basic", MultiLogTest(newLog)) 36 | t.Run(name+"-handwoven", MultilogTestAddLogAndListed(newLog)) 37 | } 38 | } 39 | 40 | func RunSubLogTests(t *testing.T) { 41 | if len(NewLogFuncs) == 0 { 42 | t.Fatal("found no multilogs") 43 | } 44 | for name, newLog := range NewLogFuncs { 45 | t.Run(name, SubLogTest(newLog)) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /offset2/test/test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "github.com/ssbc/margaret" 9 | "github.com/ssbc/margaret/codec/cbor" 10 | "github.com/ssbc/margaret/codec/json" 11 | "github.com/ssbc/margaret/codec/msgpack" 12 | "github.com/ssbc/margaret/offset2" 13 | mtest "github.com/ssbc/margaret/test" 14 | ) 15 | 16 | var newLogFuncs map[string]mtest.NewLogFunc 17 | 18 | func init() { 19 | newLogFuncs = make(map[string]mtest.NewLogFunc) 20 | 21 | codecs := map[string]mtest.NewCodecFunc{ 22 | "json": json.New, 23 | "msgpack": msgpack.New, 24 | "cbor": cbor.New, 25 | } 26 | 27 | buildNewLogFunc := func(newCodec mtest.NewCodecFunc) mtest.NewLogFunc { 28 | return func(name string, tipe interface{}) (margaret.Log, error) { 29 | // name = strings.Replace(name, "/", "_", -1) 30 | return offset2.Open(name, newCodec(tipe)) 31 | } 32 | } 33 | 34 | for cname, newCodec := range codecs { 35 | mtest.Register("offset2/"+cname, buildNewLogFunc(newCodec)) 36 | newLogFuncs["offset2/"+cname] = buildNewLogFunc(newCodec) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Henry Bubert, Jan Winkelmann 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /indexes/obv.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package indexes 6 | 7 | import ( 8 | "sync" 9 | 10 | "github.com/ssbc/go-luigi" 11 | ) 12 | 13 | // NewObservable returns a regular observable that calls f when the last registration is cancelled. 14 | // This is used to garbage-collect observables from the maps in the indexes. 15 | func NewObservable(v interface{}, f func()) luigi.Observable { 16 | return &observable{ 17 | Observable: luigi.NewObservable(v), 18 | f: f, 19 | } 20 | } 21 | 22 | // observable is a regular luigi.Observable that calls f once all registrations are cancelled 23 | type observable struct { 24 | luigi.Observable 25 | 26 | l sync.Mutex 27 | i int 28 | f func() 29 | } 30 | 31 | // Register registers sink with the observable. 32 | func (obv *observable) Register(sink luigi.Sink) func() { 33 | obv.l.Lock() 34 | defer obv.l.Unlock() 35 | 36 | obv.i++ 37 | cancel := obv.Observable.Register(sink) 38 | 39 | return func() { 40 | cancel() 41 | 42 | obv.l.Lock() 43 | defer obv.l.Unlock() 44 | 45 | obv.i-- 46 | 47 | if obv.i == 0 { 48 | obv.f() 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /internal/persist/badger/badger.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package badger 6 | 7 | import ( 8 | "fmt" 9 | 10 | "github.com/dgraph-io/badger/v3" 11 | "github.com/ssbc/margaret/internal/persist" 12 | ) 13 | 14 | type BadgerSaver struct { 15 | db *badger.DB 16 | 17 | // shared means the backing db is shared with other indexes 18 | // this controls the closing behavior. 19 | // shared instances need to be closed independantly. 20 | shared bool 21 | 22 | keyPrefix []byte 23 | } 24 | 25 | var _ persist.Saver = (*BadgerSaver)(nil) 26 | 27 | // Close closes the backing database if it's not shared. 28 | func (sl *BadgerSaver) Close() error { 29 | if sl.shared { 30 | return nil 31 | } 32 | return sl.db.Close() 33 | } 34 | 35 | // NewStandalone opens 36 | func NewStandalone(path string) (*BadgerSaver, error) { 37 | var ms BadgerSaver 38 | 39 | var err error 40 | 41 | o := BadgerOpts(path) 42 | ms.db, err = badger.Open(o) 43 | if err != nil { 44 | return nil, fmt.Errorf("failed to create KV %s: %w", path, err) 45 | } 46 | 47 | return &ms, nil 48 | } 49 | 50 | func NewShared(db *badger.DB, keyPrefix []byte) (*BadgerSaver, error) { 51 | var ms BadgerSaver 52 | ms.db = db 53 | ms.shared = true 54 | ms.keyPrefix = keyPrefix 55 | return &ms, nil 56 | } 57 | -------------------------------------------------------------------------------- /indexes/mkv/test/test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "fmt" 9 | "io/ioutil" 10 | "os" 11 | "path/filepath" 12 | 13 | "modernc.org/kv" 14 | 15 | "github.com/ssbc/margaret/indexes" 16 | libmkv "github.com/ssbc/margaret/indexes/mkv" 17 | "github.com/ssbc/margaret/indexes/test" 18 | ) 19 | 20 | func init() { 21 | newSeqSetterIdx := func(name string, tipe interface{}) (indexes.SeqSetterIndex, error) { 22 | os.RemoveAll("testrun") 23 | os.MkdirAll("testrun", 0700) 24 | dir, err := ioutil.TempDir("./testrun", "mkv") 25 | if err != nil { 26 | return nil, fmt.Errorf("error creating tempdir: %w", err) 27 | } 28 | 29 | opts := &kv.Options{} 30 | db, err := kv.Create(filepath.Join(dir, "db"), opts) 31 | if err != nil { 32 | return nil, fmt.Errorf("error opening test database (%s): %w", dir, err) 33 | } 34 | 35 | return libmkv.NewIndex(db, tipe), nil 36 | } 37 | 38 | toSetterIdx := func(f test.NewSeqSetterIndexFunc) test.NewSetterIndexFunc { 39 | return func(name string, tipe interface{}) (indexes.SetterIndex, error) { 40 | idx, err := f(name, tipe) 41 | return idx, err 42 | } 43 | } 44 | 45 | test.RegisterSeqSetterIndex("mkv", newSeqSetterIdx) 46 | test.RegisterSetterIndex("mkv", toSetterIdx(newSeqSetterIdx)) 47 | } 48 | -------------------------------------------------------------------------------- /indexes/test/registry.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "testing" 9 | 10 | mtest "github.com/ssbc/margaret/test" 11 | ) 12 | 13 | var ( 14 | NewSetterIndexFuncs map[string]NewSetterIndexFunc 15 | NewSeqSetterIndexFuncs map[string]NewSeqSetterIndexFunc 16 | ) 17 | 18 | func init() { 19 | NewSetterIndexFuncs = map[string]NewSetterIndexFunc{} 20 | NewSeqSetterIndexFuncs = map[string]NewSeqSetterIndexFunc{} 21 | } 22 | 23 | func RegisterSetterIndex(name string, f NewSetterIndexFunc) { 24 | NewSetterIndexFuncs[name] = f 25 | } 26 | 27 | func RegisterSeqSetterIndex(name string, f NewSeqSetterIndexFunc) { 28 | NewSeqSetterIndexFuncs[name] = f 29 | } 30 | 31 | func RunSetterIndexTests(t *testing.T) { 32 | for name, newIndex := range NewSetterIndexFuncs { 33 | t.Run(name, TestSetterIndex(newIndex)) 34 | } 35 | } 36 | 37 | func RunSeqSetterIndexTests(t *testing.T) { 38 | for name, newIndex := range NewSeqSetterIndexFuncs { 39 | t.Run(name, TestSeqSetterIndex(newIndex)) 40 | } 41 | } 42 | 43 | func RunSinkIndexTests(t *testing.T) { 44 | for logname, newLog := range mtest.NewLogFuncs { 45 | for idxname, newSeqSetterIdx := range NewSeqSetterIndexFuncs { 46 | t.Run(logname+"/"+idxname, TestSinkIndex(newLog, newSeqSetterIdx)) 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package margaret // import "github.com/ssbc/margaret" 6 | 7 | import ( 8 | "errors" 9 | 10 | "github.com/ssbc/go-luigi" 11 | ) 12 | 13 | // Log stores entries sequentially, which can be queried individually using Get or as streams using Query. 14 | type Log interface { 15 | // Seq returns the current sequence number, which is also the number of entries in the log 16 | Seqer 17 | 18 | // Changes returns an observable that holds the current sequence number 19 | Changes() luigi.Observable 20 | 21 | // Get returns the entry with sequence number seq 22 | Get(seq int64) (interface{}, error) 23 | 24 | // Query returns a stream that is constrained by the passed query specification 25 | Query(...QuerySpec) (luigi.Source, error) 26 | 27 | // Append appends a new entry to the log 28 | Append(interface{}) (int64, error) 29 | } 30 | 31 | type oob struct{} 32 | 33 | // OOB is an out of bounds error 34 | var OOB oob 35 | 36 | func (oob) Error() string { 37 | return "out of bounds" 38 | } 39 | 40 | // IsOutOfBounds returns whether a particular error is an out-of-bounds error 41 | func IsOutOfBounds(err error) bool { 42 | _, ok := err.(oob) 43 | return ok 44 | } 45 | 46 | type Alterer interface { 47 | Null(int64) error 48 | 49 | Replace(int64, []byte) error 50 | } 51 | 52 | var ErrNulled = errors.New("margaret: Entry Nulled") 53 | 54 | func IsErrNulled(err error) bool { 55 | return errors.Is(err, ErrNulled) 56 | } 57 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | go 1.16 6 | 7 | module github.com/ssbc/margaret 8 | 9 | require ( 10 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 11 | github.com/dgraph-io/badger/v3 v3.2103.3 12 | github.com/dgraph-io/sroar v0.0.0-20220527172339-b92b7eaaf6e0 13 | github.com/go-logfmt/logfmt v0.5.1 // indirect 14 | github.com/golang/glog v1.0.0 // indirect 15 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 16 | github.com/golang/protobuf v1.5.2 // indirect 17 | github.com/golang/snappy v0.0.4 // indirect 18 | github.com/google/flatbuffers v22.10.26+incompatible // indirect 19 | github.com/hashicorp/errwrap v1.1.0 // indirect 20 | github.com/hashicorp/go-multierror v1.1.1 // indirect 21 | github.com/keks/persist v0.0.0-20210520094901-9bdd97c1fad2 22 | github.com/klauspost/compress v1.15.12 // indirect 23 | github.com/mattn/go-sqlite3 v1.14.16 24 | github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2 25 | github.com/pkg/errors v0.9.1 26 | github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa // indirect 27 | github.com/ssbc/go-luigi v0.3.7-0.20221019204020-324065b9a7c6 28 | github.com/stretchr/testify v1.8.1 29 | github.com/ugorji/go/codec v1.2.7 30 | go.mindeco.de v1.12.0 31 | go.opencensus.io v0.23.0 // indirect 32 | golang.org/x/net v0.23.0 // indirect 33 | google.golang.org/protobuf v1.33.0 // indirect 34 | modernc.org/fileutil v1.1.1 // indirect 35 | modernc.org/internal v1.0.5 // indirect 36 | modernc.org/kv v1.0.4 37 | modernc.org/lldb v1.0.4 // indirect 38 | modernc.org/sortutil v1.1.1 // indirect 39 | ) 40 | -------------------------------------------------------------------------------- /offset2/journal.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package offset2 6 | 7 | import ( 8 | "encoding/binary" 9 | "fmt" 10 | "io" 11 | "os" 12 | 13 | "github.com/ssbc/margaret" 14 | ) 15 | 16 | type journal struct { 17 | *os.File 18 | } 19 | 20 | func (j *journal) readSeq() (int64, error) { 21 | stat, err := j.Stat() 22 | if err != nil { 23 | return margaret.SeqEmpty, fmt.Errorf("stat failed: %w", err) 24 | } 25 | 26 | switch sz := stat.Size(); sz { 27 | case 0: 28 | return margaret.SeqEmpty, nil 29 | case 8: 30 | // continue after switch 31 | default: 32 | return margaret.SeqEmpty, fmt.Errorf("expected file size of 8B, got %dB", sz) 33 | } 34 | 35 | _, err = j.Seek(0, io.SeekStart) 36 | if err != nil { 37 | return margaret.SeqEmpty, fmt.Errorf("could not seek to start of file: %w", err) 38 | } 39 | 40 | var seq int64 41 | err = binary.Read(j, binary.BigEndian, &seq) 42 | if err != nil { 43 | return margaret.SeqErrored, fmt.Errorf("error reading seq: %w", err) 44 | } 45 | return seq, nil 46 | } 47 | 48 | func (j *journal) bump() (int64, error) { 49 | seq, err := j.readSeq() 50 | if err != nil { 51 | return margaret.SeqEmpty, fmt.Errorf("error reading old journal value: %w", err) 52 | } 53 | 54 | _, err = j.Seek(0, io.SeekStart) 55 | if err != nil { 56 | return margaret.SeqEmpty, fmt.Errorf("could not seek to start of file: %w", err) 57 | } 58 | 59 | seq = seq + 1 60 | err = binary.Write(j, binary.BigEndian, seq) 61 | if err != nil { 62 | return margaret.SeqEmpty, fmt.Errorf("error writing seq: %w", err) 63 | } 64 | 65 | return seq, nil 66 | } 67 | -------------------------------------------------------------------------------- /indexes/test/seqsetidx.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "context" 9 | "testing" 10 | 11 | "github.com/ssbc/margaret/indexes" 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | type NewSeqSetterIndexFunc func(name string, tipe interface{}) (indexes.SeqSetterIndex, error) 17 | 18 | func TestSeqSetterIndex(newIdx NewSeqSetterIndexFunc) func(*testing.T) { 19 | return func(t *testing.T) { 20 | t.Run("Sequential", TestSeqSetterIndexSequential(newIdx)) 21 | } 22 | } 23 | 24 | func TestSeqSetterIndexSequential(newIdx NewSeqSetterIndexFunc) func(*testing.T) { 25 | return func(t *testing.T) { 26 | a := assert.New(t) 27 | r := require.New(t) 28 | ctx := context.Background() 29 | 30 | idx, err := newIdx(t.Name(), "str") 31 | r.NoError(err) 32 | r.NotNil(idx) 33 | 34 | seq, err := idx.GetSeq() 35 | a.NoError(err, "returned by GetSeq before setting") 36 | a.EqualValues(-1, seq, "returned by GetSeq before setting") 37 | 38 | err = idx.Set(ctx, "test", "omg what is this") 39 | r.NoError(err, "error setting value") 40 | 41 | err = idx.SetSeq(0) 42 | a.NoError(err, "returned by SetSeq") 43 | 44 | obv, err := idx.Get(ctx, "test") 45 | r.NoError(err, "error getting observable") 46 | r.NotNil(obv) 47 | 48 | seq, err = idx.GetSeq() 49 | a.NoError(err, "returned by GetSeq after setting") 50 | a.EqualValues(0, seq, "returned by GetSeq after setting") 51 | 52 | v, err := obv.Value() 53 | a.NoError(err, "error getting value") 54 | a.Equal(v, "omg what is this") 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /test/get.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test // import "github.com/ssbc/margaret/test" 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func LogTestGet(f NewLogFunc) func(*testing.T) { 17 | type testcase struct { 18 | tipe interface{} 19 | values []interface{} 20 | result []interface{} 21 | } 22 | 23 | mkTest := func(tc testcase) func(*testing.T) { 24 | return func(t *testing.T) { 25 | a := assert.New(t) 26 | r := require.New(t) 27 | 28 | log, err := f(t.Name(), tc.tipe) 29 | r.NoError(err, "error creating log") 30 | r.NotNil(log, "returned log is nil") 31 | 32 | defer func() { 33 | if namer, ok := log.(interface{ FileName() string }); ok { 34 | r.NoError(os.RemoveAll(namer.FileName()), "error deleting log after test") 35 | } 36 | }() 37 | 38 | for i, v := range tc.values { 39 | seq, err := log.Append(v) 40 | r.NoError(err, "error appending to log") 41 | r.EqualValues(i, seq, "sequence missmatch") 42 | } 43 | 44 | for i, v_ := range tc.result { 45 | v, err := log.Get(int64(i)) 46 | a.NoError(err, "error getting value at position", i) 47 | a.Equal(v, v_, "value mismatch at position", i) 48 | } 49 | } 50 | } 51 | 52 | tcs := []testcase{ 53 | { 54 | tipe: 0, 55 | values: []interface{}{1, 2, 3}, 56 | result: []interface{}{1, 2, 3}, 57 | }, 58 | } 59 | 60 | return func(t *testing.T) { 61 | for i, tc := range tcs { 62 | t.Run(fmt.Sprint(i), mkTest(tc)) 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /indexes/sinkindex.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package indexes 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | 11 | "github.com/ssbc/go-luigi" 12 | "github.com/ssbc/margaret" 13 | ) 14 | 15 | type StreamProcFunc func(context.Context, int64, interface{}, SetterIndex) error 16 | 17 | func NewSinkIndex(f StreamProcFunc, idx SeqSetterIndex) SinkIndex { 18 | return &sinkIndex{ 19 | idx: idx, 20 | f: f, 21 | } 22 | } 23 | 24 | type sinkIndex struct { 25 | idx SeqSetterIndex 26 | f StreamProcFunc 27 | } 28 | 29 | func (r *sinkIndex) QuerySpec() margaret.QuerySpec { 30 | seq, err := r.idx.GetSeq() 31 | if err != nil { 32 | // wrap error in erroring queryspec 33 | return margaret.ErrorQuerySpec(err) 34 | } 35 | 36 | return margaret.MergeQuerySpec(margaret.Gt(seq), margaret.SeqWrap(true)) 37 | } 38 | 39 | func (idx *sinkIndex) Pour(ctx context.Context, v interface{}) error { 40 | switch tv := v.(type) { 41 | case margaret.SeqWrapper: 42 | err := idx.f(ctx, tv.Seq(), tv.Value(), idx.idx) 43 | if err != nil { 44 | return fmt.Errorf("error calling setter func: %w", err) 45 | } 46 | err = idx.idx.SetSeq(tv.Seq()) 47 | if err != nil { 48 | return fmt.Errorf("error setting sequence number: %w", err) 49 | } 50 | return nil 51 | case error: 52 | if margaret.IsErrNulled(tv) { 53 | return nil 54 | } 55 | return tv 56 | 57 | default: 58 | return fmt.Errorf("expecting seqwrapped value (%T)", v) 59 | } 60 | 61 | } 62 | 63 | func (idx *sinkIndex) Close() error { 64 | return idx.idx.Close() 65 | } 66 | 67 | func (idx *sinkIndex) Get(ctx context.Context, a Addr) (luigi.Observable, error) { 68 | return idx.idx.Get(ctx, a) 69 | } 70 | -------------------------------------------------------------------------------- /codec/json/codec.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package json // import "github.com/ssbc/margaret/codec/json" 6 | 7 | import ( 8 | "encoding/json" 9 | "io" 10 | "reflect" 11 | 12 | "github.com/ssbc/margaret" 13 | ) 14 | 15 | // New creates a json codec that decodes into values of type tipe. 16 | func New(tipe interface{}) margaret.Codec { 17 | if tipe == nil { 18 | return &codec{any: true} 19 | } 20 | 21 | t := reflect.TypeOf(tipe) 22 | isPtr := t.Kind() == reflect.Ptr 23 | if isPtr { 24 | t = t.Elem() 25 | } 26 | 27 | return &codec{ 28 | tipe: t, 29 | asPtr: isPtr, 30 | } 31 | } 32 | 33 | type codec struct { 34 | tipe reflect.Type 35 | asPtr bool 36 | any bool 37 | } 38 | 39 | func (*codec) Marshal(v interface{}) ([]byte, error) { 40 | return json.Marshal(v) 41 | } 42 | 43 | func (c *codec) Unmarshal(data []byte) (interface{}, error) { 44 | var v interface{} 45 | if !c.any { 46 | v = reflect.New(c.tipe).Interface() 47 | } 48 | 49 | err := json.Unmarshal(data, v) 50 | 51 | if !c.asPtr { 52 | v = reflect.ValueOf(v).Elem().Interface() 53 | } 54 | 55 | return v, err 56 | } 57 | 58 | func (*codec) NewEncoder(w io.Writer) margaret.Encoder { 59 | return json.NewEncoder(w) 60 | } 61 | 62 | func (c *codec) NewDecoder(r io.Reader) margaret.Decoder { 63 | return &decoder{ 64 | tipe: c.tipe, 65 | dec: json.NewDecoder(r), 66 | asPtr: c.asPtr, 67 | } 68 | } 69 | 70 | type decoder struct { 71 | tipe reflect.Type 72 | dec *json.Decoder 73 | asPtr bool 74 | } 75 | 76 | func (dec *decoder) Decode() (interface{}, error) { 77 | v := reflect.New(dec.tipe).Interface() 78 | err := dec.dec.Decode(v) 79 | 80 | if !dec.asPtr { 81 | v = reflect.ValueOf(v).Elem().Interface() 82 | } 83 | 84 | return v, err 85 | } 86 | -------------------------------------------------------------------------------- /offset2/offset.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package offset2 6 | 7 | import ( 8 | "encoding/binary" 9 | "fmt" 10 | "io" 11 | "os" 12 | 13 | "github.com/ssbc/margaret" 14 | ) 15 | 16 | type offset struct { 17 | *os.File 18 | } 19 | 20 | func (o *offset) readOffset(seq int64) (int64, error) { 21 | _, err := o.Seek(int64(seq)*8, io.SeekStart) 22 | if err != nil { 23 | return -1, fmt.Errorf("seek failed:%w", err) 24 | } 25 | 26 | var ofst int64 27 | err = binary.Read(o, binary.BigEndian, &ofst) 28 | if err != nil { 29 | return -1, fmt.Errorf("error reading offset %d: %w", seq, err) 30 | } 31 | return ofst, nil 32 | } 33 | 34 | func (o *offset) readLastOffset() (int64, int64, error) { 35 | stat, err := o.Stat() 36 | if err != nil { 37 | return 0, margaret.SeqEmpty, fmt.Errorf("stat failed:%w", err) 38 | } 39 | 40 | sz := stat.Size() 41 | if sz == 0 { 42 | return 0, margaret.SeqEmpty, nil 43 | } 44 | 45 | // this should be off-by-one-error-free: 46 | // sz is 8 when there is one entry, and the first entry has seq 0 47 | seqOfst := int64(sz/8 - 1) 48 | 49 | var ofstData int64 50 | err = binary.Read(io.NewSectionReader(o, sz-8, 8), binary.BigEndian, &ofstData) 51 | if err != nil { 52 | return 0, margaret.SeqEmpty, fmt.Errorf("error reading entry:%w", err) 53 | } 54 | 55 | return ofstData, seqOfst, nil 56 | } 57 | 58 | func (o *offset) append(ofst int64) (int64, error) { 59 | ofstOfst, err := o.Seek(0, io.SeekEnd) 60 | if err != nil { 61 | return margaret.SeqEmpty, fmt.Errorf("could not seek to end of file:%w", err) 62 | } 63 | seq := int64(ofstOfst / 8) 64 | 65 | err = binary.Write(o, binary.BigEndian, ofst) 66 | if err != nil { 67 | return margaret.SeqEmpty, fmt.Errorf("error writing offset:%w", err) 68 | } 69 | return seq, nil 70 | } 71 | -------------------------------------------------------------------------------- /multilog/roaring/badger/cmd/mbdump/main.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package main 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "os" 11 | 12 | "github.com/dgraph-io/badger/v3" 13 | "github.com/dgraph-io/sroar" 14 | "github.com/pkg/errors" 15 | pbadger "github.com/ssbc/margaret/internal/persist/badger" 16 | "go.mindeco.de/logging" 17 | ) 18 | 19 | var check = logging.CheckFatal 20 | 21 | func main() { 22 | if len(os.Args) < 2 { 23 | fmt.Fprintf(os.Stderr, "usage: %s (hasAddr)", os.Args[0]) 24 | os.Exit(1) 25 | } 26 | logging.SetupLogging(nil) 27 | // log := logging.Logger(os.Args[0]) 28 | 29 | dir := os.Args[1] 30 | 31 | opts := pbadger.BadgerOpts(dir) 32 | 33 | db, err := badger.Open(opts) 34 | check(errors.Wrap(err, "error opening database")) 35 | 36 | err = db.View(func(txn *badger.Txn) error { 37 | 38 | opts := badger.DefaultIteratorOptions 39 | iter := txn.NewIterator(opts) 40 | for iter.Rewind(); iter.Valid(); iter.Next() { 41 | it := iter.Item() 42 | k := it.Key() 43 | 44 | var dataLen int 45 | var debugData string 46 | err = it.Value(func(v []byte) error { 47 | dataLen = len(v) 48 | if bytes.HasPrefix(k, []byte("mlog-")) { 49 | bmap := sroar.FromBuffer(v) 50 | debugData = bmap.String() 51 | } else { 52 | debugData = string(v) 53 | } 54 | return nil 55 | }) 56 | check(err) 57 | 58 | fmt.Printf("%q: %d\n", string(k), dataLen) 59 | fmt.Println(debugData + "\n") 60 | 61 | } 62 | iter.Close() 63 | 64 | return nil 65 | }) 66 | check(err) 67 | 68 | check(db.Close()) 69 | // // check has 70 | // if len(os.Args) > 2 { 71 | // addr := indexes.Addr(os.Args[2]) 72 | // has, err := multilog.Has(mlog, addr) 73 | // log.Log("mlog", "has", "addr", addr, "has?", has, "hasErr", err) 74 | // } 75 | } 76 | -------------------------------------------------------------------------------- /internal/seqobsv/simple_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package seqobsv_test 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | 11 | "github.com/ssbc/margaret/internal/seqobsv" 12 | ) 13 | 14 | func ExampleInc() { 15 | sobs := seqobsv.New(0) 16 | fmt.Println(sobs.Value()) 17 | 18 | newV := sobs.Inc() 19 | fmt.Println(newV) 20 | 21 | fmt.Println(sobs.Inc()) 22 | 23 | // Output: 24 | // 0 25 | // 1 26 | // 2 27 | } 28 | 29 | func TestWaitSimple(t *testing.T) { 30 | 31 | sobs := seqobsv.New(0) 32 | 33 | if sobs.Value() != 0 { 34 | t.Fatal("start should be 0") 35 | } 36 | ch := sobs.WaitFor(4) 37 | 38 | go func() { 39 | for i := 0; i < 5; i++ { 40 | sobs.Inc() 41 | } 42 | }() 43 | 44 | <-ch 45 | 46 | if sobs.Value() < 4 { 47 | t.Fatal("should be 5 now") 48 | } 49 | } 50 | 51 | func TestWaitMultipleRead(t *testing.T) { 52 | 53 | sobs := seqobsv.New(0) 54 | 55 | if sobs.Value() != 0 { 56 | t.Fatal("start should be 0") 57 | } 58 | 59 | ch := sobs.WaitFor(200) 60 | 61 | go func() { 62 | for { 63 | select { 64 | case <-ch: 65 | break 66 | default: 67 | } 68 | sobs.Value() 69 | } 70 | }() 71 | 72 | go func() { 73 | for { 74 | select { 75 | case <-ch: 76 | break 77 | default: 78 | } 79 | sobs.Value() 80 | } 81 | }() 82 | 83 | go func() { 84 | for { 85 | select { 86 | case <-ch: 87 | break 88 | default: 89 | } 90 | sobs.Value() 91 | } 92 | }() 93 | 94 | go func() { 95 | for i := 0; i < 201; i++ { 96 | sobs.Inc() 97 | // time.Sleep(time.Second / 100) 98 | // t.Log(i) 99 | } 100 | }() 101 | 102 | <-ch 103 | 104 | if sobs.Value() < 200 { 105 | t.Fatal("should be 200 now") 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /codec/cbor/cbor.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package cbor 6 | 7 | import ( 8 | "bytes" 9 | "io" 10 | "reflect" 11 | 12 | "github.com/pkg/errors" 13 | "github.com/ssbc/margaret" 14 | ugorjiCodec "github.com/ugorji/go/codec" 15 | ) 16 | 17 | // New creates a msgpack codec 18 | // tipe is required because our Decode() interface doesn't take an argument 19 | func New(tipe interface{}) margaret.Codec { 20 | ch := ugorjiCodec.CborHandle{} 21 | // ch.Canonical = true 22 | ch.StructToArray = true 23 | 24 | c := &codec{ 25 | handle: &ch, 26 | } 27 | if tipe == nil { 28 | c.any = true 29 | return c 30 | } 31 | t := reflect.TypeOf(tipe) 32 | isPtr := t.Kind() == reflect.Ptr 33 | if isPtr { 34 | t = t.Elem() 35 | } 36 | c.tipe = t 37 | return c 38 | } 39 | 40 | type codec struct { 41 | tipe reflect.Type 42 | any bool 43 | handle *ugorjiCodec.CborHandle 44 | } 45 | 46 | func (c *codec) Marshal(v interface{}) ([]byte, error) { 47 | var buf bytes.Buffer 48 | enc := c.NewEncoder(&buf) 49 | err := enc.Encode(v) 50 | return buf.Bytes(), errors.Wrap(err, "cbor codec: encode failed") 51 | } 52 | 53 | func (c *codec) Unmarshal(data []byte) (interface{}, error) { 54 | dec := c.NewDecoder(bytes.NewReader(data)) 55 | return dec.Decode() 56 | } 57 | 58 | func (c *codec) NewEncoder(w io.Writer) margaret.Encoder { 59 | return ugorjiCodec.NewEncoder(w, c.handle) 60 | } 61 | 62 | func (c *codec) NewDecoder(r io.Reader) margaret.Decoder { 63 | dec := ugorjiCodec.NewDecoder(r, c.handle) 64 | return &decoder{tipe: c.tipe, dec: dec} 65 | } 66 | 67 | type decoder struct { 68 | tipe reflect.Type 69 | dec *ugorjiCodec.Decoder 70 | } 71 | 72 | func (dec *decoder) Decode() (interface{}, error) { 73 | v := reflect.New(dec.tipe).Interface() 74 | err := dec.dec.Decode(&v) 75 | return reflect.ValueOf(v).Elem().Interface(), err 76 | } 77 | -------------------------------------------------------------------------------- /indexes/index.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package indexes 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | 11 | "io" 12 | 13 | "github.com/ssbc/go-luigi" 14 | "github.com/ssbc/margaret" 15 | ) 16 | 17 | // Addr is an address (or key) in the index. 18 | // TODO maybe not use a string but a Stringer or 19 | type Addr string 20 | 21 | func (a Addr) String() string { 22 | return fmt.Sprintf("index-address:%q", string(a)) 23 | } 24 | 25 | // Index provides an index table keyed by Addr. 26 | // Often also implements Setter. 27 | type Index interface { 28 | // Get returns the an observable of the value stored at the address. 29 | // Getting an unset value retuns a valid Observable with a value 30 | // of type Unset and a nil error. 31 | Get(context.Context, Addr) (luigi.Observable, error) 32 | } 33 | 34 | // UnsetValue is the value of observable returned by idx.Get() when the 35 | // requested address has not been set yet. 36 | type UnsetValue struct { 37 | Addr Addr 38 | } 39 | 40 | type Setter interface { 41 | // Set sets a value in the index 42 | Set(context.Context, Addr, interface{}) error 43 | 44 | // Delete deletes a value from the index 45 | Delete(context.Context, Addr) error 46 | } 47 | 48 | // SetterIndex is an index that can be updated using calls to Set and Delete. 49 | type SetterIndex interface { 50 | Index 51 | Setter 52 | 53 | Flush() error 54 | } 55 | 56 | // SinkIndex is an index that is updated by processing a stream. 57 | type SinkIndex interface { 58 | luigi.Sink 59 | 60 | QuerySpec() margaret.QuerySpec 61 | } 62 | 63 | type SeqSetterIndex interface { 64 | SetterIndex 65 | 66 | SetSeq(int64) error 67 | GetSeq() (int64, error) 68 | 69 | io.Closer 70 | } 71 | 72 | // TODO maybe provide other index builders as well, e.g. for managing 73 | // sets: add and remove values from and to sets, stored at address 74 | -------------------------------------------------------------------------------- /internal/seqobsv/seqobsv.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | // Package seqobsv wants to supply an observable value sepcialized for sequence numbers in append-only logs. 6 | // It should be fine for access from multiple goroutines. 7 | // 8 | // These values only go up by one. For margaret they start with 0. 9 | // 10 | package seqobsv 11 | 12 | import ( 13 | "math" 14 | "sync" 15 | ) 16 | 17 | type Observable struct { 18 | mu sync.Mutex 19 | val uint64 20 | 21 | waiters waitMap 22 | } 23 | 24 | type emptyChan chan struct{} 25 | type waitMap map[uint64][]emptyChan 26 | 27 | // New creates a new Observable 28 | func New(start uint64) *Observable { 29 | return &Observable{ 30 | val: start, 31 | waiters: make(waitMap), 32 | } 33 | } 34 | 35 | // Value returns the current value 36 | func (seq *Observable) Value() uint64 { 37 | seq.mu.Lock() 38 | v := seq.val 39 | seq.mu.Unlock() 40 | return v 41 | } 42 | 43 | func (seq *Observable) Seq() int64 { 44 | seq.mu.Lock() 45 | v := seq.val 46 | seq.mu.Unlock() 47 | if v > math.MaxInt64 { 48 | panic("bigger then int64") 49 | } 50 | return int64(v) 51 | } 52 | 53 | func (seq *Observable) Inc() uint64 { 54 | seq.mu.Lock() 55 | curr := seq.val 56 | 57 | if waiters, has := seq.waiters[curr]; has { 58 | for _, ch := range waiters { 59 | close(ch) 60 | } 61 | delete(seq.waiters, curr) 62 | } 63 | seq.val = seq.val + 1 64 | currVal := seq.val 65 | seq.mu.Unlock() 66 | return currVal 67 | } 68 | 69 | func (seq *Observable) WaitFor(n uint64) <-chan struct{} { 70 | seq.mu.Lock() 71 | defer seq.mu.Unlock() 72 | ch := make(emptyChan) 73 | if n < seq.val { 74 | go func() { close(ch) }() 75 | return ch 76 | } 77 | 78 | waitersForN, has := seq.waiters[n] 79 | if !has { 80 | waitersForN = make([]emptyChan, 0) 81 | } 82 | 83 | seq.waiters[n] = append(waitersForN, ch) 84 | return ch 85 | } 86 | -------------------------------------------------------------------------------- /internal/persist/sqlite/sqlite.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package sqlite 6 | 7 | import ( 8 | "database/sql" 9 | "os" 10 | "path/filepath" 11 | 12 | "github.com/pkg/errors" 13 | "github.com/ssbc/margaret/internal/persist" 14 | ) 15 | 16 | type SqliteSaver struct { 17 | db *sql.DB 18 | } 19 | 20 | var _ persist.Saver = (*SqliteSaver)(nil) 21 | 22 | func (sl SqliteSaver) Close() error { 23 | return sl.db.Close() 24 | } 25 | 26 | func New(path string) (*SqliteSaver, error) { 27 | 28 | s, err := os.Stat(path) 29 | if os.IsNotExist(err) { 30 | if filepath.Dir(path) == "" { 31 | path = "." 32 | } 33 | err = os.MkdirAll(path, 0700) 34 | if err != nil { 35 | return nil, errors.Wrap(err, "failed to create path location") 36 | } 37 | s, err = os.Stat(path) 38 | if err != nil { 39 | return nil, errors.Wrap(err, "failed to stat created path location") 40 | } 41 | } else if err != nil { 42 | return nil, errors.Wrap(err, "failed to stat path location") 43 | } 44 | if s.IsDir() { 45 | path = filepath.Join(path, "log.db") 46 | } 47 | 48 | db, err := sql.Open("sqlite3", path) 49 | if err != nil { 50 | return nil, errors.Wrapf(err, "failed to open sqlite file: %s", path) 51 | } 52 | var version int 53 | err = db.QueryRow(`PRAGMA user_version`).Scan(&version) 54 | if err == sql.ErrNoRows || version == 0 { // new file or old schema 55 | 56 | if _, err := db.Exec(schemaVersion1); err != nil { 57 | return nil, errors.Wrap(err, "persist/sqlite: failed to init schema v1") 58 | } 59 | 60 | } else if err != nil { 61 | return nil, errors.Wrapf(err, "persist/sqlite: schema version lookup failed %s", path) 62 | } 63 | 64 | return &SqliteSaver{ 65 | db: db, 66 | }, nil 67 | } 68 | 69 | const schemaVersion1 = ` 70 | CREATE TABLE persisted_roaring ( 71 | key varchar PRIMARY KEY, 72 | data blob 73 | ); 74 | 75 | PRAGMA user_version = 1; 76 | ` 77 | -------------------------------------------------------------------------------- /codec/msgpack/msgpack.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package msgpack // import "github.com/ssbc/margaret/codec/msgpack" 6 | 7 | import ( 8 | "bytes" 9 | "io" 10 | "reflect" 11 | 12 | "github.com/ssbc/margaret" 13 | 14 | "github.com/pkg/errors" 15 | ugorjiCodec "github.com/ugorji/go/codec" 16 | ) 17 | 18 | // New creates a msgpack codec 19 | // tipe is required because our Decode() interface doesn't take an argument 20 | func New(tipe interface{}) margaret.Codec { 21 | ch := ugorjiCodec.MsgpackHandle{} 22 | ch.Canonical = true 23 | 24 | c := &codec{ 25 | handle: &ch, 26 | } 27 | if tipe == nil { 28 | c.any = true 29 | return c 30 | } 31 | t := reflect.TypeOf(tipe) 32 | isPtr := t.Kind() == reflect.Ptr 33 | if isPtr { 34 | t = t.Elem() 35 | } 36 | c.tipe = t 37 | return c 38 | } 39 | 40 | type codec struct { 41 | tipe reflect.Type 42 | any bool 43 | handle *ugorjiCodec.MsgpackHandle 44 | } 45 | 46 | func (c *codec) Marshal(v interface{}) ([]byte, error) { 47 | var buf bytes.Buffer 48 | enc := c.NewEncoder(&buf) 49 | err := enc.Encode(v) 50 | return buf.Bytes(), errors.Wrap(err, "msgpack codec: encode failed") 51 | } 52 | 53 | func (c *codec) Unmarshal(data []byte) (interface{}, error) { 54 | dec := c.NewDecoder(bytes.NewReader(data)) 55 | return dec.Decode() 56 | } 57 | 58 | func (c *codec) NewEncoder(w io.Writer) margaret.Encoder { 59 | return ugorjiCodec.NewEncoder(w, c.handle) 60 | } 61 | 62 | func (c *codec) NewDecoder(r io.Reader) margaret.Decoder { 63 | dec := ugorjiCodec.NewDecoder(r, c.handle) 64 | return &decoder{tipe: c.tipe, dec: dec} 65 | } 66 | 67 | type decoder struct { 68 | tipe reflect.Type 69 | dec *ugorjiCodec.Decoder 70 | } 71 | 72 | func (dec *decoder) Decode() (interface{}, error) { 73 | v := reflect.New(dec.tipe).Interface() 74 | err := dec.dec.Decode(&v) 75 | return reflect.ValueOf(v).Elem().Interface(), err 76 | } 77 | -------------------------------------------------------------------------------- /indexes/mapidx/map.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package mapidx // import "github.com/ssbc/margaret/indexes/mapidx" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "sync" 11 | 12 | "github.com/ssbc/go-luigi" 13 | "github.com/ssbc/margaret" 14 | "github.com/ssbc/margaret/indexes" 15 | ) 16 | 17 | // New returns a new map based index 18 | func New() indexes.SeqSetterIndex { 19 | return &mapSetterIndex{ 20 | m: make(map[indexes.Addr]luigi.Observable), 21 | curSeq: margaret.SeqEmpty, 22 | } 23 | } 24 | 25 | type mapSetterIndex struct { 26 | m map[indexes.Addr]luigi.Observable 27 | curSeq int64 28 | l sync.Mutex 29 | } 30 | 31 | func (idx *mapSetterIndex) Flush() error { return nil } 32 | func (idx *mapSetterIndex) Close() error { return nil } 33 | 34 | func (idx *mapSetterIndex) Get(_ context.Context, addr indexes.Addr) (luigi.Observable, error) { 35 | idx.l.Lock() 36 | defer idx.l.Unlock() 37 | 38 | obv, ok := idx.m[addr] 39 | if ok { 40 | return obv, nil 41 | } 42 | 43 | obv = luigi.NewObservable(indexes.UnsetValue{Addr: addr}) 44 | idx.m[addr] = obv 45 | 46 | return obv, nil 47 | } 48 | 49 | func (idx *mapSetterIndex) Set(_ context.Context, addr indexes.Addr, v interface{}) error { 50 | idx.l.Lock() 51 | defer idx.l.Unlock() 52 | 53 | obv, ok := idx.m[addr] 54 | if ok { 55 | err := obv.Set(v) 56 | if err != nil { 57 | return fmt.Errorf("error setting observable: %w", err) 58 | } 59 | return nil 60 | } 61 | 62 | obv = luigi.NewObservable(v) 63 | idx.m[addr] = obv 64 | 65 | return nil 66 | } 67 | 68 | func (idx *mapSetterIndex) Delete(_ context.Context, addr indexes.Addr) error { 69 | idx.l.Lock() 70 | defer idx.l.Unlock() 71 | 72 | obv, ok := idx.m[addr] 73 | if ok { 74 | err := obv.Set(indexes.UnsetValue{Addr: addr}) 75 | if err != nil { 76 | return fmt.Errorf("error setting observable: %w", err) 77 | } 78 | } 79 | 80 | return nil 81 | } 82 | 83 | func (idx *mapSetterIndex) GetSeq() (int64, error) { 84 | idx.l.Lock() 85 | defer idx.l.Unlock() 86 | 87 | return idx.curSeq, nil 88 | } 89 | 90 | func (idx *mapSetterIndex) SetSeq(seq int64) error { 91 | idx.l.Lock() 92 | defer idx.l.Unlock() 93 | 94 | idx.curSeq = seq 95 | 96 | return nil 97 | } 98 | -------------------------------------------------------------------------------- /indexes/badger/test/test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "crypto/rand" 9 | "encoding/hex" 10 | "fmt" 11 | "os" 12 | "path/filepath" 13 | "sync" 14 | 15 | "github.com/dgraph-io/badger/v3" 16 | 17 | "github.com/ssbc/margaret/indexes" 18 | libadger "github.com/ssbc/margaret/indexes/badger" 19 | "github.com/ssbc/margaret/indexes/test" 20 | pbadger "github.com/ssbc/margaret/internal/persist/badger" 21 | ) 22 | 23 | func init() { 24 | newStandaloneSeqSetterIdx := func(name string, tipe interface{}) (indexes.SeqSetterIndex, error) { 25 | dir := filepath.Join("testrun", name) 26 | os.RemoveAll(dir) 27 | os.MkdirAll(dir, 0700) 28 | 29 | opts := pbadger.BadgerOpts(dir) 30 | 31 | db, err := badger.Open(opts) 32 | if err != nil { 33 | return nil, fmt.Errorf("error opening test database (%s): %w", dir, err) 34 | } 35 | 36 | return libadger.NewIndex(db, tipe), nil 37 | } 38 | 39 | var ( 40 | initDB sync.Once 41 | sharedDB *badger.DB 42 | ) 43 | 44 | newSharedSeqSetterIdx := func(name string, tipe interface{}) (indexes.SeqSetterIndex, error) { 45 | 46 | initDB.Do(func() { 47 | dir := filepath.Join("testrun", "badger-shared") 48 | os.RemoveAll(dir) 49 | os.MkdirAll(dir, 0700) 50 | 51 | opts := pbadger.BadgerOpts(dir) 52 | 53 | var err error 54 | sharedDB, err = badger.Open(opts) 55 | if err != nil { 56 | panic(fmt.Errorf("error opening test database (%s): %w", dir, err)) 57 | } 58 | }) 59 | 60 | keyPrefix := make([]byte, 16) 61 | rand.Read(keyPrefix) 62 | 63 | return libadger.NewIndexWithKeyPrefix(sharedDB, tipe, []byte(hex.EncodeToString(keyPrefix))), nil 64 | } 65 | 66 | toSetterIdx := func(f test.NewSeqSetterIndexFunc) test.NewSetterIndexFunc { 67 | return func(name string, tipe interface{}) (indexes.SetterIndex, error) { 68 | idx, err := f(name, tipe) 69 | return idx, err 70 | } 71 | } 72 | 73 | test.RegisterSeqSetterIndex("badger-standalone", newStandaloneSeqSetterIdx) 74 | test.RegisterSetterIndex("badger-standalone", toSetterIdx(newStandaloneSeqSetterIdx)) 75 | 76 | test.RegisterSeqSetterIndex("badger-shared", newSharedSeqSetterIdx) 77 | test.RegisterSetterIndex("badger-shared", toSetterIdx(newSharedSeqSetterIdx)) 78 | } 79 | -------------------------------------------------------------------------------- /offset2/data.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package offset2 6 | 7 | import ( 8 | "bytes" 9 | "encoding/binary" 10 | "fmt" 11 | "io" 12 | "os" 13 | 14 | "github.com/ssbc/margaret" 15 | ) 16 | 17 | type data struct { 18 | *os.File 19 | 20 | buf [8]byte 21 | } 22 | 23 | func (d *data) frameReader(ofst int64) (io.Reader, error) { 24 | var sz int64 25 | err := binary.Read(io.NewSectionReader(d, ofst, 8), binary.BigEndian, &sz) 26 | if err != nil { 27 | return nil, fmt.Errorf("error reading payload length: %w", err) 28 | } 29 | 30 | if sz < 0 { 31 | return nil, margaret.ErrNulled 32 | } 33 | 34 | return io.NewSectionReader(d, ofst+8, sz), nil 35 | } 36 | 37 | func (d *data) readFrame(data []byte, ofst int64) (int, error) { 38 | sr := io.NewSectionReader(d, ofst, 8) 39 | 40 | var sz int64 41 | err := binary.Read(sr, binary.BigEndian, &sz) 42 | if err != nil { 43 | return 0, fmt.Errorf("error reading payload length: %w", err) 44 | } 45 | 46 | return d.ReadAt(data, ofst+8) 47 | } 48 | 49 | func (d *data) getFrameSize(ofst int64) (int64, error) { 50 | _, err := d.ReadAt(d.buf[:], ofst) 51 | if err != nil { 52 | return -1, fmt.Errorf("error reading payload length: %w", err) 53 | } 54 | 55 | buf := bytes.NewBuffer(d.buf[:]) 56 | 57 | var sz int64 58 | err = binary.Read(buf, binary.BigEndian, &sz) 59 | if err != nil { 60 | return -1, fmt.Errorf("error parsing payload length: %w", err) 61 | } 62 | 63 | return sz, nil 64 | } 65 | 66 | func (d *data) getFrame(ofst int64) ([]byte, error) { 67 | sz, err := d.getFrameSize(ofst) 68 | if err != nil { 69 | return nil, fmt.Errorf("error getting frame size: %w", err) 70 | } 71 | 72 | data := make([]byte, sz) 73 | _, err = d.readFrame(data, ofst) 74 | if err != nil { 75 | return nil, fmt.Errorf("error reading frame: %w", err) 76 | } 77 | return data, nil 78 | } 79 | 80 | func (d *data) append(data []byte) (int64, error) { 81 | ofst, err := d.Seek(0, io.SeekEnd) 82 | if err != nil { 83 | return -1, fmt.Errorf("failed to seek to end of file: %w", err) 84 | } 85 | 86 | err = binary.Write(d, binary.BigEndian, int64(len(data))) 87 | if err != nil { 88 | return -1, fmt.Errorf("writing length prefix failed: %w", err) 89 | } 90 | 91 | _, err = d.Write(data) 92 | if err != nil { 93 | return -1, fmt.Errorf("error writing data: %w", err) 94 | } 95 | return ofst, nil 96 | } 97 | -------------------------------------------------------------------------------- /multilog/roaring/test/test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "io/ioutil" 9 | "os" 10 | "path/filepath" 11 | 12 | _ "github.com/mattn/go-sqlite3" 13 | "github.com/pkg/errors" 14 | 15 | "github.com/ssbc/margaret/internal/persist/fs" 16 | "github.com/ssbc/margaret/multilog" 17 | "github.com/ssbc/margaret/multilog/roaring" 18 | "github.com/ssbc/margaret/multilog/roaring/badger" 19 | "github.com/ssbc/margaret/multilog/roaring/mkv" 20 | "github.com/ssbc/margaret/multilog/roaring/sqlite" 21 | mltest "github.com/ssbc/margaret/multilog/test" 22 | ) 23 | 24 | func init() { 25 | mltest.Register("roaring_badger", func(name string, tipe interface{}, testDir string) (multilog.MultiLog, string, error) { 26 | if testDir == "" { 27 | var err error 28 | testDir, err = ioutil.TempDir("", "roarbadger") 29 | if err != nil { 30 | return nil, "", errors.Wrap(err, "error creating tempdir") 31 | } 32 | } 33 | 34 | badgerMl, err := badger.NewStandalone(testDir) 35 | return badgerMl, testDir, err 36 | }) 37 | 38 | mltest.Register("roaring_files", func(name string, tipe interface{}, testDir string) (multilog.MultiLog, string, error) { 39 | if testDir == "" { 40 | var err error 41 | testDir, err = ioutil.TempDir("", "roarfiles") 42 | if err != nil { 43 | return nil, "", errors.Wrap(err, "error creating tempdir") 44 | } 45 | } 46 | 47 | return roaring.NewStore(fs.New(testDir)), testDir, nil 48 | }) 49 | 50 | mltest.Register("roaring_sqlite", func(name string, tipe interface{}, testDir string) (multilog.MultiLog, string, error) { 51 | if testDir == "" { 52 | var err error 53 | testDir, err = ioutil.TempDir("", "roarsqlite") 54 | if err != nil { 55 | return nil, "", errors.Wrap(err, "error creating tempdir") 56 | } 57 | } 58 | r, err := sqlite.NewMultiLog(testDir) 59 | return r, testDir, err 60 | }) 61 | 62 | mltest.Register("roaring_mkv", func(name string, tipe interface{}, testDir string) (multilog.MultiLog, string, error) { 63 | if testDir == "" { 64 | var err error 65 | testDir, err = ioutil.TempDir("", "roar_mkv") 66 | if err != nil { 67 | return nil, "", errors.Wrap(err, "error creating tempdir") 68 | } 69 | os.MkdirAll(testDir, 0700) 70 | } 71 | r, err := mkv.NewMultiLog(filepath.Join(testDir, "mkv.roar")) 72 | return r, testDir, err 73 | }) 74 | } 75 | -------------------------------------------------------------------------------- /internal/persist/sqlite/saver.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package sqlite 6 | 7 | import ( 8 | "database/sql" 9 | "encoding/hex" 10 | "fmt" 11 | 12 | "github.com/pkg/errors" 13 | "github.com/ssbc/margaret/internal/persist" 14 | ) 15 | 16 | func (s SqliteSaver) Put(key persist.Key, data []byte) error { 17 | hexKey := hex.EncodeToString(key) 18 | _, err := s.db.Exec(`insert or replace into persisted_roaring (key,data) VALUES(?,?)`, hexKey, data) 19 | if err != nil { 20 | return errors.Wrap(err, "sqlite/put: failed run delete/insert value") 21 | } 22 | return nil 23 | } 24 | 25 | func (s SqliteSaver) PutMultiple(values []persist.KeyValuePair) error { 26 | for i, kv := range values { 27 | err := s.Put(kv.Key, kv.Value) 28 | if err != nil { 29 | return fmt.Errorf("persist/seqlite: failed to put entry %d of %d (%s): %w", i, len(values), kv.Key, err) 30 | } 31 | } 32 | return nil 33 | } 34 | 35 | func (s SqliteSaver) Get(key persist.Key) ([]byte, error) { 36 | 37 | var data []byte 38 | hexKey := hex.EncodeToString(key) 39 | err := s.db.QueryRow(`SELECT data from persisted_roaring where key = ?`, hexKey).Scan(&data) 40 | if err != nil { 41 | if err == sql.ErrNoRows { 42 | return nil, persist.ErrNotFound 43 | } 44 | return nil, errors.Wrapf(err, "persist/sqlite/get(%s): failed to execute query", hexKey[:5]) 45 | } 46 | return data, nil 47 | } 48 | 49 | func (s SqliteSaver) List() ([]persist.Key, error) { 50 | var keys []persist.Key 51 | rows, err := s.db.Query(`SELECT key from persisted_roaring`) 52 | if err != nil { 53 | return nil, errors.Wrap(err, "persist/sqlite/list: failed to execute rows query") 54 | } 55 | defer rows.Close() 56 | 57 | for rows.Next() { 58 | var k string 59 | err := rows.Scan(&k) 60 | if err != nil { 61 | return nil, errors.Wrap(err, "persist/sqlite/list: failed to scan row result") 62 | } 63 | bk, err := hex.DecodeString(k) 64 | if err != nil { 65 | return nil, errors.Wrapf(err, "persist/sqlite/list: invalid key: %q", k) 66 | } 67 | keys = append(keys, bk) 68 | } 69 | 70 | return keys, rows.Err() 71 | } 72 | 73 | func (s SqliteSaver) Delete(k persist.Key) error { 74 | hexKey := hex.EncodeToString(k) 75 | _, err := s.db.Exec(`DELETE FROM persisted_roaring WHERE key = ?`, hexKey) 76 | if err != nil { 77 | return errors.Wrapf(err, "sqlite/delete: failed run delete key %q", hexKey) 78 | } 79 | return nil 80 | } 81 | -------------------------------------------------------------------------------- /multilog/sink.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package multilog 6 | 7 | import ( 8 | "context" 9 | "io" 10 | "os" 11 | "sync" 12 | 13 | "github.com/keks/persist" 14 | "github.com/pkg/errors" 15 | "github.com/ssbc/go-luigi" 16 | "github.com/ssbc/margaret" 17 | ) 18 | 19 | // Func is a processing function that consumes a stream and sets values in the multilog. 20 | type Func func(ctx context.Context, seq int64, value interface{}, mlog MultiLog) error 21 | 22 | // Sink is both a multilog and a luigi sink. Pouring values into it will append values to the multilog, usually by calling a user-defined processing function. 23 | type Sink interface { 24 | luigi.Sink 25 | QuerySpec() margaret.QuerySpec 26 | } 27 | 28 | // NewSink makes a new Sink by wrapping a MultiLog and a processing function of type Func. 29 | func NewSink(file *os.File, mlog MultiLog, f Func) Sink { 30 | return &sinkLog{ 31 | mlog: mlog, 32 | f: f, 33 | file: file, 34 | l: &sync.Mutex{}, 35 | } 36 | } 37 | 38 | type sinkLog struct { 39 | mlog MultiLog 40 | f Func 41 | file *os.File 42 | l *sync.Mutex 43 | } 44 | 45 | // Pour calls the processing function to add a value to a sublog. 46 | func (slog *sinkLog) Pour(ctx context.Context, v interface{}) error { 47 | slog.l.Lock() 48 | defer slog.l.Unlock() 49 | 50 | seq := v.(margaret.SeqWrapper) 51 | err := persist.Save(slog.file, seq.Seq()) 52 | if err != nil { 53 | return errors.Wrap(err, "error saving current sequence number") 54 | } 55 | 56 | err = slog.f(ctx, seq.Seq(), seq.Value(), slog.mlog) 57 | return errors.Wrap(err, "multilog/sink: error in processing function") 58 | } 59 | 60 | // Close does nothing. 61 | func (slog *sinkLog) Close() error { return nil } 62 | 63 | // QuerySpec returns the query spec that queries the next needed messages from the log 64 | func (slog *sinkLog) QuerySpec() margaret.QuerySpec { 65 | slog.l.Lock() 66 | defer slog.l.Unlock() 67 | 68 | var seq int64 69 | 70 | if err := persist.Load(slog.file, &seq); err != nil { 71 | if errors.Cause(err) != io.EOF { 72 | return margaret.ErrorQuerySpec(err) 73 | } 74 | 75 | seq = margaret.SeqEmpty 76 | } 77 | 78 | return margaret.MergeQuerySpec( 79 | margaret.Gt(seq), 80 | margaret.SeqWrap(true), 81 | ) 82 | } 83 | 84 | type roLog struct { 85 | margaret.Log 86 | } 87 | 88 | // Append always returns an error that indicates that this log is read only. 89 | func (roLog) Append(v interface{}) (int64, error) { 90 | return margaret.SeqEmpty, errors.New("can't append to read-only log") 91 | } 92 | -------------------------------------------------------------------------------- /test/concurrent.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test // import "github.com/ssbc/margaret/test" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "os" 11 | "sync" 12 | "testing" 13 | 14 | "github.com/stretchr/testify/assert" 15 | "github.com/stretchr/testify/require" 16 | 17 | "github.com/ssbc/margaret" 18 | ) 19 | 20 | func LogTestConcurrent(f NewLogFunc) func(*testing.T) { 21 | type testcase struct { 22 | tipe interface{} 23 | values []interface{} 24 | specs []margaret.QuerySpec 25 | result []interface{} 26 | } 27 | 28 | mkTest := func(tc testcase) func(*testing.T) { 29 | return func(t *testing.T) { 30 | a := assert.New(t) 31 | r := require.New(t) 32 | 33 | log, err := f(t.Name(), tc.tipe) 34 | r.NoError(err, "error creating log") 35 | r.NotNil(log, "returned log is nil") 36 | 37 | defer func() { 38 | if namer, ok := log.(interface{ FileName() string }); ok { 39 | r.NoError(os.RemoveAll(namer.FileName()), "error deleting log after test") 40 | } 41 | }() 42 | 43 | ctx, cancel := context.WithCancel(context.Background()) 44 | defer cancel() 45 | 46 | seq := log.Seq() 47 | a.NoError(err, "unexpected error") 48 | a.EqualValues(margaret.SeqEmpty, seq, "expected empty log") 49 | 50 | var wg sync.WaitGroup 51 | wg.Add(2) 52 | go func() { 53 | defer wg.Done() 54 | 55 | src, err := log.Query(tc.specs...) 56 | a.NoError(err, "error querying log") 57 | 58 | for i, exp := range tc.result { 59 | v, err := src.Next(ctx) 60 | a.NoError(err, "error in call to Next()") 61 | a.Equal(exp, v, "result doesn't match") 62 | 63 | if t.Failed() { 64 | t.Log("error in iteration", i) 65 | } 66 | } 67 | }() 68 | 69 | go func() { 70 | defer wg.Done() 71 | 72 | for i, v := range tc.values { 73 | seq, err := log.Append(v) 74 | a.NoError(err, "error appending to log") 75 | a.EqualValues(i, seq, "sequence missmatch") 76 | 77 | if t.Failed() { 78 | t.Log("error in iteration", i) 79 | } 80 | } 81 | }() 82 | 83 | wg.Wait() 84 | } 85 | } 86 | 87 | tcs := []testcase{ 88 | { 89 | tipe: 0, 90 | values: []interface{}{1, 2, 3}, 91 | result: []interface{}{1, 2, 3}, 92 | specs: []margaret.QuerySpec{margaret.Live(true)}, 93 | }, 94 | { 95 | tipe: 0, 96 | values: []interface{}{1, 2, 3}, 97 | result: []interface{}{1, 2}, 98 | specs: []margaret.QuerySpec{margaret.Live(true), margaret.Limit(2)}, 99 | }, 100 | } 101 | 102 | return func(t_ *testing.T) { 103 | for i, tc := range tcs { 104 | t_.Run(fmt.Sprint(i), mkTest(tc)) 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /internal/persist/fs/fs.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package fs 6 | 7 | import ( 8 | "encoding/hex" 9 | "fmt" 10 | "io/ioutil" 11 | "os" 12 | "path/filepath" 13 | "strings" 14 | 15 | "github.com/pkg/errors" 16 | "github.com/ssbc/margaret/internal/persist" 17 | ) 18 | 19 | type Saver struct { 20 | base string 21 | } 22 | 23 | var _ persist.Saver = (*Saver)(nil) 24 | 25 | func New(base string) *Saver { 26 | os.MkdirAll(base, 0700) 27 | return &Saver{base: base} 28 | } 29 | 30 | func (s Saver) Close() error { return nil } 31 | 32 | func (s Saver) fnameForKey(k []byte) string { 33 | var fname string 34 | hexKey := hex.EncodeToString(k) 35 | if len(hexKey) > 10 { 36 | fname = filepath.Join(s.base, hexKey[:5], hexKey[5:]) 37 | os.MkdirAll(filepath.Dir(fname), 0700) 38 | } else { 39 | fname = filepath.Join(s.base, hexKey) 40 | } 41 | return fname 42 | } 43 | 44 | func (s Saver) Put(key persist.Key, data []byte) error { 45 | err := ioutil.WriteFile(s.fnameForKey(key), data, 0700) 46 | if err != nil { 47 | return errors.Wrap(err, "roaringfiles: file write failed") 48 | } 49 | return nil 50 | } 51 | 52 | func (s Saver) PutMultiple(values []persist.KeyValuePair) error { 53 | for i, kv := range values { 54 | err := s.Put(kv.Key, kv.Value) 55 | if err != nil { 56 | return fmt.Errorf("roaringfiles/putMultiple: failed to set entry %d (%s): %w", i, kv.Key, err) 57 | } 58 | } 59 | return nil 60 | } 61 | 62 | func (s Saver) Get(key persist.Key) ([]byte, error) { 63 | d, err := ioutil.ReadFile(s.fnameForKey(key)) 64 | if err != nil { 65 | if os.IsNotExist(err) { 66 | return nil, persist.ErrNotFound 67 | } 68 | return nil, errors.Wrap(err, "persist/fs: error in read transaction") 69 | } 70 | return d, nil 71 | } 72 | 73 | func (s Saver) List() ([]persist.Key, error) { 74 | var list []persist.Key 75 | 76 | err := filepath.Walk(s.base, func(path string, info os.FileInfo, err error) error { 77 | if err != nil { 78 | return err 79 | } 80 | if info.IsDir() { 81 | return nil 82 | } 83 | 84 | name := strings.TrimPrefix(path, s.base+"/") 85 | if name[5] == '/' { 86 | var b = []byte(name) 87 | b = append(b[:5], b[6:]...) 88 | name = string(b) 89 | } 90 | bk, err := hex.DecodeString(name) 91 | if err != nil { 92 | return errors.Wrap(err, "roaringfiles: invalid path") 93 | } 94 | 95 | list = append(list, bk) 96 | return nil 97 | }) 98 | if err != nil { 99 | return nil, errors.Wrap(err, "persist/fs: walk iteration failed") 100 | } 101 | return list, nil 102 | } 103 | 104 | func (s Saver) Delete(k persist.Key) error { 105 | fname := s.fnameForKey(k) 106 | err := os.Remove(fname) 107 | if err != nil && !os.IsNotExist(err) { 108 | return err 109 | } 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /multilog/test/multilog_live.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "context" 9 | "testing" 10 | "time" 11 | 12 | "github.com/pkg/errors" 13 | 14 | "github.com/ssbc/go-luigi" 15 | "github.com/ssbc/margaret" 16 | "github.com/ssbc/margaret/indexes" 17 | "github.com/stretchr/testify/assert" 18 | "github.com/stretchr/testify/require" 19 | ) 20 | 21 | func MultilogLiveQueryCheck(f NewLogFunc) func(*testing.T) { 22 | return func(t *testing.T) { 23 | a := assert.New(t) 24 | r := require.New(t) 25 | ctx, cancel := context.WithCancel(context.TODO()) 26 | 27 | mlog, _, err := f(t.Name(), int64(0), "") 28 | r.NoError(err) 29 | 30 | // empty yet 31 | addrs, err := mlog.List() 32 | r.NoError(err, "error listing mlog") 33 | r.Len(addrs, 0) 34 | 35 | testLogs := map[indexes.Addr][]int64{ 36 | "fii": {1, 2, 3}, 37 | "faa": {100, 200, 300}, 38 | "foo": {4, 5, 6}, 39 | "fum": {7, 8, 9}, 40 | } 41 | 42 | // fill in some values 43 | for name, vals := range testLogs { 44 | slog, err := mlog.Get(name) 45 | r.NoError(err) 46 | 47 | for i, v := range vals { 48 | _, err := slog.Append(v) 49 | r.NoError(err, "valied to append %s:%d", name, i) 50 | } 51 | 52 | r.EqualValues(slog.Seq(), len(vals)-1) 53 | } 54 | 55 | logOfFaa, err := mlog.Get(indexes.Addr("faa")) 56 | r.NoError(err) 57 | 58 | // produce new values in the background 59 | go func() { 60 | time.Sleep(time.Second / 10) 61 | slog, err := mlog.Get(indexes.Addr("faa")) 62 | if err != nil { 63 | panic(err) 64 | } 65 | for tv := 400; tv < 2000; tv += 100 { 66 | appendedSeq, err := slog.Append(tv) 67 | if err != nil { 68 | panic(err) 69 | } 70 | t.Log(tv, " inserted as:", appendedSeq) 71 | // !!!! handbrake to reduce chan send shedule madness 72 | time.Sleep(time.Second / 10) 73 | // !!!!! 74 | } 75 | time.Sleep(time.Second / 2) 76 | cancel() 77 | }() 78 | 79 | seqSrc, err := logOfFaa.Query( 80 | margaret.Gt(2), 81 | margaret.Live(true), 82 | margaret.SeqWrap(true), 83 | ) 84 | r.NoError(err) 85 | 86 | var expSeq = 3 87 | var expVal = 400 88 | for { 89 | swv, err := seqSrc.Next(ctx) 90 | if err != nil { 91 | if luigi.IsEOS(err) || errors.Is(err, context.Canceled) { 92 | t.Log("canceled", err, swv) 93 | a.Equal(expSeq, 19) 94 | break 95 | } 96 | r.NoError(err) 97 | } 98 | sw := swv.(margaret.SeqWrapper) 99 | 100 | gotVal := sw.Value().(int64) 101 | 102 | a.EqualValues(expVal, gotVal, "wrong actual val") 103 | expVal += 100 104 | 105 | a.EqualValues(expSeq, sw.Seq(), "wrong seq value from query") 106 | t.Log(expSeq, sw.Seq()) 107 | expSeq++ 108 | } 109 | 110 | r.NoError(mlog.Close()) 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /internal/persist/badger/shared_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package badger 6 | 7 | import ( 8 | "bytes" 9 | "crypto/rand" 10 | "os" 11 | "path/filepath" 12 | "testing" 13 | 14 | "github.com/dgraph-io/badger/v3" 15 | "github.com/ssbc/margaret/internal/persist" 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | func TestSharedBadger(t *testing.T) { 20 | r := require.New(t) 21 | 22 | path := filepath.Join("testrun", t.Name()) 23 | os.RemoveAll(path) 24 | os.Mkdir(path, 0700) 25 | 26 | o := BadgerOpts(path) 27 | db, err := badger.Open(o) 28 | r.NoError(err) 29 | 30 | // make sure each bucket can use keys as if they are alone 31 | collidingKey := persist.Key("meins") 32 | 33 | // create two shared instances on the same backing db 34 | fooBucket, err := NewShared(db, []byte("foo")) 35 | r.NoError(err) 36 | 37 | barBucket, err := NewShared(db, []byte("bar")) 38 | r.NoError(err) 39 | 40 | // write two chunks of random data to them 41 | fooData := make([]byte, 32) 42 | rand.Read(fooData) 43 | err = fooBucket.Put(collidingKey, fooData) 44 | r.NoError(err) 45 | 46 | barData := make([]byte, 32) 47 | rand.Read(barData) 48 | err = barBucket.Put(collidingKey, barData) 49 | r.NoError(err) 50 | 51 | // should both have just one key 52 | fooKeys, err := fooBucket.List() 53 | r.NoError(err) 54 | r.Len(fooKeys, 1) 55 | r.Equal(collidingKey, fooKeys[0]) 56 | 57 | barKeys, err := barBucket.List() 58 | r.NoError(err) 59 | r.Len(barKeys, 1) 60 | r.Equal(collidingKey, barKeys[0]) 61 | 62 | // make sure they didnt overwrite each other 63 | fooGot, err := fooBucket.Get(collidingKey) 64 | r.NoError(err) 65 | r.Equal(fooData, fooGot) 66 | 67 | barGot, err := barBucket.Get(collidingKey) 68 | r.NoError(err) 69 | r.Equal(barData, barGot) 70 | 71 | // closing a shared should be a noop 72 | r.NoError(fooBucket.Close()) 73 | r.NoError(barBucket.Close()) 74 | 75 | r.False(db.IsClosed()) 76 | r.NoError(db.Close()) 77 | 78 | // reopen 79 | db, err = badger.Open(o) 80 | r.NoError(err) 81 | 82 | // create two shared instances on the same backing db 83 | fooBucket, err = NewShared(db, []byte("foo")) 84 | r.NoError(err) 85 | 86 | fooKeys, err = fooBucket.List() 87 | r.NoError(err) 88 | r.Len(fooKeys, 1) 89 | r.Equal(collidingKey, fooKeys[0]) 90 | 91 | // manual lookup 92 | var hasFoo, hasBar bool 93 | db.View(func(txn *badger.Txn) error { 94 | iter := txn.NewIterator(badger.DefaultIteratorOptions) 95 | defer iter.Close() 96 | 97 | for iter.Rewind(); iter.Valid(); iter.Next() { 98 | it := iter.Item() 99 | 100 | k := it.Key() 101 | 102 | if bytes.Equal(k, []byte("foomeins")) { 103 | hasFoo = true 104 | } 105 | if bytes.Equal(k, []byte("barmeins")) { 106 | hasBar = true 107 | } 108 | } 109 | return nil 110 | }) 111 | 112 | r.True(hasFoo, "foo not found") 113 | r.True(hasBar, "bar not found") 114 | } 115 | -------------------------------------------------------------------------------- /indexes/test/setidx.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "context" 9 | "testing" 10 | 11 | "github.com/ssbc/go-luigi" 12 | "github.com/ssbc/margaret/indexes" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | type NewSetterIndexFunc func(name string, tipe interface{}) (indexes.SetterIndex, error) 17 | 18 | func TestSetterIndex(newIdx NewSetterIndexFunc) func(*testing.T) { 19 | return func(t *testing.T) { 20 | t.Run("Sequential", TestSetterIndexSequential(newIdx)) 21 | t.Run("Observable", TestSetterIndexObservable(newIdx)) 22 | } 23 | } 24 | 25 | func TestSetterIndexSequential(newIdx NewSetterIndexFunc) func(*testing.T) { 26 | return func(t *testing.T) { 27 | ctx := context.Background() 28 | r := require.New(t) 29 | 30 | idx, err := newIdx(t.Name(), "str") 31 | r.NoError(err, "error creating index") 32 | 33 | err = idx.Set(ctx, "test", "omg what is this") 34 | r.NoError(err, "error setting value") 35 | 36 | obv, err := idx.Get(ctx, "test") 37 | r.NoError(err, "error getting observable") 38 | 39 | v, err := obv.Value() 40 | r.NoError(err, "error getting value") 41 | 42 | if v != "omg what is this" { 43 | t.Errorf("expected %q but got %q (type: %T)", "omg what is this", v, v) 44 | } 45 | } 46 | } 47 | 48 | func TestSetterIndexObservable(newIdx NewSetterIndexFunc) func(*testing.T) { 49 | return func(t *testing.T) { 50 | ctx := context.Background() 51 | r := require.New(t) 52 | 53 | idx, err := newIdx(t.Name(), "str") 54 | r.NoError(err, "error creating index") 55 | 56 | obv, err := idx.Get(ctx, "test") 57 | r.NoError(err, "error getting observable") 58 | 59 | var i int 60 | first := make(chan struct{}) 61 | closed := make(chan struct{}) 62 | rxExp := []interface{}{ 63 | indexes.UnsetValue{"test"}, 64 | "omg what is this", 65 | "so rad", 66 | "wowzers", 67 | indexes.UnsetValue{"test"}, 68 | } 69 | 70 | var cancel func() 71 | cancel = obv.Register(luigi.FuncSink(func(ctx context.Context, v interface{}, err error) error { 72 | if i == 0 { 73 | close(first) 74 | } 75 | 76 | if i == len(rxExp)-1 { 77 | t.Log("got all messages, canceling registration") 78 | defer cancel() 79 | } 80 | 81 | defer func() { i++ }() 82 | 83 | if err != nil { 84 | if err != (luigi.EOS{}) { 85 | t.Log("sink closed with non-EOS error:", err) 86 | } 87 | 88 | if i == len(rxExp) { 89 | close(closed) 90 | } else { 91 | t.Errorf("unexpected close: i=%d", i) 92 | } 93 | 94 | return nil 95 | } 96 | 97 | if i > len(rxExp)-1 { 98 | return nil 99 | } 100 | 101 | if v != rxExp[i] { 102 | t.Errorf("expecting %q, but got %q", rxExp[i], v) 103 | } 104 | 105 | return nil 106 | })) 107 | 108 | <-first 109 | 110 | exp := []string{ 111 | "omg what is this", 112 | "so rad", 113 | "wowzers", 114 | } 115 | 116 | for _, v := range exp { 117 | err = idx.Set(ctx, "test", v) 118 | if err != nil { 119 | t.Errorf("error setting value to %q: %s", v, err) 120 | } 121 | } 122 | 123 | err = idx.Delete(ctx, "test") 124 | r.NoError(err, "error deleting value") 125 | 126 | <-closed 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /multilog/roaring/sublog.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package roaring 6 | 7 | import ( 8 | "fmt" 9 | 10 | "github.com/dgraph-io/sroar" 11 | "github.com/ssbc/go-luigi" 12 | 13 | "github.com/ssbc/margaret" 14 | "github.com/ssbc/margaret/internal/persist" 15 | "github.com/ssbc/margaret/internal/seqobsv" 16 | "github.com/ssbc/margaret/multilog" 17 | ) 18 | 19 | type sublog struct { 20 | mlog *MultiLog 21 | 22 | key persist.Key 23 | seq *seqobsv.Observable 24 | luigiObsv luigi.Observable 25 | bmap *sroar.Bitmap 26 | 27 | dirty bool 28 | 29 | deleted bool 30 | } 31 | 32 | func (log *sublog) Seq() int64 { 33 | return log.seq.Seq() - 1 34 | } 35 | 36 | func (log *sublog) Changes() luigi.Observable { 37 | return log.luigiObsv 38 | } 39 | 40 | func (log *sublog) Get(seq int64) (interface{}, error) { 41 | log.mlog.l.Lock() 42 | defer log.mlog.l.Unlock() 43 | return log.get(seq) 44 | } 45 | 46 | func (log *sublog) get(seq int64) (interface{}, error) { 47 | if log.deleted { 48 | return nil, multilog.ErrSublogDeleted 49 | } 50 | 51 | if seq < 0 { 52 | return nil, luigi.EOS{} 53 | } 54 | 55 | v, err := log.bmap.Select(uint64(seq)) 56 | if err != nil { 57 | return nil, luigi.EOS{} 58 | } 59 | return int64(v), err 60 | } 61 | 62 | func (log *sublog) Query(specs ...margaret.QuerySpec) (luigi.Source, error) { 63 | log.mlog.l.Lock() 64 | defer log.mlog.l.Unlock() 65 | if log.deleted { 66 | return nil, multilog.ErrSublogDeleted 67 | } 68 | qry := &query{ 69 | log: log, 70 | 71 | lt: margaret.SeqEmpty, 72 | nextSeq: margaret.SeqEmpty, 73 | 74 | limit: -1, //i.e. no limit 75 | } 76 | 77 | for _, spec := range specs { 78 | err := spec(qry) 79 | if err != nil { 80 | return nil, err 81 | } 82 | } 83 | 84 | return qry, nil 85 | } 86 | 87 | func (log *sublog) Append(v interface{}) (int64, error) { 88 | log.mlog.l.Lock() 89 | defer log.mlog.l.Unlock() 90 | if log.deleted { 91 | return margaret.SeqSublogDeleted, multilog.ErrSublogDeleted 92 | } 93 | val, ok := v.(int64) 94 | if !ok { 95 | switch tv := v.(type) { 96 | case int: 97 | val = int64(tv) 98 | case int64: 99 | val = int64(tv) 100 | case uint32: 101 | val = int64(tv) 102 | default: 103 | return int64(-2), fmt.Errorf("roaringfiles: not a sequence (%T)", v) 104 | } 105 | } 106 | if val < 0 { 107 | return margaret.SeqErrored, fmt.Errorf("roaringfiles can only store positive numbers") 108 | } 109 | 110 | log.bmap.Set(uint64(val)) 111 | 112 | log.dirty = true 113 | log.seq.Inc() 114 | 115 | count := log.bmap.GetCardinality() - 1 116 | newSeq := int64(count) 117 | 118 | err := log.luigiObsv.Set(newSeq) 119 | if err != nil { 120 | err = fmt.Errorf("roaringfiles: failed to update sequence: %w", err) 121 | return margaret.SeqErrored, err 122 | } 123 | return newSeq, nil 124 | } 125 | 126 | func (log *sublog) store() error { 127 | if log.deleted { 128 | return multilog.ErrSublogDeleted 129 | } 130 | data := log.bmap.ToBuffer() 131 | 132 | var err error 133 | err = log.mlog.store.Put(log.key, data) 134 | if err != nil { 135 | return fmt.Errorf("roaringfiles: file write failed: %w", err) 136 | } 137 | return nil 138 | } 139 | -------------------------------------------------------------------------------- /offset2/alter_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package offset2 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "os" 11 | "path/filepath" 12 | "strconv" 13 | "testing" 14 | 15 | "github.com/ssbc/go-luigi" 16 | "github.com/ssbc/margaret" 17 | mjson "github.com/ssbc/margaret/codec/json" 18 | "github.com/stretchr/testify/assert" 19 | "github.com/stretchr/testify/require" 20 | ) 21 | 22 | var _ margaret.Alterer = (*OffsetLog)(nil) 23 | 24 | func TestNull(t *testing.T) { 25 | os.RemoveAll("testrun") 26 | tevs := []testEvent{ 27 | testEvent{"hello", 23}, 28 | testEvent{"world", 42}, 29 | testEvent{"world", 161}, 30 | testEvent{"world", 1312}, 31 | testEvent{"moar", 1234}, 32 | } 33 | 34 | for i := 0; i < len(tevs); i++ { 35 | var seq = int64(i) 36 | t.Run(strconv.Itoa(i), nullOne(tevs, seq)) 37 | } 38 | } 39 | 40 | func nullOne(tevs []testEvent, nullSeq int64) func(*testing.T) { 41 | return func(t *testing.T) { 42 | //setup 43 | r := require.New(t) 44 | a := assert.New(t) 45 | 46 | name := filepath.Join("testrun", t.Name()) 47 | 48 | log, err := Open(name, mjson.New(&testEvent{})) 49 | r.NoError(err, "error during log creation") 50 | 51 | // fill 52 | for i, ev := range tevs { 53 | seq, err := log.Append(ev) 54 | r.NoError(err, "failed to append event %d", i) 55 | r.Equal(int64(i), seq, "sequence missmatch") 56 | } 57 | 58 | r.NoError(log.Close()) 59 | 60 | // reopen for const check 61 | log, err = Open(name, mjson.New(&testEvent{})) 62 | r.NoError(err, "error reopening log") 63 | 64 | seq := log.Seq() 65 | r.EqualValues(int64(len(tevs)-1), seq, "sequence missmatch") 66 | 67 | err = log.Null(nullSeq) 68 | r.NoError(err, "failed null") 69 | 70 | // make sure we can null twice without an error 71 | err = log.Null(nullSeq) 72 | r.NoError(err, "failed null (again)") 73 | 74 | // reopen after null 75 | r.NoError(log.Close()) 76 | log, err = Open(name, mjson.New(&testEvent{})) 77 | r.NoError(err, "error reopening log #2") 78 | 79 | // get loop 80 | for i := 0; i < len(tevs); i++ { 81 | v, err := log.Get(int64(i)) 82 | if i == int(nullSeq) { 83 | r.True(errors.Is(err, margaret.ErrNulled)) 84 | r.True(margaret.IsErrNulled(err)) 85 | r.Nil(v) 86 | } else { 87 | r.NoError(err, "error reopening log") 88 | te, ok := v.(*testEvent) 89 | r.True(ok) 90 | a.Equal(tevs[i], *te) 91 | } 92 | } 93 | 94 | // pump drain 95 | ctx := context.TODO() 96 | src, err := log.Query() 97 | r.NoError(err) 98 | 99 | i := 0 100 | snk := luigi.FuncSink(func(ctx context.Context, v interface{}, err error) error { 101 | if err != nil { 102 | if luigi.IsEOS(err) { 103 | return nil 104 | } 105 | return err 106 | } 107 | if i == int(nullSeq) { 108 | r.Equal(margaret.ErrNulled, v) 109 | } 110 | i++ 111 | return nil 112 | }) 113 | 114 | err = luigi.Pump(ctx, snk, src) 115 | r.NoError(err) 116 | r.Equal(len(tevs), i) 117 | 118 | // manual drain 119 | src, err = log.Query() 120 | r.NoError(err) 121 | 122 | i = 0 123 | for { 124 | v, err := src.Next(ctx) 125 | // fmt.Println(i, v, err) 126 | if luigi.IsEOS(err) { 127 | break 128 | } 129 | if i == int(nullSeq) { 130 | a.Equal(margaret.ErrNulled, v) 131 | } 132 | i++ 133 | } 134 | r.Equal(len(tevs), i) 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | # Margaret [![Go Reference](https://pkg.go.dev/badge/github.com/ssbc/margaret.svg)](https://pkg.go.dev/github.com/ssbc/margaret) ![[Github Actions Tests](https://github.com/ssbc/margaret/actions/workflows/go.yml)](https://github.com/ssbc/margaret/actions/workflows/go.yml/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/ssbc/margaret)](https://goreportcard.com/report/github.com/ssbc/margaret) [![REUSE status](https://api.reuse.software/badge/github.com/ssbc/margaret)](https://api.reuse.software/info/github.com/ssbc/margaret) 8 | 9 | Margaret is [`go-ssb`](https://github.com/ssbc/go-ssb)'s [append-only](https://en.wikipedia.org/wiki/Append-only) 10 | log\* provider, and greatly inspired by [flumedb](https://github.com/flumedb/flumedb). Compatible with Go 1.13+. 11 | 12 | ![margaret the log lady, 1989 edition](https://static.wikia.nocookie.net/twinpeaks/images/6/68/Logladyreplacement.jpg/revision/latest/scale-to-width-down/500?cb=20160906170235) 13 | 14 | _the project name is inspired by Twin Peaks's character [Margaret](https://twinpeaks.fandom.com/wiki/Margaret_Lanterman) aka **the 15 | log lady**_ 16 | 17 | Margaret has the following facilities: 18 | * an append-only log interface `.Append(interface{})`, `.Get(int64)` 19 | * [queries](https://godocs.io/github.com/ssbc/margaret#Query) `.Query(...QuerySpec)` for retrieving ranges based on sequence numbers e.g. `.Gt(int64)`, or limiting the amount of data returned `.Limit(int64)` 20 | * a variety of index mechanisms, both for categorizing log entries into buckets and for creating virtual logs (aka sublogs) 21 | 22 | Margaret is one of a few key components that make the [go implementation of ssb](https://github.com/ssbc/go-ssb/) tick, for example: 23 | * [`ssb/sbot`](https://github.com/ssbc/go-ssb/) uses margaret for storing each peer's data 24 | 25 | ### Log storage 26 | Margaret outputs data according to the [`offset2`](https://godocs.io/github.com/ssbc/margaret/offset2) format, which is inspired by (but significantly differs from) [`flumelog-offset`](https://github.com/flumedb/flumelog-offset). 27 | 28 | In brief: margaret stores the data of _all logs_ in the three following files: 29 | * `data` stores the actual data (with a length-prefix before each entry) 30 | * `ofst` indexes the starting locations for each data entry in `data` 31 | * `jrnl` an integrity checking mechanism for all three files; a checksum of sorts, [more details](https://github.com/ssbc/margaret/blob/master/offset2/log.go#L215) 32 | 33 | ## More details 34 | There are a few concepts that might be tough to digest for newcomers on first approach: 35 | 36 | * multilogs, a kind of _tree_-based index, where each leaf is a margaret.Log 37 | * in other words: it creates virtual sublogs that map to entries in an offset log (see log storage above) 38 | * `margaret/indexes` similar to leveldb indexes (arbitrary key-value stores) 39 | * sublogs (and rxLog/receiveLog/offsetLog and its equivalence to offset.log) 40 | * queries 41 | * zeroing out, or replacing, written data 42 | 43 | For more on these concepts, visit the [dev.scutttlebutt.nz](https://dev.scuttlebutt.nz/#/golang/) portal for in-depth explanations. 44 | 45 | 46 | \* margaret is technically an append-_based_ log, as there is support for both zeroing out and 47 | replacing items in the log after they have been written. Given the relative ubiquity of 48 | append-only logs & their uses, it's easier to just say append-only log. 49 | -------------------------------------------------------------------------------- /qry.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package margaret // import "github.com/ssbc/margaret" 6 | 7 | //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o mock/qry.go . Query 8 | 9 | // Query is the interface implemented by the concrete log implementations that collects the constraints of the query. 10 | type Query interface { 11 | // Gt makes the source return only items with sequence numbers > seq. 12 | Gt(seq int64) error 13 | // Gte makes the source return only items with sequence numbers >= seq. 14 | Gte(seq int64) error 15 | // Lt makes the source return only items with sequence numbers < seq. 16 | Lt(seq int64) error 17 | // Lte makes the source return only items with sequence numbers <= seq. 18 | Lte(seq int64) error 19 | // Limit makes the source return only up to n items. 20 | Limit(n int) error 21 | 22 | // Reverse makes the source return the lastest values first 23 | Reverse(yes bool) error 24 | 25 | // Live makes the source block at the end of the log and wait for new values 26 | // that are being appended. 27 | Live(bool) error 28 | 29 | // SeqWrap makes the source return values that contain both the item and its 30 | // sequence number, instead of the item alone. 31 | SeqWrap(bool) error 32 | } 33 | 34 | // QuerySpec is a constraint on the query. 35 | type QuerySpec func(Query) error 36 | 37 | // MergeQuerySpec collects several contraints and merges them into one. 38 | func MergeQuerySpec(spec ...QuerySpec) QuerySpec { 39 | return func(qry Query) error { 40 | for _, f := range spec { 41 | err := f(qry) 42 | if err != nil { 43 | return err 44 | } 45 | } 46 | 47 | return nil 48 | } 49 | } 50 | 51 | // ErrorQuerySpec makes the log.Query call return the passed error. 52 | func ErrorQuerySpec(err error) QuerySpec { 53 | return func(Query) error { 54 | return err 55 | } 56 | } 57 | 58 | // Gt makes the source return only items with sequence numbers > seq. 59 | func Gt(s int64) QuerySpec { 60 | return func(q Query) error { 61 | return q.Gt(s) 62 | } 63 | } 64 | 65 | // Gte makes the source return only items with sequence numbers >= seq. 66 | func Gte(s int64) QuerySpec { 67 | return func(q Query) error { 68 | return q.Gte(s) 69 | } 70 | } 71 | 72 | // Lt makes the source return only items with sequence numbers < seq. 73 | func Lt(s int64) QuerySpec { 74 | return func(q Query) error { 75 | return q.Lt(s) 76 | } 77 | } 78 | 79 | // Lte makes the source return only items with sequence numbers <= seq. 80 | func Lte(s int64) QuerySpec { 81 | return func(q Query) error { 82 | return q.Lte(s) 83 | } 84 | } 85 | 86 | // Limit makes the source return only up to n items. 87 | func Limit(n int) QuerySpec { 88 | return func(q Query) error { 89 | return q.Limit(n) 90 | } 91 | } 92 | 93 | // Live makes the source block at the end of the log and wait for new values 94 | // that are being appended. 95 | func Live(live bool) QuerySpec { 96 | return func(q Query) error { 97 | return q.Live(live) 98 | } 99 | } 100 | 101 | // SeqWrap makes the source return values that contain both the item and its 102 | // sequence number, instead of the item alone. 103 | func SeqWrap(wrap bool) QuerySpec { 104 | return func(q Query) error { 105 | return q.SeqWrap(wrap) 106 | } 107 | } 108 | 109 | func Reverse(yes bool) QuerySpec { 110 | return func(q Query) error { 111 | return q.Reverse(yes) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /internal/persist/mkv/saver.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package mkv 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "io" 11 | 12 | "github.com/pkg/errors" 13 | "github.com/ssbc/margaret/internal/persist" 14 | ) 15 | 16 | const pageSize = 64 * 1024 17 | 18 | func (s ModernSaver) Put(key persist.Key, data []byte) error { 19 | if len(data) < pageSize { 20 | return s.db.Set(append(key, 0), data) 21 | } 22 | var ( 23 | i int 24 | page []byte 25 | ) 26 | for i, page = range splitPages(data) { 27 | if i > 255 { 28 | return errors.Errorf("persist/mkv: storage pageing exceeded") 29 | } 30 | if err := s.db.Set(append(key, byte(i)), page); err != nil { 31 | return errors.Wrapf(err, "shard%d set failed", i) 32 | } 33 | } 34 | olderPagers, _, err := s.db.Seek(append(key, byte(i+1))) 35 | if err != nil { 36 | return err 37 | } 38 | for { 39 | k, _, err := olderPagers.Next() 40 | if err != nil { 41 | if err == io.EOF { 42 | break 43 | } 44 | return errors.Wrap(err, "scraping old pages failed") 45 | } 46 | err = s.db.Delete(k) 47 | if err != nil { 48 | return err 49 | } 50 | } 51 | return nil 52 | } 53 | 54 | func (s ModernSaver) PutMultiple(values []persist.KeyValuePair) error { 55 | for i, kv := range values { 56 | err := s.Put(kv.Key, kv.Value) 57 | if err != nil { 58 | return fmt.Errorf("persist/mkv: failed to put entry %d of %d (%s): %w", i, len(values), kv.Key, err) 59 | } 60 | } 61 | return nil 62 | } 63 | 64 | func splitPages(data []byte) [][]byte { 65 | pages := make([][]byte, len(data)/pageSize+1) 66 | i := 0 67 | for len(data) > pageSize { 68 | pages[i], data = data[:pageSize], data[pageSize:] 69 | i++ 70 | } 71 | pages[i] = data 72 | return pages 73 | } 74 | 75 | func (s ModernSaver) Get(key persist.Key) ([]byte, error) { 76 | var data []byte 77 | enum, _, err := s.db.Seek(key) 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | for { 83 | k, d, err := enum.Next() 84 | if err != nil { 85 | if err == io.EOF { 86 | break 87 | } 88 | return nil, err 89 | } 90 | if bytes.Equal(k[:len(k)-1], key) { 91 | data = append(data, d...) 92 | } else { 93 | break 94 | } 95 | } 96 | if len(data) == 0 { 97 | return nil, persist.ErrNotFound 98 | } 99 | 100 | return data, nil 101 | } 102 | 103 | func (s ModernSaver) List() ([]persist.Key, error) { 104 | has := make(map[string]struct{}) 105 | var keys []persist.Key 106 | iter, err := s.db.SeekFirst() 107 | if err != nil { 108 | if err == io.EOF { 109 | return keys, nil 110 | } 111 | return nil, err 112 | } 113 | for { 114 | k, _, err := iter.Next() 115 | if err != nil { 116 | if err == io.EOF { 117 | break 118 | } 119 | return nil, err 120 | } 121 | pk := persist.Key(k[:len(k)-1]) 122 | if _, hit := has[pk.String()]; !hit { 123 | keys = append(keys, pk) 124 | has[pk.String()] = struct{}{} 125 | } 126 | } 127 | return keys, nil 128 | } 129 | 130 | func (s ModernSaver) Delete(rm persist.Key) error { 131 | enum, _, err := s.db.Seek(rm) 132 | if err != nil { 133 | return err 134 | } 135 | for { 136 | k, _, err := enum.Next() 137 | if err != nil { 138 | if err == io.EOF { 139 | break 140 | } 141 | return err 142 | } 143 | 144 | if !bytes.HasPrefix(k, rm) { 145 | break 146 | } 147 | 148 | if err := s.db.Delete(k); err != nil { 149 | return err 150 | } 151 | 152 | } 153 | return nil 154 | } 155 | -------------------------------------------------------------------------------- /offset2/alter_replace_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package offset2 6 | 7 | import ( 8 | "context" 9 | "encoding/json" 10 | "os" 11 | "path/filepath" 12 | "strconv" 13 | "strings" 14 | "testing" 15 | 16 | "github.com/ssbc/go-luigi" 17 | mjson "github.com/ssbc/margaret/codec/json" 18 | "github.com/stretchr/testify/assert" 19 | "github.com/stretchr/testify/require" 20 | ) 21 | 22 | func TestReplace(t *testing.T) { 23 | os.RemoveAll("testrun") 24 | tevs := []testEvent{ 25 | testEvent{"hello", 23}, 26 | testEvent{"world", 42}, 27 | testEvent{"world", 161}, 28 | testEvent{"world", 1312}, 29 | testEvent{"moar", 1234}, 30 | testEvent{strings.Repeat("ACAB", 191), 1312}, 31 | testEvent{"s", 1}, // small 32 | } 33 | 34 | for i := 0; i < len(tevs); i++ { 35 | var seq = int64(i) 36 | t.Run(strconv.Itoa(i), replaceOne(tevs, seq)) 37 | } 38 | } 39 | 40 | func replaceOne(tevs []testEvent, nullSeq int64) func(*testing.T) { 41 | return func(t *testing.T) { 42 | //setup 43 | r := require.New(t) 44 | a := assert.New(t) 45 | 46 | name := filepath.Join("testrun", t.Name()) 47 | 48 | log, err := Open(name, mjson.New(&testEvent{})) 49 | r.NoError(err, "error during log creation") 50 | 51 | for i, ev := range tevs { 52 | seq, err := log.Append(ev) 53 | r.NoError(err, "failed to append event %d", i) 54 | r.EqualValues(i, seq, "sequence missmatch") 55 | } 56 | 57 | repEvt := testEvent{"REPLACE", 0} 58 | replaceData, err := json.Marshal(repEvt) 59 | r.NoError(err) 60 | 61 | // reopen for const check 62 | r.NoError(log.Close()) 63 | log, err = Open(name, mjson.New(&testEvent{})) 64 | r.NoError(err, "error reopening log") 65 | 66 | seq := log.Seq() 67 | r.EqualValues(len(tevs)-1, seq, "sequence missmatch") 68 | 69 | err = log.Replace(nullSeq, replaceData) 70 | r.NoError(err, "failed get current value") 71 | r.NoError(log.Close()) 72 | 73 | // reopen after null 74 | log, err = Open(name, mjson.New(&testEvent{})) 75 | r.NoError(err, "error reopening log #2") 76 | 77 | // get loop 78 | for i := 0; i < len(tevs); i++ { 79 | v, err := log.Get(int64(i)) 80 | r.NoError(err, "error reading from log") 81 | te, ok := v.(*testEvent) 82 | r.True(ok, "wrong type: %T %v", v, v) 83 | if i == int(nullSeq) { 84 | a.Equal(repEvt, *te) 85 | } else { 86 | a.Equal(tevs[i], *te) 87 | } 88 | } 89 | 90 | // pump drain 91 | ctx := context.TODO() 92 | src, err := log.Query() 93 | r.NoError(err) 94 | 95 | i := 0 96 | snk := luigi.FuncSink(func(ctx context.Context, v interface{}, err error) error { 97 | if err != nil { 98 | if luigi.IsEOS(err) { 99 | return nil 100 | } 101 | return err 102 | } 103 | te, ok := v.(*testEvent) 104 | r.True(ok, "wrong type: %T %v", v, v) 105 | if i == int(nullSeq) { 106 | a.Equal(repEvt, *te) 107 | } else { 108 | a.Equal(tevs[i], *te) 109 | } 110 | i++ 111 | return nil 112 | }) 113 | 114 | err = luigi.Pump(ctx, snk, src) 115 | r.NoError(err) 116 | r.Equal(len(tevs), i) 117 | 118 | // manual drain 119 | src, err = log.Query() 120 | r.NoError(err) 121 | 122 | i = 0 123 | for { 124 | v, err := src.Next(ctx) 125 | if luigi.IsEOS(err) { 126 | break 127 | } 128 | te, ok := v.(*testEvent) 129 | r.True(ok, "wrong type: %T %v", v, v) 130 | if i == int(nullSeq) { 131 | a.Equal(repEvt, *te) 132 | } else { 133 | a.Equal(tevs[i], *te) 134 | } 135 | i++ 136 | } 137 | r.Equal(len(tevs), i) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /internal/persist/test/general_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "crypto/rand" 9 | "crypto/sha256" 10 | "io" 11 | "os" 12 | "path/filepath" 13 | "testing" 14 | 15 | "github.com/ssbc/margaret/internal/persist" 16 | "github.com/ssbc/margaret/internal/persist/badger" 17 | "github.com/ssbc/margaret/internal/persist/fs" 18 | "github.com/ssbc/margaret/internal/persist/mkv" 19 | "github.com/ssbc/margaret/internal/persist/sqlite" 20 | "github.com/stretchr/testify/require" 21 | 22 | _ "github.com/mattn/go-sqlite3" 23 | ) 24 | 25 | func SimpleSaver(mk func(*testing.T) persist.Saver) func(*testing.T) { 26 | 27 | return func(t *testing.T) { 28 | p := mk(t) 29 | 30 | r := require.New(t) 31 | 32 | l, err := p.List() 33 | r.NoError(err) 34 | r.Len(l, 0, "%v", l) 35 | 36 | k := persist.Key{0, 0, 0, 1} 37 | d, err := p.Get(k) 38 | r.EqualError(err, persist.ErrNotFound.Error()) 39 | r.Nil(d) 40 | 41 | testData := []byte("fooo") 42 | 43 | err = p.Put(k, testData) 44 | r.NoError(err) 45 | 46 | l, err = p.List() 47 | r.NoError(err) 48 | r.Len(l, 1) 49 | r.Equal(k, l[0]) 50 | 51 | d, err = p.Get(k) 52 | r.NoError(err) 53 | r.Equal(d, testData) 54 | 55 | // mkvs limit is 64k 56 | n := 160 * 1024 57 | bigKey, big := makeRandData(r, n) 58 | err = p.Put(bigKey, big) 59 | r.NoError(err) 60 | 61 | l, err = p.List() 62 | r.NoError(err) 63 | r.Len(l, 2) 64 | r.Equal(k, l[0]) 65 | r.Equal(bigKey, l[1]) 66 | 67 | bigdata, err := p.Get(bigKey) 68 | r.NoError(err) 69 | r.Equal(n, len(bigdata)) 70 | r.Equal(big, bigdata) 71 | 72 | //make something smaller to check dealloc of pages 73 | _, smaller := makeRandData(r, 75*1024) 74 | err = p.Put(bigKey, smaller) 75 | r.NoError(err) 76 | 77 | // test listing 78 | l, err = p.List() 79 | r.NoError(err) 80 | r.Len(l, 2) 81 | r.Equal(k, l[0]) 82 | r.Equal(bigKey, l[1]) 83 | 84 | getSmaller, err := p.Get(bigKey) 85 | r.NoError(err) 86 | r.Equal(len(smaller), len(getSmaller)) 87 | r.Equal(smaller, getSmaller) 88 | 89 | err = p.Delete(k) 90 | r.NoError(err) 91 | 92 | l, err = p.List() 93 | r.NoError(err) 94 | r.Len(l, 1) 95 | 96 | r.NoError(p.Close()) 97 | } 98 | } 99 | 100 | func makeRandData(r *require.Assertions, n int) (persist.Key, []byte) { 101 | big := make([]byte, n) 102 | h := sha256.New() 103 | tr := io.TeeReader(rand.Reader, h) 104 | got, err := tr.Read(big) 105 | r.NoError(err) 106 | r.Equal(n, got) 107 | return persist.Key(h.Sum(nil)), big 108 | } 109 | 110 | func TestSaver(t *testing.T) { 111 | t.Run("fs", SimpleSaver(makeFS)) 112 | t.Run("sqlite", SimpleSaver(makeSqlite)) 113 | t.Run("badger", SimpleSaver(makeBadger)) 114 | t.Run("kv", SimpleSaver(makeMKV)) 115 | } 116 | 117 | func makeFS(t *testing.T) persist.Saver { 118 | base := filepath.Join("testrun", t.Name()) 119 | os.RemoveAll(base) 120 | return fs.New(base) 121 | } 122 | 123 | func makeBadger(t *testing.T) persist.Saver { 124 | base := filepath.Join("testrun", t.Name()) 125 | os.RemoveAll(base) 126 | t.Log(base) 127 | s, err := badger.NewStandalone(base) 128 | if err != nil { 129 | t.Fatal(err) 130 | } 131 | return s 132 | } 133 | 134 | func makeSqlite(t *testing.T) persist.Saver { 135 | base := filepath.Join("testrun", t.Name()) 136 | os.RemoveAll(base) 137 | s, err := sqlite.New(base) 138 | if err != nil { 139 | t.Fatal(err) 140 | } 141 | return s 142 | } 143 | 144 | func makeMKV(t *testing.T) persist.Saver { 145 | base := filepath.Join("testrun", t.Name()) 146 | os.RemoveAll(base) 147 | s, err := mkv.New(base) 148 | if err != nil { 149 | t.Fatal(err) 150 | } 151 | return s 152 | } 153 | -------------------------------------------------------------------------------- /internal/persist/badger/saver.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package badger 6 | 7 | import ( 8 | "bytes" 9 | "errors" 10 | "fmt" 11 | 12 | "github.com/dgraph-io/badger/v3" 13 | "github.com/ssbc/margaret/internal/persist" 14 | ) 15 | 16 | func (s BadgerSaver) Put(key persist.Key, data []byte) error { 17 | actualKey := append(s.keyPrefix, []byte(key)...) 18 | 19 | return s.db.Update(func(txn *badger.Txn) error { 20 | return txn.Set(actualKey, data) 21 | }) 22 | } 23 | 24 | func (s BadgerSaver) PutMultiple(values []persist.KeyValuePair) error { 25 | // badger can only deal with ~18600 set operations in a single transition 26 | splitted := chunks(values, 18000) 27 | for i, chunk := range splitted { 28 | err := s.putMultiple(chunk) 29 | if err != nil { 30 | return fmt.Errorf("badger/putMultiple: chunk %d of %d failed: %w", i, len(splitted), err) 31 | } 32 | } 33 | return nil 34 | } 35 | 36 | func (s BadgerSaver) putMultiple(values []persist.KeyValuePair) error { 37 | return s.db.Update(func(txn *badger.Txn) error { 38 | for i, kv := range values { 39 | actualKey := append(s.keyPrefix, []byte(kv.Key)...) 40 | err := txn.Set(actualKey, kv.Value) 41 | if err != nil { 42 | return fmt.Errorf("failed to set entry %d of %d (%s): %w", i, len(values), kv.Key, err) 43 | } 44 | } 45 | return nil 46 | }) 47 | } 48 | 49 | // splits up the passed slice into chunks of a specific sice 50 | func chunks(pairs []persist.KeyValuePair, chunkSize int) [][]persist.KeyValuePair { 51 | if len(pairs) == 0 { 52 | return nil 53 | } 54 | divided := make([][]persist.KeyValuePair, (len(pairs)+chunkSize-1)/chunkSize) 55 | prev := 0 // previous start of a chunk 56 | i := 0 // how many chunks we processed 57 | till := len(pairs) - chunkSize 58 | for prev < till { 59 | next := prev + chunkSize 60 | divided[i] = pairs[prev:next] 61 | prev = next 62 | i++ 63 | } 64 | divided[i] = pairs[prev:] // rest (ie final chunk) 65 | return divided 66 | } 67 | 68 | func (s BadgerSaver) Get(key persist.Key) ([]byte, error) { 69 | actualKey := append(s.keyPrefix, []byte(key)...) 70 | 71 | var data []byte 72 | err := s.db.View(func(txn *badger.Txn) error { 73 | it, err := txn.Get(actualKey) 74 | if err != nil { 75 | return err 76 | } 77 | data, err = it.ValueCopy(nil) 78 | if err != nil { 79 | return err 80 | } 81 | return nil 82 | }) 83 | if err != nil { 84 | if errors.Is(err, badger.ErrKeyNotFound) { 85 | return nil, persist.ErrNotFound 86 | } 87 | return nil, err 88 | } 89 | 90 | if len(data) == 0 { 91 | return nil, persist.ErrNotFound 92 | } 93 | 94 | return data, nil 95 | } 96 | 97 | func (s BadgerSaver) List() ([]persist.Key, error) { 98 | var keys []persist.Key 99 | 100 | err := s.db.Update(func(txn *badger.Txn) error { 101 | iter := txn.NewIterator(badger.DefaultIteratorOptions) 102 | defer iter.Close() 103 | 104 | for iter.Rewind(); iter.Valid(); iter.Next() { 105 | it := iter.Item() 106 | 107 | k := it.Key() 108 | 109 | if !bytes.HasPrefix(k, s.keyPrefix) { 110 | continue 111 | } 112 | 113 | k = bytes.TrimPrefix(k, s.keyPrefix) 114 | 115 | // we need to make a copy of the key since badger reuses the slice on the next iteration 116 | var trimmedKey = make([]byte, len(k)) 117 | copy(trimmedKey, k) 118 | 119 | keys = append(keys, persist.Key(trimmedKey)) 120 | } 121 | return nil 122 | }) 123 | if err != nil { 124 | return nil, err 125 | } 126 | 127 | return keys, nil 128 | } 129 | 130 | func (s BadgerSaver) Delete(rm persist.Key) error { 131 | actualKey := append(s.keyPrefix, []byte(rm)...) 132 | return s.db.Update(func(txn *badger.Txn) error { 133 | return txn.Delete(actualKey) 134 | }) 135 | } 136 | -------------------------------------------------------------------------------- /mem/qry.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package mem // import "github.com/ssbc/margaret/mem" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | 11 | "github.com/pkg/errors" 12 | "github.com/ssbc/go-luigi" 13 | 14 | "github.com/ssbc/margaret" 15 | ) 16 | 17 | type memlogQuery struct { 18 | log *memlog 19 | cur *memlogElem 20 | 21 | gt, lt, gte, lte int64 22 | 23 | limit int 24 | live bool 25 | seqWrap bool 26 | reverse bool 27 | } 28 | 29 | func (qry *memlogQuery) seek(ctx context.Context) error { 30 | var err error 31 | 32 | if qry.gt != margaret.SeqEmpty { 33 | if qry.cur.seq > qry.gt { 34 | qry.cur = qry.log.head 35 | } 36 | 37 | for (qry.cur.seq + 1) <= qry.gt { 38 | qry.cur, err = qry.cur.waitNext(ctx, &qry.log.l) 39 | if err != nil { 40 | return err 41 | } 42 | } 43 | } else if qry.gte != margaret.SeqEmpty { 44 | if qry.cur.seq > qry.gte { 45 | qry.cur = qry.log.head 46 | } 47 | 48 | for (qry.cur.seq + 1) < qry.gte { 49 | qry.cur, err = qry.cur.waitNext(ctx, &qry.log.l) 50 | if err != nil { 51 | return err 52 | } 53 | } 54 | } 55 | 56 | return nil 57 | } 58 | 59 | func (qry *memlogQuery) Gt(s int64) error { 60 | if qry.gt != margaret.SeqEmpty || qry.gte != margaret.SeqEmpty { 61 | return fmt.Errorf("lower bound already set") 62 | } 63 | 64 | qry.gt = s 65 | return nil 66 | } 67 | 68 | func (qry *memlogQuery) Gte(s int64) error { 69 | if qry.gt != margaret.SeqEmpty || qry.gte != margaret.SeqEmpty { 70 | return fmt.Errorf("lower bound already set") 71 | } 72 | 73 | qry.gte = s 74 | return nil 75 | } 76 | 77 | func (qry *memlogQuery) Lt(s int64) error { 78 | if qry.lt != margaret.SeqEmpty || qry.lte != margaret.SeqEmpty { 79 | return fmt.Errorf("upper bound already set") 80 | } 81 | 82 | qry.lt = s 83 | return nil 84 | } 85 | 86 | func (qry *memlogQuery) Lte(s int64) error { 87 | if qry.lt != margaret.SeqEmpty || qry.lte != margaret.SeqEmpty { 88 | return fmt.Errorf("upper bound already set") 89 | } 90 | 91 | qry.lte = s 92 | return nil 93 | } 94 | 95 | func (qry *memlogQuery) Limit(n int) error { 96 | qry.limit = n 97 | return nil 98 | } 99 | 100 | func (qry *memlogQuery) Live(live bool) error { 101 | qry.live = live 102 | return nil 103 | } 104 | 105 | func (qry *memlogQuery) SeqWrap(wrap bool) error { 106 | qry.seqWrap = wrap 107 | return nil 108 | } 109 | 110 | func (qry *memlogQuery) Reverse(yes bool) error { 111 | qry.reverse = yes 112 | if yes { 113 | qry.cur = qry.log.tail 114 | } 115 | return nil 116 | } 117 | 118 | func (qry *memlogQuery) Next(ctx context.Context) (interface{}, error) { 119 | if qry.limit == 0 { 120 | return nil, luigi.EOS{} 121 | } 122 | qry.limit-- 123 | 124 | qry.log.l.Lock() 125 | defer qry.log.l.Unlock() 126 | 127 | if qry.reverse { 128 | if qry.cur == qry.log.head { 129 | return qry.cur.v, luigi.EOS{} 130 | } 131 | v := qry.cur.v 132 | qry.cur = qry.cur.prev 133 | return v, nil 134 | } 135 | 136 | if qry.cur.seq <= qry.gt || qry.cur.seq < qry.gt { 137 | err := qry.seek(ctx) 138 | if err != nil { 139 | return nil, err 140 | } 141 | } 142 | 143 | // no new data yet and non-blocking 144 | if qry.cur.next == nil && !qry.live { 145 | return nil, luigi.EOS{} 146 | } 147 | 148 | if qry.lt != margaret.SeqEmpty && !(qry.cur.seq < (qry.lt)-1) { 149 | return nil, luigi.EOS{} 150 | } else if qry.lte != margaret.SeqEmpty && !(qry.cur.seq < qry.lte) { 151 | return nil, luigi.EOS{} 152 | } 153 | 154 | var err error 155 | qry.cur, err = qry.cur.waitNext(ctx, &qry.log.l) 156 | if err != nil { 157 | return nil, errors.Wrap(err, "error waiting for next value") 158 | } 159 | 160 | if qry.seqWrap { 161 | return margaret.WrapWithSeq(qry.cur.v, qry.cur.seq), nil 162 | } 163 | return qry.cur.v, nil 164 | } 165 | -------------------------------------------------------------------------------- /mem/log.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package mem // import "github.com/ssbc/margaret/mem" 6 | 7 | import ( 8 | "context" 9 | "io" 10 | "sync" 11 | 12 | "github.com/pkg/errors" 13 | "github.com/ssbc/go-luigi" 14 | "github.com/ssbc/margaret" 15 | ) 16 | 17 | // TODO optimization idea: skip list 18 | type memlogElem struct { 19 | v interface{} 20 | seq int64 21 | next *memlogElem 22 | prev *memlogElem 23 | 24 | wait chan struct{} 25 | } 26 | 27 | func (el *memlogElem) waitNext(ctx context.Context, m *sync.Mutex) (*memlogElem, error) { 28 | // closure to localize defer. We need to lock before accessing el.next in the return. 29 | err := func() error { 30 | // yes, first unlock, then lock. We need to release the mutex to 31 | // allow Appends to happen, but we need to lock again afterwards! 32 | m.Unlock() 33 | defer m.Lock() 34 | 35 | select { 36 | // wait until new element has been added 37 | case <-el.wait: 38 | // or context is canceled 39 | case <-ctx.Done(): 40 | return ctx.Err() 41 | } 42 | 43 | return nil 44 | }() 45 | if err != nil { 46 | // return original element in error case 47 | return el, err 48 | } 49 | 50 | return el.next, nil 51 | } 52 | 53 | type memlog struct { 54 | l sync.Mutex 55 | 56 | seq luigi.Observable 57 | head, tail *memlogElem 58 | 59 | closed bool 60 | } 61 | 62 | // New returns a new in-memory log 63 | func New() margaret.Log { 64 | root := &memlogElem{ 65 | seq: margaret.SeqEmpty, 66 | wait: make(chan struct{}), 67 | } 68 | 69 | log := &memlog{ 70 | seq: luigi.NewObservable(margaret.SeqEmpty), 71 | head: root, 72 | tail: root, 73 | } 74 | 75 | return log 76 | } 77 | 78 | func (log *memlog) Close() error { 79 | log.l.Lock() 80 | defer log.l.Unlock() 81 | if log.closed { 82 | return io.ErrClosedPipe // already closed 83 | } 84 | log.closed = true 85 | return nil 86 | } 87 | 88 | func (log *memlog) Seq() int64 { 89 | return log.tail.seq 90 | } 91 | 92 | func (log *memlog) Changes() luigi.Observable { 93 | return log.seq 94 | } 95 | 96 | func (log *memlog) Get(s int64) (interface{}, error) { 97 | log.l.Lock() 98 | defer log.l.Unlock() 99 | if log.closed { 100 | return nil, io.ErrClosedPipe // already closed 101 | } 102 | 103 | var ( 104 | cur = log.head 105 | ) 106 | 107 | for cur.seq < s && cur.next != nil { 108 | cur = cur.next 109 | } 110 | 111 | if cur.seq < s { 112 | return nil, margaret.OOB 113 | } 114 | 115 | if cur.seq > s { 116 | // TODO maybe better handling of this case? 117 | panic("datastructure borked, sequence number missing") 118 | } 119 | 120 | return cur.v, nil 121 | } 122 | 123 | func (log *memlog) Query(specs ...margaret.QuerySpec) (luigi.Source, error) { 124 | log.l.Lock() 125 | defer log.l.Unlock() 126 | if log.closed { 127 | return nil, io.ErrClosedPipe // already closed 128 | } 129 | 130 | qry := &memlogQuery{ 131 | log: log, 132 | cur: log.head, 133 | 134 | gt: margaret.SeqEmpty, 135 | gte: margaret.SeqEmpty, 136 | lt: margaret.SeqEmpty, 137 | lte: margaret.SeqEmpty, 138 | 139 | limit: -1, //i.e. no limit 140 | } 141 | 142 | for _, spec := range specs { 143 | err := spec(qry) 144 | if err != nil { 145 | return nil, err 146 | } 147 | } 148 | 149 | if qry.reverse && qry.live { 150 | return nil, errors.Errorf("memlog: can't do reverse and live") 151 | } 152 | 153 | return qry, nil 154 | } 155 | 156 | func (log *memlog) Append(v interface{}) (int64, error) { 157 | log.l.Lock() 158 | defer log.l.Unlock() 159 | if log.closed { 160 | return margaret.SeqErrored, io.ErrClosedPipe // already closed 161 | } 162 | 163 | nxt := &memlogElem{ 164 | v: v, 165 | seq: log.tail.seq + 1, 166 | wait: make(chan struct{}), 167 | } 168 | 169 | log.tail.next = nxt 170 | oldtail := log.tail 171 | nxt.prev = oldtail 172 | log.tail = log.tail.next 173 | 174 | close(oldtail.wait) 175 | log.seq.Set(log.tail.seq) 176 | 177 | return log.tail.seq, nil 178 | } 179 | -------------------------------------------------------------------------------- /multilog/roaring/qry.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package roaring 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "strings" 11 | 12 | "github.com/ssbc/go-luigi" 13 | "github.com/ssbc/margaret" 14 | ) 15 | 16 | type query struct { 17 | log *sublog 18 | 19 | nextSeq, lt int64 20 | 21 | limit int 22 | live bool 23 | reverse bool 24 | seqWrap bool 25 | } 26 | 27 | func (qry *query) Gt(s int64) error { 28 | if qry.nextSeq > margaret.SeqEmpty { 29 | return fmt.Errorf("lower bound already set") 30 | } 31 | 32 | qry.nextSeq = s + 1 33 | return nil 34 | } 35 | 36 | func (qry *query) Gte(s int64) error { 37 | if qry.nextSeq > margaret.SeqEmpty { 38 | return fmt.Errorf("lower bound already set") 39 | } 40 | 41 | qry.nextSeq = s 42 | return nil 43 | } 44 | 45 | func (qry *query) Lt(s int64) error { 46 | if qry.lt != margaret.SeqEmpty { 47 | return fmt.Errorf("upper bound already set") 48 | } 49 | 50 | qry.lt = s 51 | return nil 52 | } 53 | 54 | func (qry *query) Lte(s int64) error { 55 | if qry.lt != margaret.SeqEmpty { 56 | return fmt.Errorf("upper bound already set") 57 | } 58 | 59 | qry.lt = s + 1 60 | return nil 61 | } 62 | 63 | func (qry *query) Limit(n int) error { 64 | qry.limit = n 65 | return nil 66 | } 67 | 68 | func (qry *query) Live(live bool) error { 69 | qry.live = live 70 | return nil 71 | } 72 | 73 | func (qry *query) SeqWrap(wrap bool) error { 74 | qry.seqWrap = wrap 75 | return nil 76 | } 77 | 78 | func (qry *query) Reverse(rev bool) error { 79 | qry.reverse = rev 80 | if rev { 81 | qry.nextSeq = qry.log.seq.Seq() - 1 82 | } 83 | return nil 84 | } 85 | 86 | func (qry *query) Next(ctx context.Context) (interface{}, error) { 87 | qry.log.mlog.l.Lock() 88 | 89 | if qry.limit == 0 { 90 | qry.log.mlog.l.Unlock() 91 | return nil, luigi.EOS{} 92 | } 93 | qry.limit-- 94 | 95 | if qry.nextSeq == margaret.SeqEmpty { 96 | if qry.reverse { 97 | qry.log.mlog.l.Unlock() 98 | return nil, luigi.EOS{} 99 | } 100 | qry.nextSeq = 0 101 | } 102 | 103 | if qry.lt != margaret.SeqEmpty { 104 | if qry.nextSeq >= qry.lt { 105 | qry.log.mlog.l.Unlock() 106 | return nil, luigi.EOS{} 107 | } 108 | } 109 | 110 | var v interface{} 111 | seqVal, err := qry.log.bmap.Select(uint64(qry.nextSeq)) 112 | v = int64(seqVal) 113 | if err != nil { 114 | if !strings.Contains(err.Error(), " is not less than the cardinality:") { 115 | qry.log.mlog.l.Unlock() 116 | return nil, fmt.Errorf("roaringfiles/qry: error in read transaction (%T): %w", err, err) 117 | } 118 | 119 | // key not found, so we reached the end 120 | // abort if not a live query, else wait until it's written 121 | if !qry.live { 122 | qry.log.mlog.l.Unlock() 123 | return nil, luigi.EOS{} 124 | } 125 | 126 | return qry.livequery(ctx) 127 | } 128 | 129 | if qry.seqWrap { 130 | v = margaret.WrapWithSeq(v, qry.nextSeq) 131 | if qry.reverse { 132 | qry.nextSeq-- 133 | } else { 134 | qry.nextSeq++ 135 | } 136 | qry.log.mlog.l.Unlock() 137 | return v, nil 138 | } 139 | 140 | if qry.reverse { 141 | qry.nextSeq-- 142 | } else { 143 | qry.nextSeq++ 144 | } 145 | qry.log.mlog.l.Unlock() 146 | return v, nil 147 | } 148 | 149 | func (qry *query) livequery(ctx context.Context) (interface{}, error) { 150 | thisNextSeq := qry.nextSeq 151 | qry.log.mlog.l.Unlock() 152 | 153 | var ( 154 | v interface{} 155 | err error 156 | ) 157 | 158 | select { 159 | case <-qry.log.seq.WaitFor(uint64(thisNextSeq)): 160 | v, err = qry.log.Get(thisNextSeq) 161 | if !qry.seqWrap { // simpler to have two +1's here then a defer 162 | qry.nextSeq++ 163 | } 164 | case <-ctx.Done(): 165 | err = fmt.Errorf("cancelled while waiting for value to be written: %w", ctx.Err()) 166 | } 167 | 168 | if err != nil { 169 | return nil, fmt.Errorf("livequery failed to retreive value: %w", err) 170 | } 171 | 172 | if qry.seqWrap { 173 | v = margaret.WrapWithSeq(v, qry.nextSeq) 174 | qry.nextSeq++ 175 | return v, nil 176 | } 177 | 178 | return v, err 179 | } 180 | -------------------------------------------------------------------------------- /offset2/test/pumplive.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "os" 11 | "testing" 12 | "time" 13 | 14 | "github.com/pkg/errors" 15 | "github.com/stretchr/testify/assert" 16 | "github.com/stretchr/testify/require" 17 | 18 | "github.com/ssbc/go-luigi" 19 | "github.com/ssbc/margaret" 20 | mtest "github.com/ssbc/margaret/test" 21 | ) 22 | 23 | func LogTestPumpLive(f mtest.NewLogFunc) func(*testing.T) { 24 | type testcase struct { 25 | tipe interface{} 26 | 27 | specs []margaret.QuerySpec 28 | errStr string 29 | seqWrap bool 30 | qryTime int 31 | 32 | //values1 is appended before the query starts 33 | values1 []interface{} 34 | 35 | // values2 is appended after the query starts 36 | values2 []interface{} 37 | 38 | // result1 is the expected received output before the query starts 39 | result1 []interface{} 40 | 41 | // result2 is the expected received output after the query starts 42 | result2 []interface{} 43 | } 44 | 45 | mkTest := func(tc testcase) func(*testing.T) { 46 | return func(t *testing.T) { 47 | a := assert.New(t) 48 | r := require.New(t) 49 | 50 | log, err := f(t.Name(), tc.tipe) 51 | r.NoError(err, "error creating log") 52 | r.NotNil(log, "returned log is nil") 53 | 54 | defer func() { 55 | if namer, ok := log.(interface{ FileName() string }); ok { 56 | r.NoError(os.RemoveAll(namer.FileName()), "error deleting log after test") 57 | } 58 | }() 59 | 60 | var iRes int 61 | var closed bool 62 | sink := luigi.FuncSink(func(ctx context.Context, v_ interface{}, err error) error { 63 | if err != nil { 64 | if err != (luigi.EOS{}) { 65 | t.Log("sink closed with non-EOS error:", err) 66 | } 67 | 68 | if closed { 69 | return errors.New("closing closed sink") 70 | } 71 | closed = true 72 | if iRes != len(tc.result1)+len(tc.result2) { 73 | t.Errorf("early end of stream at %d instead of %d", iRes, len(tc.result1)+len(tc.result2)) 74 | } 75 | 76 | return nil 77 | } 78 | 79 | if iRes >= len(tc.result1)+len(tc.result2) { 80 | t.Fatal("expected end but read value:", v_) 81 | } 82 | 83 | var v interface{} 84 | if iRes < len(tc.result1) { 85 | v = tc.result1[iRes] 86 | } else { 87 | v = tc.result2[iRes-len(tc.result1)] 88 | } 89 | 90 | iRes++ 91 | 92 | if tc.errStr == "" { 93 | if tc.seqWrap { 94 | sw := v.(margaret.SeqWrapper) 95 | sw_ := v_.(margaret.SeqWrapper) 96 | 97 | a.Equal(sw.Seq(), sw_.Seq(), "sequence number doesn't match") 98 | a.Equal(sw.Value(), sw_.Value(), "value doesn't match") 99 | } else { 100 | a.Equal(v, v, "values don't match") 101 | } 102 | } 103 | 104 | return nil 105 | }) 106 | 107 | // prepare context 108 | ctx := context.Background() 109 | ctx, cancel := context.WithCancel(ctx) 110 | defer cancel() 111 | 112 | // send first batch 113 | for i, v := range tc.values1 { 114 | seq, err := log.Append(v) 115 | r.NoError(err, "error appending to log") 116 | r.EqualValues(i, seq, "sequence missmatch") 117 | } 118 | 119 | // make live query and process it 120 | wait := make(chan struct{}) 121 | go func() { 122 | src, err := log.Query(append(tc.specs, margaret.Live(true))...) 123 | r.NoError(err, "error querying log") 124 | 125 | // process 126 | err = luigi.Pump(ctx, sink, src) 127 | a.Equal(context.Canceled, err, "stream copy error") 128 | 129 | // unblock main goroutine 130 | close(wait) 131 | }() 132 | 133 | // make sure the goroutine starts first 134 | time.Sleep(200 * time.Millisecond) 135 | 136 | // send second batch 137 | for i, v := range tc.values2 { 138 | seq, err := log.Append(v) 139 | r.NoError(err, "error appending to log") 140 | r.EqualValues(len(tc.values1)+i, seq, "sequence missmatch") 141 | } 142 | 143 | // cancel query processing goroutine 144 | cancel() 145 | 146 | // wait for query processing goroutine 147 | <-wait 148 | } 149 | } 150 | 151 | tcs := []testcase{ 152 | { 153 | tipe: 0, 154 | values1: []interface{}{1, 2, 3}, 155 | values2: []interface{}{4, 5, 6}, 156 | result1: []interface{}{1, 2, 3}, 157 | result2: []interface{}{4, 5, 6}, 158 | }, 159 | } 160 | 161 | return func(t *testing.T) { 162 | for i, tc := range tcs { 163 | t.Run(fmt.Sprint(i), mkTest(tc)) 164 | } 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /indexes/test/sinkindex.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "context" 9 | "os" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/require" 15 | 16 | "github.com/ssbc/go-luigi" 17 | "github.com/ssbc/margaret/indexes" 18 | mtest "github.com/ssbc/margaret/test" 19 | ) 20 | 21 | type NewSinkIndexFunc func(name string, tipe interface{}, f indexes.StreamProcFunc) (indexes.SinkIndex, error) 22 | 23 | func TestSinkIndex(newLog mtest.NewLogFunc, newIdx NewSeqSetterIndexFunc) func(*testing.T) { 24 | return func(t *testing.T) { 25 | t.Run("WithBreak", TestSinkIndexWithBreak(newLog, newIdx)) 26 | } 27 | } 28 | 29 | func TestSinkIndexWithBreak(newLog mtest.NewLogFunc, newIdx NewSeqSetterIndexFunc) func(*testing.T) { 30 | return func(t *testing.T) { 31 | a := assert.New(t) 32 | r := require.New(t) 33 | ctx := context.Background() 34 | 35 | // track this to check that we get every sequence number just once 36 | var lastSeq int64 = -1 37 | 38 | // define indexing function 39 | f := func(ctx context.Context, seq int64, v interface{}, idx indexes.SetterIndex) error { 40 | a.Equal(lastSeq+1, seq, "unexpected sequence number") 41 | lastSeq++ 42 | 43 | if strings.Contains(v.(string), "interesting") { 44 | return idx.Set(ctx, "interesting", v) 45 | } else if strings.Contains(v.(string), "boring") { 46 | return idx.Set(ctx, "boring", v) 47 | } 48 | 49 | return nil 50 | } 51 | 52 | // prepare underlying index 53 | seqSetIdx, err := newIdx(t.Name(), "str") 54 | r.NoError(err, "error creating SeqSetterIndex") 55 | 56 | // prepare sinkindex 57 | idx := indexes.NewSinkIndex(f, seqSetIdx) 58 | 59 | // prepare log 60 | log, err := newLog(t.Name(), "str") 61 | r.NoError(err, "error creating log") 62 | r.NotNil(log, "returned log is nil") 63 | 64 | // delete log file after test completion 65 | defer func() { 66 | if namer, ok := log.(interface{ FileName() string }); ok { 67 | r.NoError(os.RemoveAll(namer.FileName()), "error deleting log after test") 68 | } 69 | }() 70 | 71 | // put some values into the log 72 | _, err = log.Append("boring string") 73 | a.NoError(err, "error appending") 74 | _, err = log.Append("another boring string") 75 | a.NoError(err, "error appending") 76 | _, err = log.Append("mildly interesting string") 77 | a.NoError(err, "error appending") 78 | 79 | // pump the log into the indexer 80 | src, err := log.Query(idx.QuerySpec()) 81 | a.NoError(err, "error querying log") 82 | a.NoError(luigi.Pump(ctx, idx, src), "error pumping from queried src to SinkIndex") 83 | 84 | // check "interesting" 85 | obv, err := seqSetIdx.Get(ctx, "interesting") 86 | r.NoError(err, "error getting interesting index") 87 | r.NotNil(obv, "returned no error but got nil observable") 88 | 89 | v, err := obv.Value() 90 | a.NoError(err, "error getting interesting value from observable") 91 | a.Equal("mildly interesting string", v) 92 | 93 | // check "boring" 94 | obv, err = seqSetIdx.Get(ctx, "boring") 95 | a.NoError(err, "error getting boring index") 96 | r.NotNil(obv, "returned no error but got nil observable") 97 | 98 | v, err = obv.Value() 99 | a.NoError(err, "error getting boring value from observable") 100 | a.Equal("another boring string", v) 101 | 102 | // put some more values into the log 103 | _, err = log.Append("so-so string") 104 | a.NoError(err, "error appending") 105 | _, err = log.Append("highly interesting string") 106 | a.NoError(err, "error appending") 107 | 108 | // pump log values into the indexer 109 | src, err = log.Query(idx.QuerySpec()) 110 | a.NoError(err, "error querying log") 111 | a.NoError(luigi.Pump(ctx, idx, src), "error pumping from queried src to SinkIndex") 112 | 113 | // check "interesting" 114 | obv, err = seqSetIdx.Get(ctx, "interesting") 115 | a.NoError(err, "error getting interesting index") 116 | r.NotNil(obv, "returned no error but got nil observable") 117 | 118 | v, err = obv.Value() 119 | a.NoError(err, "error getting interesting value from observable") 120 | a.Equal("highly interesting string", v) 121 | 122 | // check "interesting" 123 | obv, err = seqSetIdx.Get(ctx, "boring") 124 | a.NoError(err, "error getting boring index") 125 | r.NotNil(obv, "returned no error but got nil observable") 126 | 127 | v, err = obv.Value() 128 | a.NoError(err, "error getting boring value from observable") 129 | a.Equal("another boring string", v) 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /multilog/test/sublog.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test // import "github.com/ssbc/margaret/multilog/test" 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "os" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/require" 15 | 16 | "github.com/ssbc/go-luigi" 17 | "github.com/ssbc/margaret" 18 | "github.com/ssbc/margaret/indexes" 19 | ) 20 | 21 | func SubLogTestGet(f NewLogFunc) func(*testing.T) { 22 | type testcase struct { 23 | tipe interface{} 24 | specs []margaret.QuerySpec 25 | values map[indexes.Addr][]interface{} 26 | errStr string 27 | live bool 28 | seqWrap bool 29 | } 30 | 31 | mkTest := func(tc testcase) func(*testing.T) { 32 | return func(t *testing.T) { 33 | a := assert.New(t) 34 | r := require.New(t) 35 | 36 | /* 37 | - make multilog 38 | - append values to sublogs 39 | - query all sublogs 40 | - check if entries match 41 | */ 42 | 43 | // make multilog 44 | mlog, dir, err := f(t.Name(), tc.tipe, "") 45 | r.NoError(err, "error creating multilog") 46 | 47 | // append values 48 | for addr, values := range tc.values { 49 | slog, err := mlog.Get(addr) 50 | r.NoError(err, "error getting sublog") 51 | 52 | // check empty 53 | r.EqualValues(slog.Seq(), margaret.SeqEmpty) 54 | 55 | ev, err := slog.Get(margaret.SeqEmpty) 56 | r.Error(err) 57 | r.True(errors.Is(err, luigi.EOS{}), "unexpected error: %s", err) 58 | r.Nil(ev) 59 | 60 | for i, v := range values { 61 | seq, err := slog.Append(v) 62 | r.NoError(err, "error appending to log") 63 | r.EqualValues(i, seq, "sequence missmatch") 64 | } 65 | 66 | // check full 67 | r.EqualValues(len(values)-1, slog.Seq()) 68 | } 69 | 70 | // check if multilog entries match 71 | for addr, results := range tc.values { 72 | slog, err := mlog.Get(addr) 73 | r.NoError(err, "error getting sublog") 74 | r.NotNil(slog, "retrieved sublog is nil") 75 | 76 | var v_ interface{} 77 | err = nil 78 | 79 | for seq, v := range results { 80 | v_, err = slog.Get(int64(seq)) 81 | if tc.errStr == "" { 82 | if tc.seqWrap { 83 | sw := v.(margaret.SeqWrapper) 84 | sw_ := v_.(margaret.SeqWrapper) 85 | 86 | a.Equal(sw.Seq(), sw_.Seq(), "sequence number doesn't match") 87 | a.Equal(sw.Value(), sw_.Value(), "value doesn't match") 88 | } else { 89 | a.EqualValues(v, v_, "values don't match") 90 | } 91 | } 92 | if err != nil { 93 | break 94 | } 95 | } 96 | 97 | if err != nil && tc.errStr == "" { 98 | t.Errorf("unexpected error: %+v", err) 99 | } else if err == nil && tc.errStr != "" { 100 | t.Errorf("expected error %q but got nil", tc.errStr) 101 | } else if tc.errStr != "" && err.Error() != tc.errStr { 102 | t.Errorf("expected error %q but got %q", tc.errStr, err) 103 | } 104 | 105 | currSeq := slog.Seq() 106 | v, err := slog.Get(currSeq) 107 | r.NoError(err) 108 | r.NotNil(v) 109 | v, err = slog.Get(currSeq + 1) 110 | r.Error(err) 111 | r.Equal(luigi.EOS{}, err) 112 | r.Nil(v) 113 | } 114 | 115 | r.NoError(mlog.Close(), "failed to close testlog") 116 | 117 | if t.Failed() { 118 | t.Log("db location:", dir) 119 | } else { 120 | os.RemoveAll(dir) 121 | } 122 | } 123 | } 124 | 125 | tcs := []testcase{ 126 | { 127 | tipe: int64(0), 128 | specs: []margaret.QuerySpec{margaret.Live(true)}, 129 | live: true, 130 | values: map[indexes.Addr][]interface{}{ 131 | indexes.Addr([]byte{0, 0, 0, 2}): {2, 4, 6, 8, 10, 12, 14, 16, 18}, 132 | indexes.Addr([]byte{0, 0, 0, 3}): {3, 6, 9, 12, 15, 18}, 133 | indexes.Addr([]byte{0, 0, 0, 4}): {4, 8, 12, 16}, 134 | indexes.Addr([]byte{0, 0, 0, 5}): {5, 10, 15}, 135 | indexes.Addr([]byte{0, 0, 0, 6}): {6, 12, 18}, 136 | indexes.Addr([]byte{0, 0, 0, 7}): {7, 14}, 137 | indexes.Addr([]byte{0, 0, 0, 8}): {8, 16}, 138 | indexes.Addr([]byte{0, 0, 0, 9}): {9, 18}, 139 | indexes.Addr([]byte{0, 0, 0, 10}): {10}, 140 | indexes.Addr([]byte{0, 0, 0, 11}): {11}, 141 | indexes.Addr([]byte{0, 0, 0, 12}): {12}, 142 | indexes.Addr([]byte{0, 0, 0, 12}): {12}, 143 | indexes.Addr([]byte{0, 0, 0, 13}): {13}, 144 | indexes.Addr([]byte{0, 0, 0, 14}): {14}, 145 | indexes.Addr([]byte{0, 0, 0, 15}): {15}, 146 | indexes.Addr([]byte{0, 0, 0, 16}): {16}, 147 | indexes.Addr([]byte{0, 0, 0, 17}): {17}, 148 | indexes.Addr([]byte{0, 0, 0, 18}): {18}, 149 | indexes.Addr([]byte{0, 0, 0, 19}): {19}, 150 | }, 151 | }, 152 | } 153 | 154 | return func(t *testing.T) { 155 | for i, tc := range tcs { 156 | t.Run(fmt.Sprint(i), mkTest(tc)) 157 | } 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /offset2/rw_test.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package offset2 6 | 7 | import ( 8 | "context" 9 | "io/ioutil" 10 | "os" 11 | "testing" 12 | 13 | "github.com/ssbc/go-luigi" 14 | mjson "github.com/ssbc/margaret/codec/json" 15 | "github.com/stretchr/testify/require" 16 | ) 17 | 18 | type testEvent struct { 19 | Foo string `json:",omitempty"` 20 | Bar int `json:",omitempty"` 21 | } 22 | 23 | func TestReadWrite(t *testing.T) { 24 | //setup 25 | r := require.New(t) 26 | name, err := ioutil.TempDir("", t.Name()) 27 | r.NoError(err) 28 | 29 | log, err := Open(name, mjson.New(&testEvent{})) 30 | r.NoError(err, "error during log creation") 31 | 32 | // cleanup 33 | defer func() { 34 | if t.Failed() { 35 | t.Logf("log data directory at %q was not deleted due to test failure", name) 36 | } else { 37 | os.RemoveAll(name) 38 | } 39 | }() 40 | 41 | // fill 42 | tevs := []testEvent{ 43 | testEvent{"hello", 23}, 44 | testEvent{"world", 42}, 45 | testEvent{"world", 161}, 46 | testEvent{"world", 1312}, 47 | } 48 | for i, ev := range tevs { 49 | seq, err := log.Append(ev) 50 | r.NoError(err, "failed to append event %d", i) 51 | r.Equal(int64(i), seq, "sequence missmatch") 52 | } 53 | 54 | // read 55 | for i := 0; i < len(tevs); i++ { 56 | v, err := log.Get(int64(i)) 57 | r.NoError(err, "failed to get event %d", i) 58 | 59 | ev, ok := v.(*testEvent) 60 | r.True(ok, "failed to cast event %d. got %T", i, v) 61 | r.Equal(*ev, tevs[i]) 62 | } 63 | } 64 | 65 | // make sure that the sequence is picked up after opening an existing log 66 | func TestWriteAndWriteAgain(t *testing.T) { 67 | //setup 68 | r := require.New(t) 69 | name, err := ioutil.TempDir("", t.Name()) 70 | r.NoError(err) 71 | 72 | log, err := Open(name, mjson.New(&testEvent{})) 73 | r.NoError(err, "error during log creation") 74 | 75 | // fill 76 | tevs := []testEvent{ 77 | testEvent{"hello", 23}, 78 | testEvent{"world", 42}, 79 | testEvent{"world", 161}, 80 | testEvent{"world", 1312}, 81 | } 82 | for i, ev := range tevs { 83 | seq, err := log.Append(ev) 84 | r.NoError(err, "failed to append event %d", i) 85 | r.Equal(int64(i), seq, "sequence missmatch") 86 | } 87 | 88 | log, err = Open(name, mjson.New(&testEvent{})) 89 | r.NoError(err, "error during log creation") 90 | // fill again 91 | for i, ev := range tevs { 92 | seq, err := log.Append(ev) 93 | r.NoError(err, "failed to do 2nd append %d", i) 94 | r.Equal(int64(len(tevs)+i), seq, "sequence missmatch %d", i) 95 | } 96 | 97 | // close 98 | r.NoError(log.Close()) 99 | 100 | _, err = log.Append(23) 101 | r.NotNil(err) 102 | 103 | log, err = Open(name, mjson.New(&testEvent{})) 104 | r.NoError(err, "error during log creation") 105 | 106 | currSeq := log.Seq() 107 | r.NoError(err, "failed to get current sequence") 108 | r.EqualValues(int64(2*len(tevs)-1), currSeq) 109 | 110 | // read by seq 111 | for i := 0; i < 2*len(tevs); i++ { 112 | v, err := log.Get(int64(i)) 113 | r.NoError(err, "failed to get event %d", i) 114 | 115 | ev, ok := v.(*testEvent) 116 | r.True(ok, "failed to cast event %d. got %T", i, v) 117 | r.Equal(*ev, tevs[i%len(tevs)]) 118 | } 119 | 120 | src, err := log.Query() 121 | r.NoError(err, "failed to open query") 122 | var ( 123 | ctx = context.TODO() 124 | seq int64 125 | ) 126 | for { 127 | v, err := src.Next(ctx) 128 | if luigi.IsEOS(err) { 129 | break 130 | } else if err != nil { 131 | r.NoError(err, "error during next draining") 132 | } 133 | t.Log(v, seq) 134 | seq++ 135 | // TODO: v has no sequence unless we put it in the values ourselvs..? 136 | } 137 | 138 | r.NoError(log.Close()) 139 | // cleanup 140 | if t.Failed() { 141 | t.Log("log was written to ", name) 142 | } else { 143 | os.RemoveAll(name) 144 | } 145 | } 146 | 147 | // should be able to recover from journal in the future 148 | func TestRecover(t *testing.T) { 149 | //setup 150 | r := require.New(t) 151 | name, err := ioutil.TempDir("", t.Name()) 152 | r.NoError(err) 153 | 154 | log, err := Open(name, mjson.New(&testEvent{})) 155 | r.NoError(err, "error during log creation") 156 | 157 | // fill 158 | tevs := []testEvent{ 159 | testEvent{"hello", 23}, 160 | testEvent{"world", 42}, 161 | testEvent{"world", 161}, 162 | testEvent{"world", 1312}, 163 | } 164 | for i, ev := range tevs { 165 | seq, err := log.Append(ev) 166 | r.NoError(err, "failed to append event %d", i) 167 | r.Equal(int64(i), seq, "sequence missmatch") 168 | } 169 | 170 | // close 171 | r.NoError(log.Close()) 172 | 173 | // reopen and corrupt 174 | log, err = Open(name, mjson.New(&testEvent{})) 175 | r.NoError(err, "error during log open") 176 | 177 | // assuming journal was increased only 178 | seq, err := log.jrnl.bump() 179 | r.NoError(err) 180 | r.EqualValues(seq, len(tevs)) // +1-1 181 | 182 | r.NoError(log.Close()) 183 | 184 | log, err = Open(name, mjson.New(&testEvent{})) 185 | r.NoError(err, "error while recover") 186 | r.NotNil(log) 187 | 188 | v := log.Seq() 189 | r.NoError(err, "error while recover") 190 | r.EqualValues(v, len(tevs)-1) 191 | } 192 | -------------------------------------------------------------------------------- /indexes/mkv/index.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package mkv 6 | 7 | import ( 8 | "context" 9 | "encoding" 10 | "encoding/binary" 11 | "encoding/json" 12 | "errors" 13 | "fmt" 14 | "math" 15 | "reflect" 16 | "sync" 17 | 18 | "github.com/ssbc/go-luigi" 19 | "github.com/ssbc/margaret" 20 | "github.com/ssbc/margaret/indexes" 21 | "modernc.org/kv" 22 | ) 23 | 24 | func NewIndex(db *kv.DB, tipe interface{}) indexes.SeqSetterIndex { 25 | return &index{ 26 | db: db, 27 | tipe: tipe, 28 | obvs: make(map[indexes.Addr]luigi.Observable), 29 | curSeq: int64(-2), 30 | } 31 | } 32 | 33 | type index struct { 34 | l sync.Mutex 35 | db *kv.DB 36 | obvs map[indexes.Addr]luigi.Observable 37 | tipe interface{} 38 | curSeq int64 39 | } 40 | 41 | func (idx *index) Flush() error { return nil } 42 | 43 | func (idx *index) Close() error { return idx.db.Close() } 44 | 45 | func (idx *index) Get(ctx context.Context, addr indexes.Addr) (luigi.Observable, error) { 46 | idx.l.Lock() 47 | defer idx.l.Unlock() 48 | 49 | obv, ok := idx.obvs[addr] 50 | if ok { 51 | return obv, nil 52 | } 53 | 54 | t := reflect.TypeOf(idx.tipe) 55 | v := reflect.New(t).Interface() 56 | 57 | data, err := idx.db.Get(nil, []byte(addr)) 58 | if data == nil { 59 | obv := indexes.NewObservable(indexes.UnsetValue{addr}, idx.deleter(addr)) 60 | idx.obvs[addr] = obv 61 | return roObv{obv}, nil 62 | } 63 | if err != nil { 64 | return nil, fmt.Errorf("error loading data from store:%w", err) 65 | } 66 | 67 | if um, ok := v.(encoding.BinaryUnmarshaler); ok { 68 | if t.Kind() != reflect.Ptr { 69 | v = reflect.ValueOf(v).Elem().Interface() 70 | } 71 | 72 | err = um.UnmarshalBinary(data) 73 | return nil, fmt.Errorf("error unmarshaling using custom marshaler:%w", err) 74 | } 75 | 76 | err = json.Unmarshal(data, v) 77 | if err != nil { 78 | return nil, fmt.Errorf("error unmarshaling using json marshaler:%w", err) 79 | } 80 | 81 | if t.Kind() != reflect.Ptr { 82 | v = reflect.ValueOf(v).Elem().Interface() 83 | } 84 | 85 | obv = indexes.NewObservable(v, idx.deleter(addr)) 86 | idx.obvs[addr] = obv 87 | return roObv{obv}, nil 88 | } 89 | 90 | func (idx *index) deleter(addr indexes.Addr) func() { 91 | return func() { 92 | delete(idx.obvs, addr) 93 | } 94 | } 95 | 96 | func (idx *index) Set(ctx context.Context, addr indexes.Addr, v interface{}) error { 97 | var ( 98 | raw []byte 99 | err error 100 | ) 101 | 102 | if m, ok := v.(encoding.BinaryMarshaler); ok { 103 | raw, err = m.MarshalBinary() 104 | if err != nil { 105 | return fmt.Errorf("error marshaling value using custom marshaler:%w", err) 106 | } 107 | } else { 108 | raw, err = json.Marshal(v) 109 | if err != nil { 110 | return fmt.Errorf("error marshaling value using json marshaler:%w", err) 111 | } 112 | } 113 | 114 | err = idx.db.Set([]byte(addr), raw) 115 | if err != nil { 116 | return fmt.Errorf("error in store:%w", err) 117 | } 118 | 119 | idx.l.Lock() 120 | defer idx.l.Unlock() 121 | 122 | obv, ok := idx.obvs[addr] 123 | if ok { 124 | err = obv.Set(v) 125 | if err != nil { 126 | return fmt.Errorf("error setting value in observable:%w", err) 127 | } 128 | } 129 | 130 | return nil 131 | } 132 | 133 | func (idx *index) Delete(ctx context.Context, addr indexes.Addr) error { 134 | err := idx.db.Delete([]byte(addr)) 135 | if err != nil { 136 | return fmt.Errorf("error in store:%w", err) 137 | } 138 | 139 | idx.l.Lock() 140 | defer idx.l.Unlock() 141 | 142 | obv, ok := idx.obvs[addr] 143 | if ok { 144 | err = obv.Set(indexes.UnsetValue{addr}) 145 | if err != nil { 146 | return fmt.Errorf("error setting value in observable:%w", err) 147 | } 148 | } 149 | 150 | return nil 151 | } 152 | 153 | func (idx *index) SetSeq(seq int64) error { 154 | var ( 155 | raw = make([]byte, 8) 156 | err error 157 | addr indexes.Addr = "__current_observable" 158 | ) 159 | 160 | binary.BigEndian.PutUint64(raw, uint64(seq)) 161 | 162 | err = idx.db.Set([]byte(addr), raw) 163 | if err != nil { 164 | return fmt.Errorf("error during mkv update (%T): %w", seq, err) 165 | } 166 | 167 | idx.l.Lock() 168 | defer idx.l.Unlock() 169 | 170 | idx.curSeq = seq 171 | 172 | return nil 173 | } 174 | 175 | func (idx *index) GetSeq() (int64, error) { 176 | var addr = "__current_observable" 177 | 178 | idx.l.Lock() 179 | defer idx.l.Unlock() 180 | 181 | if idx.curSeq != -2 { 182 | return idx.curSeq, nil 183 | } 184 | 185 | data, err := idx.db.Get(nil, []byte(addr)) 186 | if err != nil { 187 | return margaret.SeqErrored, fmt.Errorf("error getting item:%w", err) 188 | } 189 | if data == nil { 190 | return margaret.SeqEmpty, nil 191 | } 192 | 193 | if l := len(data); l != 8 { 194 | return margaret.SeqErrored, fmt.Errorf("expected data of length 8, got %v", l) 195 | } 196 | 197 | val := binary.BigEndian.Uint64(data) 198 | if val > math.MaxInt64 { 199 | return margaret.SeqErrored, fmt.Errorf("current value bigger then sequence (int64)") 200 | } 201 | 202 | idx.curSeq = int64(val) 203 | 204 | return idx.curSeq, nil 205 | } 206 | 207 | type roObv struct { 208 | luigi.Observable 209 | } 210 | 211 | func (obv roObv) Set(interface{}) error { 212 | return errors.New("read-only observable") 213 | } 214 | -------------------------------------------------------------------------------- /offset2/test/pump.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test 6 | 7 | import ( 8 | "context" 9 | "os" 10 | "testing" 11 | 12 | "github.com/pkg/errors" 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/require" 15 | 16 | "github.com/ssbc/go-luigi" 17 | "github.com/ssbc/margaret" 18 | mtest "github.com/ssbc/margaret/test" 19 | ) 20 | 21 | func LogTestPump(f mtest.NewLogFunc) func(*testing.T) { 22 | type testcase struct { 23 | name string 24 | tipe interface{} 25 | values []interface{} 26 | specs []margaret.QuerySpec 27 | result []interface{} 28 | errStr string 29 | live bool 30 | seqWrap bool 31 | qryTime int 32 | } 33 | 34 | mkTest := func(tc testcase) func(*testing.T) { 35 | return func(t *testing.T) { 36 | a := assert.New(t) 37 | r := require.New(t) 38 | 39 | log, err := f(t.Name(), tc.tipe) 40 | r.NoError(err, "error creating log") 41 | r.NotNil(log, "returned log is nil") 42 | 43 | defer func() { 44 | if namer, ok := log.(interface{ FileName() string }); ok { 45 | r.NoError(os.RemoveAll(namer.FileName()), "error deleting log after test") 46 | } 47 | }() 48 | 49 | ctx := context.Background() 50 | ctx, cancel := context.WithCancel(ctx) 51 | defer cancel() 52 | 53 | var iRes int 54 | var closed bool 55 | sink := luigi.FuncSink(func(ctx context.Context, v_ interface{}, err error) error { 56 | if err != nil { 57 | if err != (luigi.EOS{}) { 58 | t.Log("sink closed with non-EOS error:", err) 59 | } 60 | 61 | if closed { 62 | return errors.New("closing closed sink") 63 | } 64 | closed = true 65 | if iRes != len(tc.result) { 66 | t.Errorf("early end of stream at %d instead of %d", iRes, len(tc.result)) 67 | } 68 | 69 | return nil 70 | } 71 | 72 | if iRes >= len(tc.result) { 73 | t.Fatal("expected end but read value:", v_) 74 | } 75 | v := tc.result[iRes] 76 | iRes++ 77 | 78 | if tc.errStr == "" { 79 | if tc.seqWrap { 80 | sw := v.(margaret.SeqWrapper) 81 | sw_ := v_.(margaret.SeqWrapper) 82 | 83 | a.Equal(sw.Seq(), sw_.Seq(), "sequence number doesn't match") 84 | a.Equal(sw.Value(), sw_.Value(), "value doesn't match") 85 | } else { 86 | a.Equal(v, v_, "values don't match: %d", iRes) 87 | } 88 | } 89 | 90 | // iRes has been incremented so if at the end, it is the same as the length 91 | if tc.live && iRes == len(tc.result) { 92 | cancel() 93 | } 94 | return nil 95 | }) 96 | 97 | src, err := log.Query(tc.specs...) 98 | r.NoError(err, "error querying log") 99 | 100 | for i, v := range tc.values { 101 | seq, err := log.Append(v) 102 | r.NoError(err, "error appending to log") 103 | r.EqualValues(i, seq, "sequence missmatch") 104 | } 105 | 106 | if tc.live { 107 | cancel() 108 | } 109 | 110 | err = luigi.Pump(ctx, sink, src) 111 | if tc.live { 112 | a.Equal(context.Canceled, err, "stream copy error") 113 | } else { 114 | a.NoError(err, "stream copy error") 115 | } 116 | } 117 | } 118 | 119 | tcs := []testcase{ 120 | { 121 | name: "simple", 122 | tipe: 0, 123 | values: []interface{}{1, 2, 3}, 124 | result: []interface{}{1, 2, 3}, 125 | }, 126 | 127 | { 128 | name: "gt", 129 | tipe: 0, 130 | values: []interface{}{1, 2, 3}, 131 | result: []interface{}{2, 3}, 132 | specs: []margaret.QuerySpec{margaret.Gt(0)}, 133 | }, 134 | 135 | { 136 | name: "gte1", 137 | tipe: 0, 138 | values: []interface{}{1, 2, 3}, 139 | result: []interface{}{2, 3}, 140 | specs: []margaret.QuerySpec{margaret.Gte(1)}, 141 | }, 142 | 143 | { 144 | name: "lt2", 145 | tipe: 0, 146 | values: []interface{}{1, 2, 3}, 147 | result: []interface{}{1, 2}, 148 | specs: []margaret.QuerySpec{margaret.Lt(2)}, 149 | }, 150 | 151 | { 152 | name: "lte1", 153 | tipe: 0, 154 | values: []interface{}{1, 2, 3}, 155 | result: []interface{}{1, 2}, 156 | specs: []margaret.QuerySpec{margaret.Lte(1)}, 157 | }, 158 | 159 | { 160 | name: "limit2", 161 | tipe: 0, 162 | values: []interface{}{1, 2, 3}, 163 | result: []interface{}{1, 2}, 164 | specs: []margaret.QuerySpec{margaret.Limit(2)}, 165 | }, 166 | 167 | { 168 | name: "reverse", 169 | tipe: 0, 170 | values: []interface{}{5, 4, 3, 2, 1}, 171 | result: []interface{}{1, 2, 3, 4, 5}, 172 | specs: []margaret.QuerySpec{margaret.Reverse(true)}, 173 | }, 174 | 175 | { 176 | name: "live", 177 | tipe: 0, 178 | values: []interface{}{1, 2, 3}, 179 | result: []interface{}{1, 2, 3}, 180 | specs: []margaret.QuerySpec{margaret.Live(true)}, 181 | live: true, 182 | }, 183 | 184 | { 185 | name: "seqWrap", 186 | tipe: 0, 187 | values: []interface{}{1, 2, 3}, 188 | result: []interface{}{ 189 | margaret.WrapWithSeq(1, 0), 190 | margaret.WrapWithSeq(2, 1), 191 | margaret.WrapWithSeq(3, 2), 192 | }, 193 | specs: []margaret.QuerySpec{margaret.SeqWrap(true)}, 194 | seqWrap: true, 195 | }, 196 | } 197 | 198 | return func(t *testing.T) { 199 | for _, tc := range tcs { 200 | t.Run(tc.name, mkTest(tc)) 201 | } 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /multilog/roaring/multilog.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package roaring 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "fmt" 11 | stdlog "log" 12 | "sync" 13 | "time" 14 | 15 | "github.com/dgraph-io/sroar" 16 | "github.com/ssbc/go-luigi" 17 | "github.com/ssbc/margaret/indexes" 18 | 19 | "github.com/ssbc/margaret" 20 | "github.com/ssbc/margaret/internal/persist" 21 | "github.com/ssbc/margaret/internal/seqobsv" 22 | "github.com/ssbc/margaret/multilog" 23 | ) 24 | 25 | // NewStore returns a new multilog that is only good to store sequences 26 | // It uses files to store roaring bitmaps directly. 27 | // for this it turns the indexes.Addrs into a hex string. 28 | func NewStore(store persist.Saver) *MultiLog { 29 | ctx, cancel := context.WithCancel(context.TODO()) 30 | ml := &MultiLog{ 31 | store: store, 32 | l: &sync.Mutex{}, 33 | sublogs: make(map[indexes.Addr]*sublog), 34 | 35 | processing: ctx, 36 | done: cancel, 37 | batcherClosed: make(chan struct{}), 38 | tickPersist: time.NewTicker(13 * time.Second), 39 | } 40 | go ml.writeBatches() 41 | return ml 42 | } 43 | 44 | func (log *MultiLog) writeBatches() { 45 | for { 46 | select { 47 | case <-log.tickPersist.C: 48 | case <-log.processing.Done(): 49 | close(log.batcherClosed) 50 | return 51 | } 52 | err := log.Flush() 53 | if err != nil { 54 | stdlog.Println("flush trigger failed", err) 55 | } 56 | } 57 | } 58 | 59 | func (log *MultiLog) Flush() error { 60 | log.l.Lock() 61 | defer log.l.Unlock() 62 | return log.flushAllSublogs() 63 | } 64 | 65 | func (log *MultiLog) flushAllSublogs() error { 66 | var dirtySublogs []persist.KeyValuePair 67 | for addr, sublog := range log.sublogs { 68 | if sublog.dirty { 69 | dirtySublogs = append(dirtySublogs, persist.KeyValuePair{ 70 | Key: persist.Key(addr), 71 | Value: sublog.bmap.ToBuffer(), 72 | }) 73 | sublog.dirty = false 74 | } 75 | } 76 | 77 | err := log.store.PutMultiple(dirtySublogs) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | return nil 83 | } 84 | 85 | type MultiLog struct { 86 | store persist.Saver 87 | 88 | l *sync.Mutex 89 | sublogs map[indexes.Addr]*sublog 90 | 91 | processing context.Context 92 | done context.CancelFunc 93 | 94 | batcherClosed chan struct{} 95 | tickPersist *time.Ticker 96 | } 97 | 98 | func (log *MultiLog) Get(addr indexes.Addr) (margaret.Log, error) { 99 | log.l.Lock() 100 | defer log.l.Unlock() 101 | return log.openSublog(addr) 102 | } 103 | 104 | // openSublog alters the sublogs map, take the lock first! 105 | func (log *MultiLog) openSublog(addr indexes.Addr) (*sublog, error) { 106 | slog, has := log.sublogs[addr] 107 | if has { 108 | return slog, nil 109 | } 110 | 111 | pk := persist.Key(addr) 112 | 113 | var seq int64 114 | 115 | r, err := log.loadBitmap(pk) 116 | if errors.Is(err, persist.ErrNotFound) { 117 | seq = margaret.SeqEmpty 118 | r = sroar.NewBitmap() 119 | } else if err != nil { 120 | return nil, err 121 | } else { 122 | seq = int64(r.GetCardinality()) 123 | } 124 | 125 | var obsV uint64 126 | if seq > 0 { 127 | obsV = uint64(seq) 128 | } 129 | 130 | slog = &sublog{ 131 | mlog: log, 132 | key: pk, 133 | seq: seqobsv.New(obsV), 134 | luigiObsv: luigi.NewObservable(seq), 135 | bmap: r, 136 | } 137 | // the better idea is to have a store that can collece puts 138 | log.sublogs[addr] = slog 139 | return slog, nil 140 | } 141 | 142 | // LoadInternalBitmap loads the raw roaringbitmap for key 143 | func (log *MultiLog) LoadInternalBitmap(key indexes.Addr) (*sroar.Bitmap, error) { 144 | if err := log.Flush(); err != nil { 145 | return nil, err 146 | } 147 | bmap, err := log.loadBitmap([]byte(key)) 148 | if err != nil { 149 | if errors.Is(err, persist.ErrNotFound) { 150 | return nil, multilog.ErrSublogNotFound 151 | } 152 | return nil, err 153 | } 154 | return bmap, nil 155 | } 156 | 157 | func (log *MultiLog) loadBitmap(key []byte) (*sroar.Bitmap, error) { 158 | data, err := log.store.Get(key) 159 | if err != nil { 160 | return nil, fmt.Errorf("roaringfiles: invalid stored bitfield %s: %w", key, err) 161 | } 162 | 163 | return sroar.FromBuffer(data), nil 164 | } 165 | 166 | func (log *MultiLog) Delete(addr indexes.Addr) error { 167 | log.l.Lock() 168 | defer log.l.Unlock() 169 | 170 | if sl, ok := log.sublogs[addr]; ok { 171 | sl.deleted = true 172 | sl.luigiObsv.Set(multilog.ErrSublogDeleted) 173 | sl.seq = seqobsv.New(0) 174 | delete(log.sublogs, addr) 175 | } 176 | 177 | return log.store.Delete(persist.Key(addr)) 178 | } 179 | 180 | // List returns a list of all stored sublogs 181 | func (log *MultiLog) List() ([]indexes.Addr, error) { 182 | log.l.Lock() 183 | defer log.l.Unlock() 184 | 185 | err := log.loadAll() 186 | if err != nil { 187 | return nil, err 188 | } 189 | 190 | list := make([]indexes.Addr, len(log.sublogs)) 191 | i := 0 192 | for addr, sublog := range log.sublogs { 193 | if sublog.bmap.GetCardinality() == 0 { 194 | continue 195 | } 196 | list[i] = addr 197 | i++ 198 | } 199 | list = list[:i] // cut off the skipped ones 200 | 201 | return list, nil 202 | } 203 | 204 | func (log *MultiLog) loadAll() error { 205 | keys, err := log.store.List() 206 | if err != nil { 207 | return fmt.Errorf("roaringfiles: store iteration failed: %w", err) 208 | } 209 | for _, bk := range keys { 210 | _, err := log.openSublog(indexes.Addr(bk)) 211 | if err != nil { 212 | return fmt.Errorf("roaringfiles: broken bitmap file (%s): %w", bk, err) 213 | } 214 | } 215 | return nil 216 | } 217 | 218 | func (log *MultiLog) Close() error { 219 | log.done() 220 | log.tickPersist.Stop() 221 | <-log.batcherClosed 222 | 223 | if err := log.Flush(); err != nil { 224 | return fmt.Errorf("roaringfiles: close failed to flush: %w", err) 225 | } 226 | 227 | return log.store.Close() 228 | } 229 | -------------------------------------------------------------------------------- /test/simple.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test // import "github.com/ssbc/margaret/test" 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "os" 11 | "strings" 12 | "testing" 13 | "time" 14 | 15 | "github.com/stretchr/testify/assert" 16 | "github.com/stretchr/testify/require" 17 | 18 | "github.com/ssbc/go-luigi" 19 | "github.com/ssbc/margaret" 20 | ) 21 | 22 | func LogTestSimple(f NewLogFunc) func(*testing.T) { 23 | type testcase struct { 24 | name string 25 | tipe interface{} 26 | values []interface{} 27 | specs []margaret.QuerySpec 28 | result []interface{} 29 | errStr string 30 | live bool 31 | seqWrap bool 32 | } 33 | 34 | mkTest := func(tc testcase) func(*testing.T) { 35 | return func(t *testing.T) { 36 | a := assert.New(t) 37 | r := require.New(t) 38 | 39 | log, err := f(t.Name(), tc.tipe) 40 | r.NoError(err, "error creating log") 41 | r.NotNil(log, "returned log is nil") 42 | 43 | defer func() { 44 | if namer, ok := log.(interface{ FileName() string }); ok { 45 | r.NoError(os.RemoveAll(namer.FileName()), "error deleting log after test") 46 | } 47 | }() 48 | 49 | for i, v := range tc.values { 50 | seq, err := log.Append(v) 51 | r.NoError(err, "error appending to log") 52 | r.EqualValues(i, seq, "sequence missmatch") 53 | 54 | } 55 | 56 | src, err := log.Query(tc.specs...) 57 | r.NoError(err, "error querying log") 58 | 59 | ctx := context.Background() 60 | ctx, cancel := context.WithCancel(ctx) 61 | defer cancel() 62 | 63 | waiter := make(chan struct{}) 64 | var v_ interface{} 65 | err = nil 66 | 67 | for _, v := range tc.result { 68 | go func() { 69 | select { 70 | case <-time.After(time.Millisecond): 71 | t.Log("canceling context") 72 | cancel() 73 | case <-waiter: 74 | } 75 | }() 76 | 77 | v_, err = src.Next(ctx) 78 | if tc.errStr == "" { 79 | if tc.seqWrap { 80 | sw := v.(margaret.SeqWrapper) 81 | sw_ := v_.(margaret.SeqWrapper) 82 | 83 | a.Equal(sw.Seq(), sw_.Seq(), "sequence number doesn't match") 84 | a.Equal(sw.Value(), sw_.Value(), "value doesn't match") 85 | } else { 86 | a.EqualValues(v, v_, "values don't match") 87 | } 88 | } 89 | if err != nil { 90 | break 91 | } 92 | waiter <- struct{}{} 93 | } 94 | 95 | if err != nil && tc.errStr == "" { 96 | t.Errorf("unexpected error %+v", err) 97 | } else if err == nil && tc.errStr != "" { 98 | t.Errorf("expected error %q but got nil", tc.errStr) 99 | } else if tc.errStr != "" && err.Error() != tc.errStr { 100 | t.Errorf("expected error %q but got %q", tc.errStr, err) 101 | } 102 | 103 | go func() { 104 | select { 105 | case <-time.After(time.Millisecond): 106 | cancel() 107 | case <-waiter: 108 | } 109 | }() 110 | 111 | v, err := src.Next(ctx) 112 | if !tc.live && !luigi.IsEOS(err) { 113 | t.Errorf("expected end-of-stream but got %+v (value: %v)", err, v) 114 | } else if tc.live && !errors.Is(err, context.Canceled) { 115 | t.Errorf("expected context canceled but got %v, %+v", v, err) 116 | } 117 | 118 | select { 119 | case <-time.After(time.Millisecond): 120 | cancel() 121 | case waiter <- struct{}{}: 122 | } 123 | } 124 | } 125 | 126 | tcs := []testcase{ 127 | { 128 | name: "simple", 129 | tipe: 0, 130 | values: []interface{}{1, 2, 3}, 131 | result: []interface{}{1, 2, 3}, 132 | }, 133 | 134 | { 135 | name: "reverse", 136 | tipe: 0, 137 | values: []interface{}{1, 2, 3, 4, 5}, 138 | result: []interface{}{5, 4, 3, 2, 1}, 139 | specs: []margaret.QuerySpec{margaret.Reverse(true)}, 140 | }, 141 | 142 | { 143 | name: "reverse-false", 144 | tipe: 0, 145 | values: []interface{}{1, 2, 3, 4, 5}, 146 | result: []interface{}{1, 2, 3, 4, 5}, 147 | specs: []margaret.QuerySpec{margaret.Reverse(false)}, 148 | }, 149 | 150 | { 151 | name: "gt0", 152 | tipe: 0, 153 | values: []interface{}{1, 2, 3}, 154 | result: []interface{}{2, 3}, 155 | specs: []margaret.QuerySpec{margaret.Gt(0)}, 156 | }, 157 | 158 | { 159 | name: "gte1", 160 | tipe: 0, 161 | values: []interface{}{1, 2, 3}, 162 | result: []interface{}{2, 3}, 163 | specs: []margaret.QuerySpec{margaret.Gte(1)}, 164 | }, 165 | 166 | { 167 | name: "lt2", 168 | tipe: 0, 169 | values: []interface{}{1, 2, 3}, 170 | result: []interface{}{1, 2}, 171 | specs: []margaret.QuerySpec{margaret.Lt(2)}, 172 | }, 173 | 174 | { 175 | name: "lte1", 176 | tipe: 0, 177 | values: []interface{}{1, 2, 3}, 178 | result: []interface{}{1, 2}, 179 | specs: []margaret.QuerySpec{margaret.Lte(1)}, 180 | }, 181 | 182 | { 183 | name: "limit2", 184 | tipe: 0, 185 | values: []interface{}{1, 2, 3}, 186 | result: []interface{}{1, 2}, 187 | specs: []margaret.QuerySpec{margaret.Limit(2)}, 188 | }, 189 | 190 | { 191 | name: "EOS", 192 | tipe: 0, 193 | values: []interface{}{1, 2}, 194 | result: []interface{}{1, 2, 3}, 195 | errStr: "end of stream", 196 | }, 197 | 198 | // BUG(cryptix): the iterators needs to be improved to handle these correctly (https://github.com/ssbc/margaret/issues/6) 199 | // { 200 | // name: "reverse and gte", 201 | // tipe: 0, 202 | // values: []interface{}{1, 2, 3, 4, 5}, 203 | // result: []interface{}{5, 4, 3, 2}, 204 | // specs: []margaret.QuerySpec{margaret.Reverse(true), margaret.Gte(int64(2))}, 205 | // }, 206 | 207 | // { 208 | // name: "reverse and lt", 209 | // tipe: 0, 210 | // values: []interface{}{1, 2, 3, 4, 5}, 211 | // result: []interface{}{3, 2, 1}, 212 | // specs: []margaret.QuerySpec{margaret.Reverse(true), margaret.Lt(int64(4))}, 213 | // }, 214 | 215 | { 216 | name: "live", 217 | tipe: 0, 218 | values: []interface{}{1, 2, 3}, 219 | result: []interface{}{1, 2, 3}, 220 | specs: []margaret.QuerySpec{margaret.Live(true)}, 221 | live: true, 222 | }, 223 | 224 | { 225 | name: "seqWrap", 226 | tipe: 0, 227 | values: []interface{}{1, 2, 3}, 228 | result: []interface{}{ 229 | margaret.WrapWithSeq(1, 0), 230 | margaret.WrapWithSeq(2, 1), 231 | margaret.WrapWithSeq(3, 2), 232 | }, 233 | specs: []margaret.QuerySpec{margaret.SeqWrap(true)}, 234 | seqWrap: true, 235 | }, 236 | } 237 | 238 | return func(t *testing.T) { 239 | for _, tc := range tcs { 240 | t.Run(tc.name, mkTest(tc)) 241 | } 242 | 243 | t.Run("invalid querys", func(t *testing.T) { 244 | r := require.New(t) 245 | 246 | // "live and reverse" 247 | log, err := f(t.Name(), 0) 248 | r.NoError(err) 249 | 250 | _, err = log.Query(margaret.Live(true), margaret.Reverse(true)) 251 | r.Error(err) 252 | r.True(strings.Contains(err.Error(), ": can't do reverse and live")) 253 | }) 254 | } 255 | } 256 | -------------------------------------------------------------------------------- /multilog/test/sink.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package test // import "github.com/ssbc/margaret/multilog/test" 6 | 7 | import ( 8 | "context" 9 | "encoding/binary" 10 | "fmt" 11 | "io/ioutil" 12 | "os" 13 | "strings" 14 | "testing" 15 | "time" 16 | 17 | "github.com/pkg/errors" 18 | "github.com/stretchr/testify/assert" 19 | "github.com/stretchr/testify/require" 20 | 21 | "github.com/ssbc/go-luigi" 22 | "github.com/ssbc/margaret" 23 | "github.com/ssbc/margaret/indexes" 24 | "github.com/ssbc/margaret/multilog" 25 | ) 26 | 27 | func SinkTestSimple(f NewLogFunc) func(*testing.T) { 28 | type testcase struct { 29 | tipe interface{} 30 | values []interface{} 31 | specs []margaret.QuerySpec 32 | f func(t *testing.T) multilog.Func 33 | result map[indexes.Addr][]interface{} 34 | errStr string 35 | live bool 36 | seqWrap bool 37 | } 38 | 39 | mkTest := func(tc testcase) func(*testing.T) { 40 | return func(t *testing.T) { 41 | a := assert.New(t) 42 | r := require.New(t) 43 | ctx := context.Background() 44 | 45 | /* 46 | - make multilog and sink 47 | - query entire log and pump stream into multilog-sink 48 | - append values to log 49 | - check if multilog entries match 50 | */ 51 | 52 | // make multilog 53 | mlog, dir, err := f(t.Name(), tc.tipe, "") 54 | r.NoError(err, "error creating multilog") 55 | defer func() { 56 | err := mlog.Close() 57 | if err != nil { 58 | t.Error("mlog close", err) 59 | } 60 | if t.Failed() { 61 | t.Log("db location:", dir) 62 | } else { 63 | os.RemoveAll(dir) 64 | } 65 | }() 66 | 67 | // make file that tracks current sequence number 68 | prefix := "curSeq-" + strings.Replace(t.Name(), "/", "_", -1) + "-" 69 | file, err := ioutil.TempFile(".", prefix) 70 | r.NoError(err, "error creating curseq file") 71 | defer func() { 72 | err := os.Remove(file.Name()) 73 | if err != nil { 74 | t.Error("seq file rm", err) 75 | } 76 | }() 77 | 78 | sink := multilog.NewSink(file, mlog, tc.f(t)) 79 | 80 | // append values 81 | for i, v := range tc.values { 82 | err := sink.Pour(ctx, margaret.WrapWithSeq(v, int64(i))) 83 | a.NoError(err, "error pouring into sink") 84 | } 85 | 86 | // check if multilog entries match 87 | for addr, results := range tc.result { 88 | slog, err := mlog.Get(addr) 89 | r.NoError(err, "error getting sublog") 90 | r.NotNil(slog, "retrieved sublog is nil") 91 | 92 | src, err := slog.Query(tc.specs...) 93 | r.NoError(err, "error querying log") 94 | 95 | ctx, cancel := context.WithCancel(ctx) 96 | defer cancel() 97 | 98 | waiter := make(chan struct{}) 99 | var v_ interface{} 100 | err = nil 101 | 102 | for _, v := range results { 103 | go func() { 104 | select { 105 | case <-time.After(50 * time.Millisecond): 106 | t.Log("canceling context") 107 | cancel() 108 | case <-waiter: 109 | } 110 | }() 111 | func() { 112 | ctx, cancel := context.WithTimeout(ctx, 50*time.Millisecond) 113 | defer cancel() 114 | 115 | v_, err = src.Next(ctx) 116 | if tc.errStr == "" { 117 | if tc.seqWrap { 118 | sw := v.(margaret.SeqWrapper) 119 | sw_ := v_.(margaret.SeqWrapper) 120 | 121 | a.Equal(sw.Seq(), sw_.Seq(), "sequence number doesn't match") 122 | a.Equal(sw.Value(), sw_.Value(), "value doesn't match") 123 | } else { 124 | a.EqualValues(v, v_, "values don't match") 125 | } 126 | } 127 | }() 128 | if err != nil { 129 | break 130 | } 131 | waiter <- struct{}{} 132 | } 133 | 134 | if err != nil && tc.errStr == "" { 135 | t.Errorf("unexpected error: %+v", err) 136 | } else if err == nil && tc.errStr != "" { 137 | t.Errorf("expected error %q but got nil", tc.errStr) 138 | } else if tc.errStr != "" && err.Error() != tc.errStr { 139 | t.Errorf("expected error %q but got %q", tc.errStr, err) 140 | } 141 | 142 | go func() { 143 | select { 144 | case <-time.After(time.Millisecond): 145 | cancel() 146 | case <-waiter: 147 | } 148 | }() 149 | 150 | v, err := src.Next(ctx) 151 | if !tc.live && !luigi.IsEOS(err) { 152 | t.Errorf("expected end-of-stream but got %+v", err) 153 | } else if tc.live && !errors.Is(err, context.Canceled) { 154 | t.Errorf("expected context canceled but got %v, %+v", v, err) 155 | } 156 | 157 | select { 158 | case <-time.After(time.Millisecond): 159 | cancel() 160 | case waiter <- struct{}{}: 161 | } 162 | } 163 | } 164 | } 165 | 166 | tcs := []testcase{ 167 | { 168 | tipe: int64(0), 169 | values: count(0, 20), 170 | f: func(t *testing.T) multilog.Func { 171 | return func(ctx context.Context, seq int64, v interface{}, mlog multilog.MultiLog) (err error) { 172 | facs := uniq(factorize(int(v.(int64)))) 173 | for _, fac := range facs { 174 | prefixBs := make([]byte, 4) 175 | binary.BigEndian.PutUint32(prefixBs, uint32(fac)) 176 | prefix := indexes.Addr(prefixBs) 177 | 178 | var slog margaret.Log 179 | slog, err = mlog.Get(prefix) 180 | if err != nil { 181 | err = errors.Wrapf(err, "error getting sublog for prefix %d", fac) 182 | return err 183 | } 184 | 185 | _, err = slog.Append(seq) 186 | if err != nil { 187 | err = errors.Wrapf(err, "error appending to sublog for prefix %d", fac) 188 | return err 189 | } 190 | } 191 | 192 | err = nil 193 | return nil 194 | } 195 | }, 196 | specs: []margaret.QuerySpec{margaret.Live(true)}, 197 | live: true, 198 | result: map[indexes.Addr][]interface{}{ 199 | indexes.Addr([]byte{0, 0, 0, 2}): []interface{}{2, 4, 6, 8, 10, 12, 14, 16, 18}, 200 | indexes.Addr([]byte{0, 0, 0, 3}): []interface{}{3, 6, 9, 12, 15, 18}, 201 | indexes.Addr([]byte{0, 0, 0, 5}): []interface{}{5, 10, 15}, 202 | indexes.Addr([]byte{0, 0, 0, 7}): []interface{}{7, 14}, 203 | indexes.Addr([]byte{0, 0, 0, 11}): []interface{}{11}, 204 | indexes.Addr([]byte{0, 0, 0, 13}): []interface{}{13}, 205 | indexes.Addr([]byte{0, 0, 0, 17}): []interface{}{17}, 206 | indexes.Addr([]byte{0, 0, 0, 19}): []interface{}{19}, 207 | }, 208 | }, 209 | } 210 | 211 | return func(t *testing.T) { 212 | for i, tc := range tcs { 213 | t.Run(fmt.Sprint(i), mkTest(tc)) 214 | } 215 | } 216 | } 217 | 218 | func count(from, to int) []interface{} { 219 | out := make([]interface{}, to-from) 220 | for i := from; i < to; i++ { 221 | out[i-from] = int64(i) 222 | } 223 | return out 224 | } 225 | 226 | func factorize(n int) []int { 227 | if n == 0 { 228 | return nil 229 | } 230 | var out []int 231 | 232 | for i := 2; n != 1; i++ { 233 | for n != 0 && n%i == 0 { 234 | out = append(out, i) 235 | n = n / i 236 | } 237 | } 238 | 239 | return out 240 | } 241 | 242 | func uniq(ints []int) []int { 243 | var out []int 244 | 245 | for _, n := range ints { 246 | if len(out) == 0 { 247 | out = append(out, n) 248 | continue 249 | } 250 | 251 | if n != out[len(out)-1] { 252 | out = append(out, n) 253 | } 254 | } 255 | 256 | return out 257 | } 258 | -------------------------------------------------------------------------------- /LICENSES/CC0-1.0.txt: -------------------------------------------------------------------------------- 1 | Creative Commons Legal Code 2 | 3 | CC0 1.0 Universal 4 | 5 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE 6 | LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN 7 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS 8 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES 9 | REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS 10 | PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM 11 | THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED 12 | HEREUNDER. 13 | 14 | Statement of Purpose 15 | 16 | The laws of most jurisdictions throughout the world automatically confer 17 | exclusive Copyright and Related Rights (defined below) upon the creator 18 | and subsequent owner(s) (each and all, an "owner") of an original work of 19 | authorship and/or a database (each, a "Work"). 20 | 21 | Certain owners wish to permanently relinquish those rights to a Work for 22 | the purpose of contributing to a commons of creative, cultural and 23 | scientific works ("Commons") that the public can reliably and without fear 24 | of later claims of infringement build upon, modify, incorporate in other 25 | works, reuse and redistribute as freely as possible in any form whatsoever 26 | and for any purposes, including without limitation commercial purposes. 27 | These owners may contribute to the Commons to promote the ideal of a free 28 | culture and the further production of creative, cultural and scientific 29 | works, or to gain reputation or greater distribution for their Work in 30 | part through the use and efforts of others. 31 | 32 | For these and/or other purposes and motivations, and without any 33 | expectation of additional consideration or compensation, the person 34 | associating CC0 with a Work (the "Affirmer"), to the extent that he or she 35 | is an owner of Copyright and Related Rights in the Work, voluntarily 36 | elects to apply CC0 to the Work and publicly distribute the Work under its 37 | terms, with knowledge of his or her Copyright and Related Rights in the 38 | Work and the meaning and intended legal effect of CC0 on those rights. 39 | 40 | 1. Copyright and Related Rights. A Work made available under CC0 may be 41 | protected by copyright and related or neighboring rights ("Copyright and 42 | Related Rights"). Copyright and Related Rights include, but are not 43 | limited to, the following: 44 | 45 | i. the right to reproduce, adapt, distribute, perform, display, 46 | communicate, and translate a Work; 47 | ii. moral rights retained by the original author(s) and/or performer(s); 48 | iii. publicity and privacy rights pertaining to a person's image or 49 | likeness depicted in a Work; 50 | iv. rights protecting against unfair competition in regards to a Work, 51 | subject to the limitations in paragraph 4(a), below; 52 | v. rights protecting the extraction, dissemination, use and reuse of data 53 | in a Work; 54 | vi. database rights (such as those arising under Directive 96/9/EC of the 55 | European Parliament and of the Council of 11 March 1996 on the legal 56 | protection of databases, and under any national implementation 57 | thereof, including any amended or successor version of such 58 | directive); and 59 | vii. other similar, equivalent or corresponding rights throughout the 60 | world based on applicable law or treaty, and any national 61 | implementations thereof. 62 | 63 | 2. Waiver. To the greatest extent permitted by, but not in contravention 64 | of, applicable law, Affirmer hereby overtly, fully, permanently, 65 | irrevocably and unconditionally waives, abandons, and surrenders all of 66 | Affirmer's Copyright and Related Rights and associated claims and causes 67 | of action, whether now known or unknown (including existing as well as 68 | future claims and causes of action), in the Work (i) in all territories 69 | worldwide, (ii) for the maximum duration provided by applicable law or 70 | treaty (including future time extensions), (iii) in any current or future 71 | medium and for any number of copies, and (iv) for any purpose whatsoever, 72 | including without limitation commercial, advertising or promotional 73 | purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each 74 | member of the public at large and to the detriment of Affirmer's heirs and 75 | successors, fully intending that such Waiver shall not be subject to 76 | revocation, rescission, cancellation, termination, or any other legal or 77 | equitable action to disrupt the quiet enjoyment of the Work by the public 78 | as contemplated by Affirmer's express Statement of Purpose. 79 | 80 | 3. Public License Fallback. Should any part of the Waiver for any reason 81 | be judged legally invalid or ineffective under applicable law, then the 82 | Waiver shall be preserved to the maximum extent permitted taking into 83 | account Affirmer's express Statement of Purpose. In addition, to the 84 | extent the Waiver is so judged Affirmer hereby grants to each affected 85 | person a royalty-free, non transferable, non sublicensable, non exclusive, 86 | irrevocable and unconditional license to exercise Affirmer's Copyright and 87 | Related Rights in the Work (i) in all territories worldwide, (ii) for the 88 | maximum duration provided by applicable law or treaty (including future 89 | time extensions), (iii) in any current or future medium and for any number 90 | of copies, and (iv) for any purpose whatsoever, including without 91 | limitation commercial, advertising or promotional purposes (the 92 | "License"). The License shall be deemed effective as of the date CC0 was 93 | applied by Affirmer to the Work. Should any part of the License for any 94 | reason be judged legally invalid or ineffective under applicable law, such 95 | partial invalidity or ineffectiveness shall not invalidate the remainder 96 | of the License, and in such case Affirmer hereby affirms that he or she 97 | will not (i) exercise any of his or her remaining Copyright and Related 98 | Rights in the Work or (ii) assert any associated claims and causes of 99 | action with respect to the Work, in either case contrary to Affirmer's 100 | express Statement of Purpose. 101 | 102 | 4. Limitations and Disclaimers. 103 | 104 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 105 | surrendered, licensed or otherwise affected by this document. 106 | b. Affirmer offers the Work as-is and makes no representations or 107 | warranties of any kind concerning the Work, express, implied, 108 | statutory or otherwise, including without limitation warranties of 109 | title, merchantability, fitness for a particular purpose, non 110 | infringement, or the absence of latent or other defects, accuracy, or 111 | the present or absence of errors, whether or not discoverable, all to 112 | the greatest extent permissible under applicable law. 113 | c. Affirmer disclaims responsibility for clearing rights of other persons 114 | that may apply to the Work or any use thereof, including without 115 | limitation any person's Copyright and Related Rights in the Work. 116 | Further, Affirmer disclaims responsibility for obtaining any necessary 117 | consents, permissions or other rights required for any use of the 118 | Work. 119 | d. Affirmer understands and acknowledges that Creative Commons is not a 120 | party to this document and has no duty or obligation with respect to 121 | this CC0 or use of the Work. 122 | -------------------------------------------------------------------------------- /offset2/qry.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package offset2 // import "github.com/ssbc/margaret/offset2" 6 | 7 | import ( 8 | "context" 9 | "errors" 10 | "fmt" 11 | "io" 12 | "os" 13 | "sync" 14 | "syscall" 15 | 16 | "github.com/ssbc/go-luigi" 17 | "github.com/ssbc/margaret" 18 | ) 19 | 20 | type offsetQuery struct { 21 | l sync.Mutex 22 | log *OffsetLog 23 | codec margaret.Codec 24 | 25 | nextSeq, lt int64 26 | 27 | limit int 28 | live bool 29 | seqWrap bool 30 | reverse bool 31 | close chan struct{} 32 | err error 33 | } 34 | 35 | func (qry *offsetQuery) Gt(s int64) error { 36 | if qry.nextSeq > margaret.SeqEmpty { 37 | return fmt.Errorf("lower bound already set") 38 | } 39 | 40 | qry.nextSeq = int64(s + 1) 41 | return nil 42 | } 43 | 44 | func (qry *offsetQuery) Gte(s int64) error { 45 | if qry.nextSeq > margaret.SeqEmpty { 46 | return fmt.Errorf("lower bound already set") 47 | } 48 | 49 | qry.nextSeq = int64(s) 50 | return nil 51 | } 52 | 53 | func (qry *offsetQuery) Lt(s int64) error { 54 | if qry.lt != margaret.SeqEmpty { 55 | return fmt.Errorf("upper bound already set") 56 | } 57 | 58 | qry.lt = int64(s) 59 | return nil 60 | } 61 | 62 | func (qry *offsetQuery) Lte(s int64) error { 63 | if qry.lt != margaret.SeqEmpty { 64 | return fmt.Errorf("upper bound already set") 65 | } 66 | 67 | qry.lt = int64(s + 1) 68 | return nil 69 | } 70 | 71 | func (qry *offsetQuery) Limit(n int) error { 72 | qry.limit = n 73 | return nil 74 | } 75 | 76 | func (qry *offsetQuery) Live(live bool) error { 77 | qry.live = live 78 | return nil 79 | } 80 | 81 | func (qry *offsetQuery) SeqWrap(wrap bool) error { 82 | qry.seqWrap = wrap 83 | return nil 84 | } 85 | 86 | func (qry *offsetQuery) Reverse(yes bool) error { 87 | qry.reverse = yes 88 | if yes { 89 | if err := qry.setCursorToLast(); err != nil { 90 | return err 91 | } 92 | } 93 | return nil 94 | } 95 | 96 | func (qry *offsetQuery) setCursorToLast() error { 97 | qry.nextSeq = qry.log.seqCurrent 98 | return nil 99 | } 100 | 101 | func (qry *offsetQuery) Next(ctx context.Context) (interface{}, error) { 102 | qry.l.Lock() 103 | defer qry.l.Unlock() 104 | 105 | if qry.limit == 0 { 106 | return nil, luigi.EOS{} 107 | } 108 | qry.limit-- 109 | 110 | if qry.nextSeq == margaret.SeqEmpty { 111 | if qry.reverse { 112 | return nil, luigi.EOS{} 113 | } 114 | qry.nextSeq = 0 115 | } 116 | 117 | qry.log.l.Lock() 118 | defer qry.log.l.Unlock() 119 | 120 | if qry.lt != margaret.SeqEmpty && !(qry.nextSeq < qry.lt) { 121 | return nil, luigi.EOS{} 122 | } 123 | 124 | _, err := qry.log.readFrame(qry.nextSeq) 125 | if errors.Is(err, io.EOF) { 126 | if !qry.live { 127 | return nil, luigi.EOS{} 128 | } 129 | 130 | wait := make(chan struct{}) 131 | var cancel func() 132 | cancel = qry.log.seqChanges.Register(luigi.FuncSink( 133 | func(ctx context.Context, v interface{}, err error) error { 134 | if err != nil { 135 | return err 136 | } 137 | if v.(int64) >= qry.nextSeq { 138 | close(wait) 139 | cancel() 140 | } 141 | 142 | return nil 143 | })) 144 | 145 | err = func() error { 146 | qry.log.l.Unlock() 147 | defer qry.log.l.Lock() 148 | 149 | select { 150 | case <-wait: 151 | case <-ctx.Done(): 152 | return ctx.Err() 153 | } 154 | return nil 155 | }() 156 | if err != nil { 157 | return nil, err 158 | } 159 | } else if errors.Is(err, margaret.ErrNulled) { 160 | // TODO: qry.skipNulled 161 | qry.nextSeq++ 162 | return margaret.ErrNulled, nil 163 | } else if err != nil { 164 | return nil, err 165 | } 166 | 167 | // we waited until the value is in the log - now read it again 168 | 169 | v, err := qry.log.readFrame(qry.nextSeq) 170 | if errors.Is(err, io.EOF) { 171 | return nil, io.ErrUnexpectedEOF 172 | } else if err != nil { 173 | return nil, err 174 | } 175 | 176 | defer func() { 177 | if qry.reverse { 178 | qry.nextSeq-- 179 | } else { 180 | qry.nextSeq++ 181 | } 182 | }() 183 | 184 | if qry.seqWrap { 185 | return margaret.WrapWithSeq(v, qry.nextSeq), nil 186 | } 187 | 188 | return v, nil 189 | } 190 | 191 | func (qry *offsetQuery) Push(ctx context.Context, sink luigi.Sink) error { 192 | // first fast fwd's until we are up to date, 193 | // then hooks us into the live log updater. 194 | cancel, err := qry.fastFwdPush(ctx, sink) 195 | if err != nil { 196 | return err 197 | } 198 | 199 | defer cancel() 200 | 201 | // block until cancelled, then clean up and return 202 | select { 203 | case <-ctx.Done(): 204 | if qry.err != nil { 205 | return qry.err 206 | } 207 | 208 | return ctx.Err() 209 | case <-qry.close: 210 | return qry.err 211 | } 212 | } 213 | 214 | func (qry *offsetQuery) fastFwdPush(ctx context.Context, sink luigi.Sink) (func(), error) { 215 | qry.log.l.Lock() 216 | defer qry.log.l.Unlock() 217 | 218 | if qry.nextSeq == margaret.SeqEmpty { 219 | if qry.reverse { 220 | // reset since log is updated since the query was created 221 | if err := qry.setCursorToLast(); err != nil { 222 | return nil, err 223 | } 224 | } else { 225 | qry.nextSeq = 0 226 | } 227 | } 228 | 229 | // determines whether we should go on 230 | hasNext := func(seq int64) bool { 231 | return qry.limit != 0 && !(qry.lt >= 0 && seq >= qry.lt) 232 | } 233 | 234 | for hasNext(qry.nextSeq) { 235 | qry.limit-- 236 | 237 | // TODO: maybe don't read the frames individually but stream over them? 238 | // i.e. don't use ReadAt but have a separate fd just for this query 239 | // and just Read that. 240 | v, err := qry.log.readFrame(qry.nextSeq) 241 | if errors.Is(err, margaret.ErrNulled) { 242 | // TODO: if qry.skipNulls 243 | v = margaret.ErrNulled 244 | } else if err != nil { 245 | if !errors.Is(err, io.EOF) { 246 | var perr *os.PathError 247 | if errors.As(err, &perr) { 248 | if perr.Op == "seek" && (errors.Is(perr.Err, syscall.EINVAL) || errors.Is(perr.Err, os.ErrInvalid)) { 249 | // seeked passed the end == EOF 250 | break 251 | } 252 | } 253 | return func() {}, err 254 | } 255 | break 256 | } 257 | 258 | if qry.seqWrap { 259 | v = margaret.WrapWithSeq(v, qry.nextSeq) 260 | } 261 | err = sink.Pour(ctx, v) 262 | if err != nil { 263 | return nil, fmt.Errorf("error pouring read value of seq(%d): %w", qry.nextSeq, err) 264 | } 265 | 266 | if qry.reverse { 267 | qry.nextSeq-- 268 | } else { 269 | qry.nextSeq++ 270 | } 271 | } 272 | 273 | if !hasNext(qry.nextSeq) { 274 | close(qry.close) 275 | return func() {}, nil 276 | } 277 | 278 | if !qry.live { 279 | close(qry.close) 280 | return func() {}, nil 281 | } 282 | 283 | var cancel func() 284 | var closed bool 285 | cancel = qry.log.bcast.Register(LockSink(luigi.FuncSink(func(ctx context.Context, v interface{}, err error) error { 286 | if err != nil { 287 | if closed { 288 | return errors.New("closing closed sink") 289 | } 290 | 291 | closed = true 292 | select { 293 | case <-qry.close: 294 | default: 295 | close(qry.close) 296 | } 297 | 298 | return nil 299 | } 300 | 301 | sw := v.(margaret.SeqWrapper) 302 | v, seq := sw.Value(), sw.Seq() 303 | 304 | if !hasNext(seq) { 305 | close(qry.close) 306 | } 307 | 308 | if qry.seqWrap { 309 | v = sw 310 | } 311 | 312 | if err := sink.Pour(ctx, v); err != nil { 313 | return fmt.Errorf("offset2/push qry: pour of next live value failed: %w", err) 314 | } 315 | 316 | return nil 317 | }))) 318 | 319 | return cancel, nil 320 | } 321 | 322 | func LockSink(sink luigi.Sink) luigi.Sink { 323 | var l sync.Mutex 324 | 325 | return luigi.FuncSink(func(ctx context.Context, v interface{}, err error) error { 326 | l.Lock() 327 | defer l.Unlock() 328 | 329 | if err != nil { 330 | cwe, ok := sink.(interface{ CloseWithError(error) error }) 331 | if ok { 332 | return cwe.CloseWithError(err) 333 | } 334 | 335 | if err != (luigi.EOS{}) { 336 | fmt.Printf("was closed with error %q but underlying sink can not be closed with error\n", err) 337 | } 338 | 339 | return sink.Close() 340 | } 341 | 342 | return sink.Pour(ctx, v) 343 | }) 344 | } 345 | -------------------------------------------------------------------------------- /indexes/badger/index.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | package badger // import "github.com/ssbc/margaret/indexes/badger" 6 | 7 | import ( 8 | "context" 9 | "encoding" 10 | "encoding/binary" 11 | "encoding/json" 12 | "errors" 13 | "fmt" 14 | "log" 15 | "math" 16 | "os" 17 | "reflect" 18 | "strconv" 19 | "sync" 20 | "time" 21 | 22 | "github.com/dgraph-io/badger/v3" 23 | "github.com/ssbc/go-luigi" 24 | 25 | "github.com/ssbc/margaret" 26 | "github.com/ssbc/margaret/indexes" 27 | ) 28 | 29 | // badger starts to complain >100k 30 | var batchFullLimit uint32 = 75000 31 | 32 | // NASTY TESTING HACK 33 | func init() { 34 | limit, has := os.LookupEnv("LIBRARIAN_WRITEALL") 35 | if has { 36 | parsed, err := strconv.ParseUint(limit, 10, 32) 37 | if err != nil { 38 | panic(err) 39 | } 40 | log.Println("[margaret/indexes/badger] overwrote batch limit", parsed) 41 | batchFullLimit = uint32(parsed) 42 | } 43 | } 44 | 45 | type setOp struct { 46 | addr []byte 47 | val []byte 48 | } 49 | 50 | type index struct { 51 | stop context.CancelFunc 52 | running context.Context 53 | 54 | l *sync.Mutex 55 | 56 | // these control periodic persistence 57 | tickPersistAll, tickIfFull *time.Ticker 58 | 59 | batchLowerLimit uint // only write if there are more batches then this 60 | batchFullLimit uint32 // more than this cause an problem in badger 61 | 62 | nextbatch []setOp 63 | 64 | db *badger.DB 65 | keyPrefix []byte 66 | 67 | obvs map[indexes.Addr]luigi.Observable 68 | tipe interface{} 69 | curSeq int64 70 | } 71 | 72 | func NewIndex(db *badger.DB, tipe interface{}) indexes.SeqSetterIndex { 73 | return newIndex(db, tipe, []byte{}) 74 | } 75 | 76 | func NewIndexWithKeyPrefix(db *badger.DB, tipe interface{}, keyPrefix []byte) indexes.SeqSetterIndex { 77 | return newIndex(db, tipe, keyPrefix) 78 | } 79 | 80 | func newIndex(db *badger.DB, tipe interface{}, keyPrefix []byte) indexes.SeqSetterIndex { 81 | ctx, cancel := context.WithCancel(context.TODO()) 82 | idx := &index{ 83 | stop: cancel, 84 | running: ctx, 85 | 86 | l: &sync.Mutex{}, 87 | 88 | tickPersistAll: time.NewTicker(17 * time.Second), 89 | tickIfFull: time.NewTicker(5 * time.Second), 90 | 91 | batchLowerLimit: 32000, 92 | batchFullLimit: batchFullLimit, 93 | nextbatch: make([]setOp, 0), 94 | 95 | keyPrefix: keyPrefix, 96 | 97 | db: db, 98 | tipe: tipe, 99 | obvs: make(map[indexes.Addr]luigi.Observable), 100 | curSeq: int64(-2), 101 | } 102 | go idx.writeBatches() 103 | return idx 104 | } 105 | 106 | func (idx *index) Flush() error { 107 | idx.l.Lock() 108 | defer idx.l.Unlock() 109 | 110 | if err := idx.flushBatch(); err != nil { 111 | return err 112 | } 113 | return nil 114 | } 115 | 116 | func (idx *index) Close() error { 117 | idx.l.Lock() 118 | defer idx.l.Unlock() 119 | 120 | idx.stop() 121 | idx.tickIfFull.Stop() 122 | idx.tickPersistAll.Stop() 123 | 124 | err := idx.flushBatch() 125 | if err != nil { 126 | return fmt.Errorf("margaret/indexes/badger: failed to flush remaining batched operations: %w", err) 127 | } 128 | 129 | if len(idx.keyPrefix) == 0 { 130 | if err := idx.db.Close(); err != nil { 131 | return fmt.Errorf("margaret/indexes/badger: failed to close backing store: %w", err) 132 | } 133 | } 134 | 135 | return nil 136 | } 137 | 138 | func (idx *index) flushBatch() error { 139 | var raw = make([]byte, 8) 140 | err := idx.db.Update(func(txn *badger.Txn) error { 141 | useq := uint64(idx.curSeq) 142 | binary.BigEndian.PutUint64(raw, useq) 143 | 144 | err := txn.Set([]byte("__current_observable"), raw) 145 | if err != nil { 146 | return fmt.Errorf("error setting seq: %w", err) 147 | } 148 | 149 | for bi, op := range idx.nextbatch { 150 | err := txn.Set(op.addr, op.val) 151 | if err != nil { 152 | return fmt.Errorf("error setting batch #%d: %w", bi, err) 153 | } 154 | } 155 | return nil 156 | }) 157 | if err != nil { 158 | return fmt.Errorf("error in badger transaction (update) %d: %w", len(idx.nextbatch), err) 159 | 160 | } 161 | idx.nextbatch = []setOp{} 162 | return nil 163 | } 164 | 165 | func (idx *index) writeBatches() { 166 | 167 | for { 168 | var writeAll = false 169 | 170 | // if this was in the same select with the ticker below, 171 | // the ticker with the smaller durration would always overrule the longer one 172 | select { 173 | case <-idx.tickPersistAll.C: 174 | writeAll = true 175 | default: 176 | } 177 | 178 | select { 179 | case <-idx.tickIfFull.C: 180 | 181 | case <-idx.running.Done(): 182 | return 183 | } 184 | idx.l.Lock() 185 | n := uint(len(idx.nextbatch)) 186 | 187 | if !writeAll { 188 | if n < idx.batchLowerLimit { 189 | idx.l.Unlock() 190 | continue 191 | } 192 | } 193 | if n == 0 { 194 | idx.l.Unlock() 195 | continue 196 | } 197 | 198 | err := idx.flushBatch() 199 | if err != nil { 200 | // TODO: maybe set error and stop further writes? 201 | log.Println("margaret/indexes: flushing failed", err) 202 | } 203 | idx.l.Unlock() 204 | } 205 | } 206 | 207 | func (idx *index) Get(ctx context.Context, addr indexes.Addr) (luigi.Observable, error) { 208 | idx.l.Lock() 209 | defer idx.l.Unlock() 210 | 211 | obv, ok := idx.obvs[addr] 212 | if ok { 213 | return obv, nil 214 | } 215 | 216 | if err := idx.flushBatch(); err != nil { 217 | return nil, err 218 | } 219 | 220 | t := reflect.TypeOf(idx.tipe) 221 | v := reflect.New(t).Interface() 222 | 223 | err := idx.db.View(func(txn *badger.Txn) error { 224 | dbKey := append(idx.keyPrefix, addr...) 225 | item, err := txn.Get(dbKey) 226 | if err != nil { 227 | return fmt.Errorf("error getting item: %w", err) 228 | } 229 | 230 | err = item.Value(func(data []byte) error { 231 | if um, ok := v.(encoding.BinaryUnmarshaler); ok { 232 | if t.Kind() != reflect.Ptr { 233 | v = reflect.ValueOf(v).Elem().Interface() 234 | } 235 | 236 | err = um.UnmarshalBinary(data) 237 | return fmt.Errorf("error unmarshaling using custom marshaler: %w", err) 238 | } 239 | 240 | err = json.Unmarshal(data, v) 241 | if err != nil { 242 | return fmt.Errorf("error unmarshaling using json marshaler: %w", err) 243 | } 244 | 245 | if t.Kind() != reflect.Ptr { 246 | v = reflect.ValueOf(v).Elem().Interface() 247 | } 248 | return nil 249 | }) 250 | if err != nil { 251 | return fmt.Errorf("error getting value: %w", err) 252 | } 253 | 254 | return err 255 | }) 256 | 257 | if err != nil && !errors.Is(err, badger.ErrKeyNotFound) { 258 | return nil, fmt.Errorf("error in badger transaction (view): %w", err) 259 | } 260 | 261 | if errors.Is(err, badger.ErrKeyNotFound) { 262 | obv = indexes.NewObservable(indexes.UnsetValue{Addr: addr}, idx.deleter(addr)) 263 | } else { 264 | obv = indexes.NewObservable(v, idx.deleter(addr)) 265 | } 266 | 267 | idx.obvs[addr] = obv 268 | 269 | return roObv{obv}, nil 270 | } 271 | 272 | func (idx *index) deleter(addr indexes.Addr) func() { 273 | return func() { 274 | delete(idx.obvs, addr) 275 | } 276 | } 277 | 278 | func (idx *index) Set(ctx context.Context, addr indexes.Addr, v interface{}) error { 279 | var ( 280 | raw []byte 281 | err error 282 | ) 283 | 284 | if m, ok := v.(encoding.BinaryMarshaler); ok { 285 | raw, err = m.MarshalBinary() 286 | if err != nil { 287 | return fmt.Errorf("error marshaling value using custom marshaler: %w", err) 288 | } 289 | } else { 290 | raw, err = json.Marshal(v) 291 | if err != nil { 292 | return fmt.Errorf("error marshaling value using json marshaler: %w", err) 293 | } 294 | } 295 | idx.l.Lock() 296 | defer idx.l.Unlock() 297 | dbKey := append(idx.keyPrefix, addr...) 298 | batchedOp := setOp{ 299 | addr: dbKey, 300 | val: raw, 301 | } 302 | idx.nextbatch = append(idx.nextbatch, batchedOp) 303 | 304 | if n := uint32(len(idx.nextbatch)); n > idx.batchFullLimit { 305 | err = idx.flushBatch() 306 | if err != nil { 307 | return fmt.Errorf("failed to write big batch (%d): %w", n, err) 308 | } 309 | } 310 | 311 | obv, ok := idx.obvs[addr] 312 | if ok { 313 | err = obv.Set(v) 314 | if err != nil { 315 | return fmt.Errorf("error setting value in observable: %w", err) 316 | } 317 | } 318 | 319 | return nil 320 | } 321 | 322 | func (idx *index) Delete(ctx context.Context, addr indexes.Addr) error { 323 | err := idx.db.Update(func(txn *badger.Txn) error { 324 | dbKey := append(idx.keyPrefix, addr...) 325 | err := txn.Delete(dbKey) 326 | if err != nil { 327 | return fmt.Errorf("error deleting item: %w", err) 328 | } 329 | return nil 330 | }) 331 | if err != nil { 332 | return fmt.Errorf("error in badger transaction (update): %w", err) 333 | } 334 | 335 | idx.l.Lock() 336 | defer idx.l.Unlock() 337 | 338 | obv, ok := idx.obvs[addr] 339 | if ok { 340 | err = obv.Set(indexes.UnsetValue{Addr: addr}) 341 | if err != nil { 342 | return fmt.Errorf("error setting value in observable: %w", err) 343 | } 344 | } 345 | 346 | return nil 347 | } 348 | 349 | var currentSeqAddr = []byte("__current_observable") 350 | 351 | func (idx *index) SetSeq(seq int64) error { 352 | idx.l.Lock() 353 | defer idx.l.Unlock() 354 | 355 | dbKey := append(idx.keyPrefix, currentSeqAddr...) 356 | 357 | raw := make([]byte, 8) 358 | binary.BigEndian.PutUint64(raw, uint64(seq)) 359 | 360 | err := idx.db.Update(func(txn *badger.Txn) error { 361 | err := txn.Set(dbKey, raw) 362 | if err != nil { 363 | return fmt.Errorf("error during setSeq update: %w", err) 364 | } 365 | return nil 366 | }) 367 | if err != nil { 368 | return err 369 | } 370 | 371 | idx.curSeq = seq 372 | return nil 373 | } 374 | 375 | func (idx *index) GetSeq() (int64, error) { 376 | 377 | dbKey := append(idx.keyPrefix, currentSeqAddr...) 378 | 379 | idx.l.Lock() 380 | defer idx.l.Unlock() 381 | 382 | if idx.curSeq != -2 { 383 | return idx.curSeq, nil 384 | } 385 | 386 | err := idx.db.View(func(txn *badger.Txn) error { 387 | item, err := txn.Get(dbKey) 388 | if err != nil { 389 | return fmt.Errorf("error getting item: %w", err) 390 | } 391 | 392 | err = item.Value(func(data []byte) error { 393 | 394 | if l := len(data); l != 8 { 395 | return fmt.Errorf("expected data of length 8, got %v", l) 396 | } 397 | 398 | val := binary.BigEndian.Uint64(data) 399 | if val > math.MaxInt64 { 400 | return fmt.Errorf("current value bigger then sequence (int64)") 401 | } 402 | 403 | idx.curSeq = int64(val) 404 | 405 | return nil 406 | }) 407 | if err != nil { 408 | return fmt.Errorf("error getting value: %w", err) 409 | } 410 | 411 | return nil 412 | }) 413 | 414 | if err != nil { 415 | if errors.Is(err, badger.ErrKeyNotFound) { 416 | return margaret.SeqEmpty, nil 417 | } 418 | return 0, fmt.Errorf("error in badger transaction (view): %w", err) 419 | } 420 | 421 | return idx.curSeq, nil 422 | } 423 | 424 | type roObv struct { 425 | luigi.Observable 426 | } 427 | 428 | func (obv roObv) Set(interface{}) error { 429 | return errors.New("read-only observable") 430 | } 431 | -------------------------------------------------------------------------------- /offset2/log.go: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 The margaret Authors 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | /*Package offset2 implements a margaret log as persisted sequence of data across multiple files. 6 | 7 | Format Defintion 8 | 9 | A log consists of three files: data, ofst and jrnl. 10 | 11 | * data: a list of length-prefixed data chunks, size is a uint64 (size++[size]byte). 12 | 13 | * ofst: a list of uint64, representing entry offsets in 'data' 14 | 15 | * jrnl keeps track of the current sequence number, see checkJournal() for more 16 | 17 | To read entry 5 in `data`, you follow these steps: 18 | 19 | 1. Seek to 5*(sizeof(uint64)=8)=40 in `ofset` and read the uint64 representing the offset in `data` 20 | 21 | 2. Seek to that offset in `data`, read the length-prefix (the uint64 for the size of the entry) 22 | 23 | 3. Finally, read that amount of data, which is your entry 24 | 25 | All uint64's are encoded in BigEndian. 26 | 27 | */ 28 | package offset2 29 | 30 | import ( 31 | "bytes" 32 | "context" 33 | "encoding/binary" 34 | "errors" 35 | "fmt" 36 | "io" 37 | "os" 38 | "path/filepath" 39 | "sync" 40 | 41 | "github.com/ssbc/go-luigi" 42 | "github.com/ssbc/margaret" 43 | ) 44 | 45 | type OffsetLog struct { 46 | l sync.Mutex 47 | name string 48 | 49 | jrnl *journal 50 | ofst *offset 51 | data *data 52 | 53 | seqCurrent int64 54 | seqChanges luigi.Observable 55 | 56 | codec margaret.Codec 57 | 58 | bcast luigi.Broadcast 59 | bcSink luigi.Sink 60 | } 61 | 62 | func (log *OffsetLog) Close() error { 63 | // TODO: close open querys? 64 | // log.l.Lock() 65 | // defer log.l.Unlock() 66 | 67 | if err := log.jrnl.Close(); err != nil { 68 | return fmt.Errorf("journal file close failed: %w", err) 69 | } 70 | 71 | if err := log.ofst.Close(); err != nil { 72 | return fmt.Errorf("offset file close failed: %w", err) 73 | } 74 | 75 | if err := log.data.Close(); err != nil { 76 | return fmt.Errorf("data file close failed: %w", err) 77 | } 78 | 79 | if err := log.bcSink.Close(); err != nil { 80 | return fmt.Errorf("log broadcast close failed: %w", err) 81 | } 82 | 83 | return nil 84 | } 85 | 86 | var _ margaret.Alterer = (*OffsetLog)(nil) 87 | 88 | // Null overwrites the entry at seq with zeros 89 | // updating is kinda odd in append-only 90 | // but in some cases you still might want to redact entries 91 | func (log *OffsetLog) Null(seq int64) error { 92 | 93 | log.l.Lock() 94 | defer log.l.Unlock() 95 | 96 | ofst, err := log.ofst.readOffset(seq) 97 | if err != nil { 98 | return fmt.Errorf("null: error read offset: %w", err) 99 | } 100 | 101 | sz, err := log.data.getFrameSize(ofst) 102 | if err != nil { 103 | return fmt.Errorf("null: get frame size failed: %w", err) 104 | } 105 | 106 | if sz < 0 { // entry already nulled 107 | return nil 108 | } 109 | 110 | var minusSz bytes.Buffer 111 | err = binary.Write(&minusSz, binary.BigEndian, -sz) 112 | if err != nil { 113 | return fmt.Errorf("null: failed to encode neg size: %d: %w", -sz, err) 114 | } 115 | 116 | _, err = log.data.WriteAt(minusSz.Bytes(), ofst) 117 | if err != nil { 118 | return fmt.Errorf("null: failed to write -1 size bytes at %d: %w", ofst, err) 119 | } 120 | 121 | nulls := make([]byte, sz) 122 | _, err = log.data.WriteAt(nulls, ofst+8) 123 | if err != nil { 124 | return fmt.Errorf("null: failed to write %d bytes at %d: %w", sz, ofst, err) 125 | } 126 | 127 | return nil 128 | } 129 | 130 | // Replace overwrites the seq entry with data. 131 | // data has to be smaller then the current entry. 132 | func (log *OffsetLog) Replace(seq int64, data []byte) error { 133 | log.l.Lock() 134 | defer log.l.Unlock() 135 | 136 | ofst, err := log.ofst.readOffset(seq) 137 | if err != nil { 138 | return fmt.Errorf("offset2/replace: error read offset: %w", err) 139 | } 140 | 141 | sz, err := log.data.getFrameSize(ofst) 142 | if err != nil { 143 | return fmt.Errorf("offset2/replace: get frame size failed: %w", err) 144 | } 145 | 146 | newSz := int64(len(data)) 147 | if sz < newSz { 148 | return fmt.Errorf("offset2/replace: can't overwrite entry with larger data (diff:%d)", newSz-sz) 149 | } 150 | 151 | nulls := make([]byte, sz) 152 | copy(nulls[:], data) 153 | 154 | _, err = log.data.WriteAt(nulls, ofst+8) 155 | if err != nil { 156 | return fmt.Errorf("offset2/replace: null: failed to write %d bytes at %d: %w", sz, ofst, err) 157 | } 158 | 159 | return nil 160 | } 161 | 162 | // Open returns a the offset log in the directory at `name`. 163 | // If it is empty or does not exist, a new log will be created. 164 | func Open(name string, cdc margaret.Codec) (*OffsetLog, error) { 165 | err := os.MkdirAll(name, 0700) 166 | if err != nil { 167 | return nil, fmt.Errorf("offset2: error making log directory at %q: %w", name, err) 168 | } 169 | 170 | pLog := filepath.Join(name, "data") 171 | fData, err := os.OpenFile(pLog, os.O_CREATE|os.O_RDWR, 0600) 172 | if err != nil { 173 | return nil, fmt.Errorf("offset2: error opening log data file at %q: %w", pLog, err) 174 | } 175 | 176 | pOfst := filepath.Join(name, "ofst") 177 | fOfst, err := os.OpenFile(pOfst, os.O_CREATE|os.O_RDWR, 0600) 178 | if err != nil { 179 | return nil, fmt.Errorf("offset2: error opening log offset file at %q: %w", pOfst, err) 180 | } 181 | 182 | pJrnl := filepath.Join(name, "jrnl") 183 | fJrnl, err := os.OpenFile(pJrnl, os.O_CREATE|os.O_RDWR, 0600) 184 | if err != nil { 185 | return nil, fmt.Errorf("offset2: error opening log journal file at %q: %w", pJrnl, err) 186 | } 187 | 188 | log := &OffsetLog{ 189 | name: name, 190 | 191 | jrnl: &journal{fJrnl}, 192 | ofst: &offset{fOfst}, 193 | data: &data{File: fData}, 194 | 195 | codec: cdc, 196 | } 197 | 198 | _, err = log.checkJournal() 199 | if err != nil { 200 | return nil, fmt.Errorf("offset2: integrity error: %w", err) 201 | } 202 | 203 | log.bcSink, log.bcast = luigi.NewBroadcast() 204 | 205 | // get current sequence by end / blocksize 206 | end, err := fOfst.Seek(0, io.SeekEnd) 207 | if err != nil { 208 | return nil, fmt.Errorf("offset2: failed to seek to end of log-offset-file: %w", err) 209 | } 210 | // assumes -1 is SeqEmpty 211 | log.seqCurrent = (end / 8) - 1 212 | log.seqChanges = luigi.NewObservable(log.seqCurrent) 213 | 214 | return log, nil 215 | } 216 | 217 | // checkJournal verifies that the last entry is consistent along the three files. 218 | // - read sequence from journal 219 | // - read last offset from offset file 220 | // - read frame size from data file at previously read offset 221 | // - check that the end of the frame is also end of file 222 | // - check that number of entries in offset file equals value in journal 223 | func (log *OffsetLog) checkJournal() (int64, error) { 224 | seqJrnl, err := log.jrnl.readSeq() 225 | if err != nil { 226 | return margaret.SeqErrored, fmt.Errorf("error reading seq: %w", err) 227 | } 228 | 229 | if seqJrnl == margaret.SeqEmpty { 230 | statOfst, err := log.ofst.Stat() 231 | if err != nil { 232 | return margaret.SeqErrored, fmt.Errorf("stat failed on offset file: %w", err) 233 | } 234 | 235 | if statOfst.Size() != 0 { 236 | return margaret.SeqErrored, errors.New("journal empty but offset file isnt") 237 | } 238 | 239 | statData, err := log.data.Stat() 240 | if err != nil { 241 | return margaret.SeqErrored, fmt.Errorf("stat failed on data file: %w", err) 242 | } 243 | 244 | if statData.Size() != 0 { 245 | return margaret.SeqErrored, errors.New("journal empty but data file isnt") 246 | } 247 | 248 | return margaret.SeqEmpty, nil 249 | } 250 | 251 | ofstData, seqOfst, err := log.ofst.readLastOffset() 252 | if err != nil { 253 | return margaret.SeqErrored, fmt.Errorf("error reading last entry of log offset file: %w", err) 254 | } 255 | 256 | diff := seqJrnl - seqOfst 257 | if diff != 0 { 258 | if diff < 0 { // more data then entries in journal (unclear how to handle) 259 | // TODO: chop of data and offset to min(journal,count(ofst)) 260 | return margaret.SeqErrored, fmt.Errorf("seq in journal does not match element count in log offset file - %d != %d", seqJrnl, seqOfst) 261 | } 262 | 263 | // recover by truncating setting journal to count(ofst) 264 | _, err = log.jrnl.Seek(0, io.SeekStart) 265 | if err != nil { 266 | return margaret.SeqErrored, fmt.Errorf("recover: could not seek to start of journal file: %w", err) 267 | } 268 | 269 | err = binary.Write(log.jrnl, binary.BigEndian, seqOfst) 270 | if err != nil { 271 | return margaret.SeqErrored, fmt.Errorf("recover: could not overwrite journal with offset seq: %w", err) 272 | } 273 | 274 | if err := log.CheckConsistency(); err != nil { 275 | return margaret.SeqErrored, fmt.Errorf("recover: check journal 2nd pass: %w", err) 276 | } 277 | } 278 | 279 | sz, err := log.data.getFrameSize(ofstData) 280 | if err != nil { 281 | return margaret.SeqErrored, fmt.Errorf("error getting frame size from log data file: %w", err) 282 | } 283 | 284 | if sz < 0 { // entry nulled 285 | // irrelevant here, just treat the nulls as regular bytes 286 | sz = -sz 287 | } 288 | 289 | stat, err := log.data.Stat() 290 | if err != nil { 291 | return margaret.SeqErrored, fmt.Errorf("error stat'ing data file: %w", err) 292 | } 293 | 294 | n := ofstData + 8 + sz 295 | d := n - stat.Size() 296 | if d != 0 { 297 | // TODO: chop off the rest 298 | return margaret.SeqErrored, fmt.Errorf("data file size difference %d", d) 299 | } 300 | 301 | return seqJrnl, nil 302 | } 303 | 304 | // CheckConsistency is an fsck for the offset log. 305 | func (log *OffsetLog) CheckConsistency() error { 306 | _, err := log.checkJournal() 307 | if err != nil { 308 | return fmt.Errorf("offset2: journal inconsistent: %w", err) 309 | } 310 | 311 | var ( 312 | ofst, nextOfst int64 313 | seq int64 314 | ) 315 | 316 | for { 317 | sz, err := log.data.getFrameSize(nextOfst) 318 | if errors.Is(err, io.EOF) { 319 | return nil 320 | } else if err != nil { 321 | return fmt.Errorf("error getting frame size: %w", err) 322 | } 323 | 324 | ofst = nextOfst 325 | 326 | if sz < 0 { // TODO: nulled with user flags 327 | sz = -sz 328 | } 329 | 330 | nextOfst += sz + 8 // 8 byte length prefix 331 | 332 | expOfst, err := log.ofst.readOffset(seq) 333 | if errors.Is(err, io.EOF) { 334 | return nil 335 | } else if err != nil { 336 | return fmt.Errorf("error reading expected offset: %w", err) 337 | } 338 | 339 | if ofst != expOfst { 340 | return fmt.Errorf("offset mismatch: offset file says %d, data file has %d", expOfst, ofst) 341 | } 342 | seq++ 343 | } 344 | } 345 | 346 | func (log *OffsetLog) Seq() int64 { 347 | log.l.Lock() 348 | defer log.l.Unlock() 349 | return log.seqCurrent 350 | } 351 | 352 | func (log *OffsetLog) Changes() luigi.Observable { 353 | return log.seqChanges 354 | } 355 | 356 | func (log *OffsetLog) Get(seq int64) (interface{}, error) { 357 | log.l.Lock() 358 | defer log.l.Unlock() 359 | 360 | v, err := log.readFrame(seq) 361 | if err != nil { 362 | if errors.Is(err, io.EOF) { 363 | return v, luigi.EOS{} 364 | } 365 | if errors.Is(err, margaret.ErrNulled) { 366 | return nil, margaret.ErrNulled 367 | } 368 | return nil, err 369 | } 370 | return v, nil 371 | } 372 | 373 | // readFrame reads and parses a frame. 374 | func (log *OffsetLog) readFrame(seq int64) (interface{}, error) { 375 | ofst, err := log.ofst.readOffset(seq) 376 | if err != nil { 377 | return nil, fmt.Errorf("error read offset of seq(%d): %w", seq, err) 378 | } 379 | 380 | r, err := log.data.frameReader(ofst) 381 | if err != nil { 382 | return nil, fmt.Errorf("error getting frame reader for seq(%d) (ofst:%d): %w", seq, ofst, err) 383 | } 384 | 385 | dec := log.codec.NewDecoder(r) 386 | v, err := dec.Decode() 387 | if err != nil { 388 | if errors.Is(err, io.EOF) { 389 | return v, luigi.EOS{} 390 | } 391 | return nil, fmt.Errorf("error decoding data for seq(%d) (ofst:%d): %w", seq, ofst, err) 392 | } 393 | return v, nil 394 | } 395 | 396 | func (log *OffsetLog) Query(specs ...margaret.QuerySpec) (luigi.Source, error) { 397 | log.l.Lock() 398 | defer log.l.Unlock() 399 | 400 | qry := &offsetQuery{ 401 | log: log, 402 | codec: log.codec, 403 | 404 | nextSeq: margaret.SeqEmpty, 405 | lt: margaret.SeqEmpty, 406 | 407 | limit: -1, //i.e. no limit 408 | close: make(chan struct{}), 409 | } 410 | 411 | for _, spec := range specs { 412 | err := spec(qry) 413 | if err != nil { 414 | return nil, err 415 | } 416 | } 417 | 418 | if qry.reverse && qry.live { 419 | return nil, fmt.Errorf("offset2: can't do reverse and live") 420 | } 421 | 422 | return qry, nil 423 | } 424 | 425 | func (log *OffsetLog) Append(v interface{}) (int64, error) { 426 | data, err := log.codec.Marshal(v) 427 | if err != nil { 428 | return margaret.SeqEmpty, fmt.Errorf("offset2: error marshaling value: %w", err) 429 | } 430 | 431 | log.l.Lock() 432 | defer log.l.Unlock() 433 | 434 | jrnlSeq, err := log.jrnl.bump() 435 | if err != nil { 436 | return margaret.SeqEmpty, fmt.Errorf("offset2: error bumping journal: %w", err) 437 | } 438 | 439 | ofst, err := log.data.append(data) 440 | if err != nil { 441 | return margaret.SeqEmpty, fmt.Errorf("offset2: error appending data: %w", err) 442 | } 443 | 444 | seq, err := log.ofst.append(ofst) 445 | if err != nil { 446 | return margaret.SeqEmpty, fmt.Errorf("offset2: error appending offset: %w", err) 447 | } 448 | 449 | if seq != jrnlSeq { 450 | return margaret.SeqEmpty, fmt.Errorf("offset2: seq mismatch: journal wants %d, offset has %d", jrnlSeq, seq) 451 | } 452 | 453 | err = log.bcSink.Pour(context.TODO(), margaret.WrapWithSeq(v, jrnlSeq)) 454 | log.seqCurrent = seq 455 | log.seqChanges.Set(seq) 456 | 457 | if err != nil { 458 | return margaret.SeqEmpty, fmt.Errorf("offset2: error while updating registerd broadcasts with new value: %w", err) 459 | } 460 | 461 | return seq, nil 462 | } 463 | 464 | func (log *OffsetLog) FileName() string { 465 | return log.name 466 | } 467 | --------------------------------------------------------------------------------