├── testing ├── helpers │ ├── badger.go │ └── trie.go └── mocks │ ├── loader │ └── mock.go │ ├── feeder.go │ ├── record_streamer.go │ ├── wal_reader.go │ ├── cache.go │ ├── vm.go │ ├── invoker.go │ ├── record_holder.go │ ├── codec.go │ ├── forest │ └── mock.go │ └── chain.go ├── .gitignore ├── ledger ├── trie │ ├── errors.go │ ├── node.go │ ├── sort.go │ ├── leaf.go │ ├── group.go │ ├── light_trie.go │ ├── branch_internal_test.go │ ├── branch.go │ ├── light_node_test.go │ ├── extension.go │ └── extension_internal_test.go ├── wal │ ├── checksum.go │ └── encoding.go └── forest │ ├── light_forest_test.go │ ├── forest.go │ └── forest_integration_test.go ├── models ├── dps │ ├── errors.go │ ├── codec.go │ ├── chain.go │ ├── badger.go │ ├── writer.go │ └── reader.go └── convert │ ├── hash.go │ ├── hash_test.go │ ├── values.go │ ├── values_test.go │ ├── types.go │ ├── paths.go │ ├── types_test.go │ └── paths_test.go ├── service ├── feeder │ ├── reader.go │ ├── clone.go │ ├── feeder.go │ └── feeder_internal_test.go ├── invoker │ ├── cache.go │ ├── config.go │ ├── vm.go │ ├── read.go │ └── read_internal_test.go ├── mapper │ ├── loader.go │ ├── feeder.go │ ├── options.go │ ├── state_internal_test.go │ ├── forest.go │ ├── pathings.go │ ├── config_internal_test.go │ ├── status.go │ ├── state.go │ ├── config.go │ ├── fsm.go │ ├── pathings_internal_test.go │ └── fsm_internal_test.go ├── storage │ ├── library.go │ ├── prefixes.go │ ├── encoding.go │ ├── encoding_internal_test.go │ └── auxiliary.go ├── loader │ ├── empty.go │ ├── checkpoint.go │ ├── index.go │ └── config.go ├── tracker │ └── record.go ├── cloud │ └── config.go ├── index │ └── config.go ├── metrics │ ├── server.go │ └── badger.go └── initializer │ ├── protocol_state_test.go │ ├── protocol_state.go │ └── catchup_blocks.go ├── .github ├── pull_request_template.md └── workflows │ └── release.yml ├── docs ├── src │ ├── api.nomnoml │ ├── flow_dps_indexer.nomnoml │ └── flow_dps_live.nomnoml ├── snapshots.md └── architecture.md ├── cmd ├── flow-dps-server │ └── README.md ├── restore-index-snapshot │ └── README.md ├── flow-dps-indexer │ └── README.md ├── flow-dps-client │ ├── README.md │ └── sporks.go ├── dictionary-generator │ └── README.md ├── create-index-snapshot │ └── README.md ├── flow-dps-live │ └── README.md └── check-duplicate-transactions │ ├── index.go │ ├── main.go │ └── protocol.go ├── api └── dps │ └── api.generate.go ├── codec ├── generator │ ├── dictionary.go │ ├── train.go │ ├── benchmark.go │ ├── template.go │ └── config.go └── zbor │ └── events.go ├── .goreleaser.yml └── engine ├── component.go └── engine.go /testing/helpers/badger.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/dgraph-io/badger/v2" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func InMemoryDB(t *testing.T) *badger.DB { 11 | t.Helper() 12 | 13 | opts := badger.DefaultOptions("") 14 | opts.InMemory = true 15 | opts.Logger = nil 16 | 17 | db, err := badger.Open(opts) 18 | require.NoError(t, err) 19 | 20 | return db 21 | } 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /cmd/check-duplicate-transactions/check-duplicate-transactions 2 | /cmd/create-index-snapshot/create-index-snapshot 3 | /cmd/dictionary-generator/dictionary-generator 4 | /cmd/flow-dps-client/flow-dps-client 5 | /cmd/flow-dps-indexer/flow-dps-indexer 6 | /cmd/flow-dps-live/flow-dps-live 7 | /cmd/flow-dps-server/flow-dps-server 8 | /cmd/restore-index-snapshot/restore-index-snapshot 9 | /bootstrap/ 10 | /data/ 11 | /flow-go/ 12 | /index/ 13 | /trie/ 14 | /samples/ 15 | *.cdc 16 | *.checkpoint 17 | *.log 18 | -------------------------------------------------------------------------------- /ledger/trie/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Optakt Labs OÜ 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy of 5 | // the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | package trie 16 | 17 | import ( 18 | "errors" 19 | ) 20 | 21 | var ( 22 | ErrPathNotFound = errors.New("path not found") 23 | ) 24 | -------------------------------------------------------------------------------- /models/dps/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Optakt Labs OÜ 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy of 5 | // the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | package dps 16 | 17 | import ( 18 | "errors" 19 | ) 20 | 21 | // Sentinel errors. 22 | var ( 23 | ErrFinished = errors.New("finished") 24 | ErrUnavailable = errors.New("unavailable") 25 | ) 26 | -------------------------------------------------------------------------------- /service/feeder/reader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Optakt Labs OÜ 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy of 5 | // the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | package feeder 16 | 17 | // WALReader represents something that can read write-ahead log records. 18 | type WALReader interface { 19 | Next() bool 20 | Err() error 21 | Record() []byte 22 | } 23 | -------------------------------------------------------------------------------- /service/invoker/cache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Optakt Labs OÜ 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy of 5 | // the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | package invoker 16 | 17 | // Cache represents a key/value store to use as a cache. 18 | type Cache interface { 19 | Get(key interface{}) (interface{}, bool) 20 | Set(key, value interface{}, cost int64) bool 21 | } 22 | -------------------------------------------------------------------------------- /service/mapper/loader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Optakt Labs OÜ 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy of 5 | // the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | package mapper 16 | 17 | import ( 18 | "github.com/optakt/flow-dps/ledger/trie" 19 | ) 20 | 21 | // Loader represents something that loads its checkpoint and builds it into a trie. 22 | type Loader interface { 23 | Trie() (*trie.Trie, error) 24 | } 25 | -------------------------------------------------------------------------------- /ledger/trie/node.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Optakt Labs OÜ 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy of 5 | // the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | package trie 16 | 17 | import ( 18 | "golang.org/x/sync/semaphore" 19 | 20 | "github.com/onflow/flow-go/ledger/common/hash" 21 | ) 22 | 23 | // Node represents a trie node. 24 | type Node interface { 25 | Hash(sema *semaphore.Weighted, height int) hash.Hash 26 | } 27 | -------------------------------------------------------------------------------- /service/mapper/feeder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Optakt Labs OÜ 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy of 5 | // the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | package mapper 16 | 17 | import ( 18 | "github.com/onflow/flow-go/ledger" 19 | ) 20 | 21 | // Feeder represents something that can be consumed to get trie updates 22 | // in chronological order. 23 | type Feeder interface { 24 | Update() (*ledger.TrieUpdate, error) 25 | } 26 | -------------------------------------------------------------------------------- /service/mapper/options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Optakt Labs OÜ 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | // use this file except in compliance with the License. You may obtain a copy of 5 | // the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations under 13 | // the License. 14 | 15 | package mapper 16 | 17 | // WithTransition specifies which TransitionFunc should be used when the state machine 18 | // has the given status. 19 | func WithTransition(status Status, transition TransitionFunc) func(*FSM) { 20 | return func(f *FSM) { 21 | f.transitions[status] = transition 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Goal of this PR 2 | 3 | Fixes # 4 | 5 | ## Checklist 6 | 7 | ### Does the PR require tests to be added or updated? 8 | 9 | This is the case if this PR updates currently unit tested code with new/removed paths, or if it adds new components 10 | that should be tested. 11 | 12 | Ideally PRs should include the tests to code they add, but if for some reason they do not, there should be a `TODO` 13 | comment in the place where the test should be, linking to a GitHub issue that details what test should be written. 14 | 15 | ### Does the PR require documentation to be added or updated? 16 | 17 | This is the case if this PR changes any of the following: 18 | 19 | * Architecture redesign 20 | * API changes 21 | * Changes in CLI flags 22 | * New/Updated indexes 23 | 24 | ### Misc 25 | 26 | - [ ] PR title will be clear as part of the changelog 27 | - [ ] PR is against the correct branch 28 | - [ ] PR is labelled appropriately 29 | - [ ] PR is linked to an issue -------------------------------------------------------------------------------- /docs/src/api.nomnoml: -------------------------------------------------------------------------------- 1 | #title: api 2 | #direction: bottom 3 | #background: #262626 4 | #stroke: #fff 5 | #fill: #888 6 | 7 | [ Flow Network] 8 | [ Flow DPS Live] 9 | [ Flow DPS Indexer] 10 | [ Index] 11 | [ DPS API] 12 | [ DPS Access API] 13 | [ Rosetta API] 14 | 15 | [