├── .dockerignore ├── .github ├── dependabot.yml └── workflows │ ├── publish-grpc-libs.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── api ├── client │ ├── client.go │ ├── client_test.go │ ├── read.go │ └── write.go ├── pb │ ├── Makefile │ ├── dart │ │ ├── LICENSE │ │ ├── README.md │ │ ├── pubspec.lock │ │ └── pubspec.yaml │ ├── javascript │ │ ├── package-lock.json │ │ └── package.json │ ├── threads.pb.go │ ├── threads.proto │ └── threads_grpc.pb.go └── service.go ├── broadcast ├── broadcast.go └── broadcast_test.go ├── cbor ├── coding.go ├── event.go └── record.go ├── common └── common.go ├── core ├── app │ └── app.go ├── db │ └── db.go ├── doc.go ├── logstore │ └── logstore.go ├── net │ ├── event.go │ ├── net.go │ ├── options.go │ └── record.go └── thread │ ├── head.go │ ├── id.go │ ├── id_test.go │ ├── identity.go │ ├── key.go │ ├── key_test.go │ ├── protocol.go │ ├── varint.go │ └── varint_test.go ├── db ├── README.md ├── bench_test.go ├── collection.go ├── collection_test.go ├── common.go ├── criterion.go ├── criterion_test.go ├── db.go ├── db_test.go ├── design.png ├── dispatcher.go ├── dispatcher_test.go ├── encode.go ├── index.go ├── keytransform │ └── keytransform.go ├── listeners.go ├── manager.go ├── manager_test.go ├── options.go ├── query.go ├── query_more_test.go ├── query_test.go └── testutils_test.go ├── dist ├── README.md └── install ├── docker-compose-dev.yml ├── docker-compose.yml ├── go.mod ├── go.sum ├── integrationtests ├── foldersync │ ├── client.go │ ├── foldersync_test.go │ └── watcher │ │ └── watcher.go ├── migration │ ├── Makefile │ ├── generate │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ └── migration_test.go └── testground │ ├── README.md │ ├── baseline-docker.toml │ ├── current-docker.toml │ ├── current-exec-bitswap-sync-race.toml │ ├── current-exec-sync-threads.toml │ ├── go.mod │ ├── go.sum │ ├── head-docker.toml │ ├── main.go │ ├── manifest.toml │ └── measurement.go ├── jsonpatcher ├── jsonpatcher.go └── jsonpatcher_test.go ├── jwt ├── jwt.go └── jwt_test.go ├── logstore ├── logstore.go ├── lstoreds │ ├── README.md │ ├── addr_book.go │ ├── addr_book_gc.go │ ├── cache.go │ ├── cyclic_batch.go │ ├── ds_test.go │ ├── headbook.go │ ├── keybook.go │ ├── logstore.go │ └── metadata.go ├── lstorehybrid │ ├── hybrid_test.go │ └── logstore.go └── lstoremem │ ├── addr_book.go │ ├── headbook.go │ ├── inmem_test.go │ ├── keybook.go │ ├── logstore.go │ └── metadata.go ├── net ├── api │ ├── client │ │ ├── client.go │ │ └── client_test.go │ ├── pb │ │ ├── Makefile │ │ ├── javascript │ │ │ ├── package-lock.json │ │ │ └── package.json │ │ ├── threadsnet.pb.go │ │ ├── threadsnet.proto │ │ └── threadsnet_grpc.pb.go │ ├── service.go │ └── test_util.go ├── client.go ├── net.go ├── net_test.go ├── pb │ ├── Makefile │ ├── custom.go │ ├── lstore.pb.go │ ├── lstore.proto │ ├── lstorepb_test.go │ ├── net.pb.go │ ├── net.proto │ └── netpb_test.go ├── queue │ ├── common.go │ ├── ff.go │ ├── ff_test.go │ ├── tp.go │ └── tp_test.go ├── record.go ├── record_test.go ├── server.go └── util │ └── util.go ├── test ├── addr_book_suite.go ├── benchmarks_suite.go ├── headbook_suite.go ├── keybook_suite.go ├── logstore_suite.go ├── metadata_suite.go └── util.go ├── threadsd └── main.go └── util ├── dscopy └── main.go └── util.go /.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | 3 | .github 4 | dist 5 | .threads* 6 | .env 7 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/api/pb/javascript" 5 | schedule: 6 | interval: daily 7 | time: "03:00" 8 | timezone: America/Los_Angeles 9 | open-pull-requests-limit: 25 10 | ignore: 11 | - dependency-name: "@types/google-protobuf" 12 | versions: 13 | - 3.15.0 14 | - 3.15.1 15 | - dependency-name: google-protobuf 16 | versions: 17 | - 3.15.0 18 | - 3.15.1 19 | - 3.15.2 20 | - 3.15.3 21 | - 3.15.4 22 | - 3.15.5 23 | - 3.15.6 24 | - 3.15.7 25 | -------------------------------------------------------------------------------- /.github/workflows/publish-grpc-libs.yml: -------------------------------------------------------------------------------- 1 | name: Publish gRPC Libs 2 | on: 3 | release: 4 | types: [published] 5 | jobs: 6 | 7 | publish_grpc_libs: 8 | name: Publish gRPC Libs 9 | runs-on: ubuntu-latest 10 | container: 11 | image: google/dart:latest 12 | steps: 13 | - name: Check out code 14 | uses: actions/checkout@v1 15 | - name: Get latest tag 16 | id: latesttag 17 | run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} 18 | - name: Install build tools 19 | run: | 20 | apt-get update && apt-get install -y build-essential 21 | - name: Set up Go 22 | uses: actions/setup-go@v1 23 | with: 24 | go-version: 1.16 25 | - name: Setup env 26 | env: 27 | ACTIONS_ALLOW_UNSECURE_COMMANDS: true 28 | run: | 29 | echo "::set-env name=GOPATH::$(go env GOPATH)" 30 | echo "::add-path::$(go env GOPATH)/bin" 31 | - name: Set up Node 32 | uses: actions/setup-node@v1 33 | with: 34 | node-version: '12.x' 35 | registry-url: 'https://registry.npmjs.org' 36 | - name: Install protoc 37 | uses: arduino/setup-protoc@master 38 | with: 39 | version: '3.17.3' 40 | - name: Install Go protoc plugins 41 | run: | 42 | go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 43 | go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1 44 | - name: Get JS dependencies 45 | run: | 46 | cd api/pb/javascript && npm install 47 | cd ../../../net/api/pb/javascript && npm install 48 | npm install -g json 49 | npm install -g yaml-cli 50 | - name: Get Dart dependencies 51 | run: | 52 | cd api/pb/dart 53 | pub get 54 | pub global activate protoc_plugin 55 | - name: Protoc generate API 56 | run: | 57 | cd api/pb 58 | make clean && make 59 | - name: Protoc generate Service 60 | run: | 61 | cd net/api/pb 62 | make clean && make 63 | - name: Publish JS API 64 | env: 65 | NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} 66 | run: | 67 | cd api/pb/javascript 68 | json -I -f package.json -e 'this.version=("${{ steps.latesttag.outputs.tag }}").replace("v", "")' 69 | npm publish --access=public 70 | - name: Publish JS Service 71 | env: 72 | NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} 73 | run: | 74 | cd net/api/pb/javascript 75 | json -I -f package.json -e 'this.version=("${{ steps.latesttag.outputs.tag }}").replace("v", "")' 76 | npm publish --access=public 77 | - name: Publish Dart API 78 | env: 79 | PUB_CREDENTIALS: ${{ secrets.PUB_CREDENTIALS }} 80 | run: | 81 | sed -e "s/api\/pb\/dart\/lib//g" -i.replace .gitignore 82 | cd api/pb/dart 83 | yaml json read ../javascript/package.json > package.yml 84 | yaml set pubspec.yaml version $(yaml get package.yml version) > tmp.yml 85 | mv tmp.yml pubspec.yaml 86 | rm package.yml 87 | echo "$PUB_CREDENTIALS" > ~/.pub-cache/credentials.json 88 | pub publish -f 89 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | release: 4 | types: [created] 5 | jobs: 6 | release-platform-builds: 7 | name: Release Builds 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Set up Go 11 | uses: actions/setup-go@v1 12 | with: 13 | go-version: 1.16 14 | - name: Check out code 15 | uses: actions/checkout@v1 16 | - name: Cache dependencies 17 | id: cache-dependencies 18 | uses: actions/cache@v1 19 | with: 20 | path: ~/go/pkg/mod 21 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 22 | restore-keys: | 23 | ${{ runner.os }}-go- 24 | - name: Get dependencies 25 | if: steps.cache-dependencies.outputs.cache-hit != 'true' 26 | run: | 27 | export PATH=${PATH}:`go env GOPATH`/bin 28 | go get -v -t -d ./... 29 | - name: Install gox 30 | run: | 31 | export PATH=${PATH}:`go env GOPATH`/bin 32 | go get github.com/mitchellh/gox 33 | - name: Compile 34 | run: | 35 | export PATH=${PATH}:`go env GOPATH`/bin 36 | gox -osarch="linux/amd64 linux/386 linux/arm darwin/amd64 darwin/arm64 windows/amd64" -output="threadsd-{{.OS}}-{{.Arch}}" ./threadsd 37 | - name: Collect artifacts 38 | run: | 39 | VERSION=${GITHUB_REF##*/} 40 | OUT=release 41 | mkdir -p ${OUT} 42 | rm threadsd/main.go 43 | cp LICENSE threadsd/ 44 | cp dist/README.md threadsd/ 45 | cp dist/install threadsd/ 46 | declare -a arr=("darwin-amd64" "darwin-arm64" "windows-amd64.exe" "linux-amd64" "linux-386" "linux-arm") 47 | for i in "${arr[@]}" 48 | do 49 | OSARCH=${i%.*} 50 | EXT=$([[ "$i" = *.* ]] && echo ".${i##*.}" || echo '') 51 | cp threadsd-${i} threadsd/threadsd${EXT} 52 | if [ "${EXT}" == ".exe" ]; then 53 | zip -r threadsd_${VERSION}_${OSARCH}.zip threadsd 54 | mv threadsd_${VERSION}_${OSARCH}.zip ${OUT}/ 55 | else 56 | tar -czvf threadsd_${VERSION}_${OSARCH}.tar.gz threadsd 57 | mv threadsd_${VERSION}_${OSARCH}.tar.gz ${OUT}/ 58 | fi 59 | done 60 | echo $(ls ./release) 61 | - name: Upload assets to release 62 | uses: AButler/upload-release-assets@v2.0 63 | with: 64 | files: 'release/threadsd_*' 65 | repo-token: ${{ secrets.GITHUB_TOKEN }} 66 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | branches: 8 | - master 9 | jobs: 10 | 11 | test: 12 | name: Test 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Set up Go 16 | uses: actions/setup-go@v1 17 | with: 18 | go-version: 1.16 19 | - name: Check out code 20 | uses: actions/checkout@v1 21 | - name: Cache dependencies 22 | id: cache-dependencies 23 | uses: actions/cache@v1 24 | with: 25 | path: ~/go/pkg/mod 26 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 27 | restore-keys: | 28 | ${{ runner.os }}-go- 29 | - name: Get dependencies 30 | if: steps.cache-dependencies.outputs.cache-hit != 'true' 31 | run: | 32 | go get -v -t -d ./... 33 | - name: Test 34 | run: SKIP_FOLDERSYNC=true go test -race ./... 35 | 36 | test-foldersync: 37 | name: Foldersync Test 38 | runs-on: ubuntu-latest 39 | steps: 40 | - name: Set up Go 41 | uses: actions/setup-go@v1 42 | with: 43 | go-version: 1.16 44 | - name: Check out code 45 | uses: actions/checkout@v1 46 | - name: Cache dependencies 47 | id: cache-dependencies 48 | uses: actions/cache@v1 49 | with: 50 | path: ~/go/pkg/mod 51 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 52 | restore-keys: | 53 | ${{ runner.os }}-go- 54 | - name: Get dependencies 55 | if: steps.cache-dependencies.outputs.cache-hit != 'true' 56 | run: | 57 | go get -v -t -d ./... 58 | - name: Test 59 | run: go test ./integrationtests/foldersync/ 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | vendor/ 16 | 17 | # JS PB files 18 | *.js 19 | *.d.ts 20 | 21 | # Dart PB Files 22 | api/pb/dart/.dart_tool 23 | api/pb/dart/.packages 24 | api/pb/dart/lib 25 | 26 | # vscode config folder 27 | .vscode/ 28 | .idea/ 29 | 30 | **/node_modules 31 | tags 32 | 33 | # Misc 34 | **.DS_Store 35 | *.swp 36 | 37 | # Other 38 | .threads*/ 39 | integrationtests/migration/generated/ 40 | .env 41 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to make participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies within all project spaces, and it also applies when 49 | an individual is representing the project or its community in public spaces. 50 | Examples of representing a project or community include using an official 51 | project e-mail address, posting via an official social media account, or acting 52 | as an appointed representative at an online or offline event. Representation of 53 | a project may be further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at contact@textile.io. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17.5-buster 2 | LABEL maintainer="Textile " 3 | 4 | # This is (in large part) copied (with love) from 5 | # https://hub.docker.com/r/ipfs/go-ipfs/dockerfile 6 | 7 | # Get source 8 | ENV SRC_DIR /go-threads 9 | 10 | # Download packages first so they can be cached. 11 | COPY go.mod go.sum $SRC_DIR/ 12 | RUN cd $SRC_DIR \ 13 | && go mod download 14 | 15 | COPY . $SRC_DIR 16 | 17 | # Install the daemon 18 | RUN cd $SRC_DIR \ 19 | && go install github.com/textileio/go-threads/threadsd 20 | 21 | # Get su-exec, a very minimal tool for dropping privileges, 22 | # and tini, a very minimal init daemon for containers 23 | ENV SUEXEC_VERSION v0.2 24 | ENV TINI_VERSION v0.16.1 25 | RUN set -x \ 26 | && cd /tmp \ 27 | && git clone https://github.com/ncopa/su-exec.git \ 28 | && cd su-exec \ 29 | && git checkout -q $SUEXEC_VERSION \ 30 | && make \ 31 | && cd /tmp \ 32 | && wget -q -O tini https://github.com/krallin/tini/releases/download/$TINI_VERSION/tini \ 33 | && chmod +x tini 34 | 35 | # Get the TLS CA certificates, they're not provided by busybox. 36 | RUN apt-get update && apt-get install -y ca-certificates 37 | 38 | # Now comes the actual target image, which aims to be as small as possible. 39 | FROM busybox:1.31.0-glibc 40 | LABEL maintainer="Textile " 41 | 42 | # Get the threads binary, entrypoint script, and TLS CAs from the build container. 43 | ENV SRC_DIR /go-threads 44 | COPY --from=0 /go/bin/threadsd /usr/local/bin/threadsd 45 | COPY --from=0 /tmp/su-exec/su-exec /sbin/su-exec 46 | COPY --from=0 /tmp/tini /sbin/tini 47 | COPY --from=0 /etc/ssl/certs /etc/ssl/certs 48 | 49 | # This shared lib (part of glibc) doesn't seem to be included with busybox. 50 | COPY --from=0 /lib/x86_64-linux-gnu/libdl.so.2 /lib/libdl.so.2 51 | 52 | # hostAddr; should be exposed to the public 53 | EXPOSE 4006 54 | # apiAddr; should *not* be exposed to the public unless intercepted by an auth system, e.g., textile 55 | EXPOSE 6006 56 | # apiProxyAddr; should *not* be exposed to the public unless intercepted by an auth system, e.g., textile 57 | EXPOSE 6007 58 | 59 | # Create the repo directory and switch to a non-privileged user. 60 | ENV THREADS_PATH /data/threads 61 | RUN mkdir -p $THREADS_PATH \ 62 | && adduser -D -h $THREADS_PATH -u 1000 -G users textile \ 63 | && chown -R textile:users $THREADS_PATH 64 | 65 | # Switch to a non-privileged user 66 | USER textile 67 | 68 | # Expose the repo as a volume. 69 | # Important this happens after the USER directive so permission are correct. 70 | VOLUME $THREADS_PATH 71 | 72 | ENTRYPOINT ["/sbin/tini", "--", "threadsd"] 73 | 74 | CMD ["--repo=/data/threads"] 75 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018-2020 Textile 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | threaddb-up: 2 | docker-compose -f docker-compose-dev.yml up --build 3 | 4 | threaddb-stop: 5 | docker-compose -f docker-compose-dev.yml stop 6 | 7 | threaddb-clean: 8 | docker-compose -f docker-compose-dev.yml down -v --remove-orphans 9 | 10 | test: 11 | go test -race -timeout 45m ./... 12 | .PHONY: test -------------------------------------------------------------------------------- /api/client/read.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | 8 | pb "github.com/textileio/go-threads/api/pb" 9 | "github.com/textileio/go-threads/core/thread" 10 | "github.com/textileio/go-threads/db" 11 | ) 12 | 13 | // ReadTransaction encapsulates a read transaction. 14 | type ReadTransaction struct { 15 | client pb.API_ReadTransactionClient 16 | dbID thread.ID 17 | collectionName string 18 | } 19 | 20 | // EndTransactionFunc must be called to end a transaction after it has been started. 21 | type EndTransactionFunc = func() error 22 | 23 | // Start starts the read transaction. 24 | func (t *ReadTransaction) Start() (EndTransactionFunc, error) { 25 | innerReq := &pb.StartTransactionRequest{ 26 | DbID: t.dbID.Bytes(), 27 | CollectionName: t.collectionName, 28 | } 29 | option := &pb.ReadTransactionRequest_StartTransactionRequest{ 30 | StartTransactionRequest: innerReq, 31 | } 32 | if err := t.client.Send(&pb.ReadTransactionRequest{ 33 | Option: option, 34 | }); err != nil { 35 | return nil, err 36 | } 37 | return t.end, nil 38 | } 39 | 40 | // Has runs a has query in the active transaction. 41 | func (t *ReadTransaction) Has(instanceIDs ...string) (bool, error) { 42 | innerReq := &pb.HasRequest{ 43 | InstanceIDs: instanceIDs, 44 | } 45 | option := &pb.ReadTransactionRequest_HasRequest{ 46 | HasRequest: innerReq, 47 | } 48 | if err := t.client.Send(&pb.ReadTransactionRequest{ 49 | Option: option, 50 | }); err != nil { 51 | return false, err 52 | } 53 | var resp *pb.ReadTransactionReply 54 | var err error 55 | if resp, err = t.client.Recv(); err != nil { 56 | return false, err 57 | } 58 | switch x := resp.GetOption().(type) { 59 | case *pb.ReadTransactionReply_HasReply: 60 | return x.HasReply.GetExists(), txnError(x.HasReply.TransactionError) 61 | default: 62 | return false, fmt.Errorf("ReadTransactionReply.Option has unexpected type %T", x) 63 | } 64 | } 65 | 66 | // FindByID gets the instance with the specified ID. 67 | func (t *ReadTransaction) FindByID(instanceID string, instance interface{}) error { 68 | innerReq := &pb.FindByIDRequest{ 69 | InstanceID: instanceID, 70 | } 71 | option := &pb.ReadTransactionRequest_FindByIDRequest{ 72 | FindByIDRequest: innerReq, 73 | } 74 | if err := t.client.Send(&pb.ReadTransactionRequest{ 75 | Option: option, 76 | }); err != nil { 77 | return err 78 | } 79 | var resp *pb.ReadTransactionReply 80 | var err error 81 | if resp, err = t.client.Recv(); err != nil { 82 | return err 83 | } 84 | switch x := resp.GetOption().(type) { 85 | case *pb.ReadTransactionReply_FindByIDReply: 86 | err := txnError(x.FindByIDReply.TransactionError) 87 | if err != nil { 88 | return err 89 | } 90 | err = json.Unmarshal(x.FindByIDReply.GetInstance(), instance) 91 | return err 92 | default: 93 | return fmt.Errorf("ReadTransactionReply.Option has unexpected type %T", x) 94 | } 95 | } 96 | 97 | // Find finds instances by query. 98 | func (t *ReadTransaction) Find(query *db.Query, dummy interface{}) (interface{}, error) { 99 | queryBytes, err := json.Marshal(query) 100 | if err != nil { 101 | return nil, err 102 | } 103 | innerReq := &pb.FindRequest{ 104 | QueryJSON: queryBytes, 105 | } 106 | option := &pb.ReadTransactionRequest_FindRequest{ 107 | FindRequest: innerReq, 108 | } 109 | if err := t.client.Send(&pb.ReadTransactionRequest{ 110 | Option: option, 111 | }); err != nil { 112 | return nil, err 113 | } 114 | var resp *pb.ReadTransactionReply 115 | if resp, err = t.client.Recv(); err != nil { 116 | return nil, err 117 | } 118 | switch x := resp.GetOption().(type) { 119 | case *pb.ReadTransactionReply_FindReply: 120 | return processFindReply(x.FindReply, dummy) 121 | default: 122 | return nil, fmt.Errorf("ReadTransactionReply.Option has unexpected type %T", x) 123 | } 124 | } 125 | 126 | // end ends the active transaction. 127 | func (t *ReadTransaction) end() error { 128 | if err := t.client.CloseSend(); err != nil { 129 | return err 130 | } 131 | if _, err := t.client.Recv(); err != nil && err != io.EOF { 132 | return err 133 | } 134 | return nil 135 | } 136 | -------------------------------------------------------------------------------- /api/pb/Makefile: -------------------------------------------------------------------------------- 1 | PB = $(wildcard *.proto) 2 | GO = $(PB:.proto=.pb.go) 3 | PROTOC_GEN_TS_PATH = "./javascript/node_modules/.bin/protoc-gen-ts" 4 | PROTOC_GEN_DART_PATH = "${HOME}/.pub-cache/bin/protoc-gen-dart" 5 | 6 | all: $(GO) 7 | 8 | %.pb.go: %.proto 9 | mkdir -p ./dart/lib && \ 10 | protoc -I=. \ 11 | --plugin="protoc-gen-ts=${PROTOC_GEN_TS_PATH}" --js_out="import_style=commonjs,binary:javascript/." --ts_out="service=grpc-web:javascript/." \ 12 | --plugin="protoc-gen-dart=${PROTOC_GEN_DART_PATH}" --dart_out="grpc:dart/lib/." \ 13 | --go_out=. --go_opt=paths=source_relative \ 14 | --go-grpc_out=. --go-grpc_opt=paths=source_relative \ 15 | $< 16 | 17 | clean: 18 | rm -f *.pb.go 19 | rm -f *pb_test.go 20 | rm -f ./javascript/*.js 21 | rm -f ./javascript/*.d.ts 22 | rm -rf ./dart/lib 23 | 24 | .PHONY: clean -------------------------------------------------------------------------------- /api/pb/dart/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018-2020 Textile 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /api/pb/dart/README.md: -------------------------------------------------------------------------------- 1 | This is a support library used by the [Dart Threads Client](https://github.com/textileio/dart-threads-client/), available on pub.dev [here](https://pub.dev/packages/threads_client). -------------------------------------------------------------------------------- /api/pb/dart/pubspec.lock: -------------------------------------------------------------------------------- 1 | # Generated by pub 2 | # See https://dart.dev/tools/pub/glossary#lockfile 3 | packages: 4 | archive: 5 | dependency: transitive 6 | description: 7 | name: archive 8 | url: "https://pub.dartlang.org" 9 | source: hosted 10 | version: "3.1.2" 11 | args: 12 | dependency: transitive 13 | description: 14 | name: args 15 | url: "https://pub.dartlang.org" 16 | source: hosted 17 | version: "1.6.0" 18 | async: 19 | dependency: "direct main" 20 | description: 21 | name: async 22 | url: "https://pub.dartlang.org" 23 | source: hosted 24 | version: "2.8.2" 25 | charcode: 26 | dependency: transitive 27 | description: 28 | name: charcode 29 | url: "https://pub.dartlang.org" 30 | source: hosted 31 | version: "1.3.1" 32 | collection: 33 | dependency: transitive 34 | description: 35 | name: collection 36 | url: "https://pub.dartlang.org" 37 | source: hosted 38 | version: "1.15.0" 39 | crypto: 40 | dependency: transitive 41 | description: 42 | name: crypto 43 | url: "https://pub.dartlang.org" 44 | source: hosted 45 | version: "3.0.1" 46 | fixnum: 47 | dependency: transitive 48 | description: 49 | name: fixnum 50 | url: "https://pub.dartlang.org" 51 | source: hosted 52 | version: "1.0.0" 53 | googleapis_auth: 54 | dependency: transitive 55 | description: 56 | name: googleapis_auth 57 | url: "https://pub.dartlang.org" 58 | source: hosted 59 | version: "1.1.0" 60 | grpc: 61 | dependency: "direct main" 62 | description: 63 | name: grpc 64 | url: "https://pub.dartlang.org" 65 | source: hosted 66 | version: "3.0.2" 67 | http: 68 | dependency: transitive 69 | description: 70 | name: http 71 | url: "https://pub.dartlang.org" 72 | source: hosted 73 | version: "0.13.3" 74 | http2: 75 | dependency: transitive 76 | description: 77 | name: http2 78 | url: "https://pub.dartlang.org" 79 | source: hosted 80 | version: "2.0.0" 81 | http_parser: 82 | dependency: transitive 83 | description: 84 | name: http_parser 85 | url: "https://pub.dartlang.org" 86 | source: hosted 87 | version: "4.0.0" 88 | matcher: 89 | dependency: transitive 90 | description: 91 | name: matcher 92 | url: "https://pub.dartlang.org" 93 | source: hosted 94 | version: "0.12.11" 95 | meta: 96 | dependency: transitive 97 | description: 98 | name: meta 99 | url: "https://pub.dartlang.org" 100 | source: hosted 101 | version: "1.7.0" 102 | path: 103 | dependency: transitive 104 | description: 105 | name: path 106 | url: "https://pub.dartlang.org" 107 | source: hosted 108 | version: "1.8.0" 109 | pedantic: 110 | dependency: transitive 111 | description: 112 | name: pedantic 113 | url: "https://pub.dartlang.org" 114 | source: hosted 115 | version: "1.11.1" 116 | protobuf: 117 | dependency: "direct main" 118 | description: 119 | name: protobuf 120 | url: "https://pub.dartlang.org" 121 | source: hosted 122 | version: "2.0.0" 123 | pub_semver: 124 | dependency: transitive 125 | description: 126 | name: pub_semver 127 | url: "https://pub.dartlang.org" 128 | source: hosted 129 | version: "1.4.4" 130 | pubspec: 131 | dependency: transitive 132 | description: 133 | name: pubspec 134 | url: "https://pub.dartlang.org" 135 | source: hosted 136 | version: "0.1.5" 137 | pubspec_version: 138 | dependency: "direct dev" 139 | description: 140 | name: pubspec_version 141 | url: "https://pub.dartlang.org" 142 | source: hosted 143 | version: "0.6.1+archived" 144 | quiver: 145 | dependency: transitive 146 | description: 147 | name: quiver 148 | url: "https://pub.dartlang.org" 149 | source: hosted 150 | version: "2.1.5" 151 | source_span: 152 | dependency: transitive 153 | description: 154 | name: source_span 155 | url: "https://pub.dartlang.org" 156 | source: hosted 157 | version: "1.8.1" 158 | stack_trace: 159 | dependency: transitive 160 | description: 161 | name: stack_trace 162 | url: "https://pub.dartlang.org" 163 | source: hosted 164 | version: "1.10.0" 165 | string_scanner: 166 | dependency: transitive 167 | description: 168 | name: string_scanner 169 | url: "https://pub.dartlang.org" 170 | source: hosted 171 | version: "1.1.0" 172 | term_glyph: 173 | dependency: transitive 174 | description: 175 | name: term_glyph 176 | url: "https://pub.dartlang.org" 177 | source: hosted 178 | version: "1.2.0" 179 | typed_data: 180 | dependency: transitive 181 | description: 182 | name: typed_data 183 | url: "https://pub.dartlang.org" 184 | source: hosted 185 | version: "1.3.0" 186 | uri: 187 | dependency: transitive 188 | description: 189 | name: uri 190 | url: "https://pub.dartlang.org" 191 | source: hosted 192 | version: "0.11.4" 193 | yaml: 194 | dependency: transitive 195 | description: 196 | name: yaml 197 | url: "https://pub.dartlang.org" 198 | source: hosted 199 | version: "2.2.1" 200 | sdks: 201 | dart: ">=2.12.0 <3.0.0" 202 | -------------------------------------------------------------------------------- /api/pb/dart/pubspec.yaml: -------------------------------------------------------------------------------- 1 | name: threads_client_grpc 2 | version: 0.0.0 3 | description: A gRPC client for interacting with a remote ThreadDB service. 4 | homepage: 'https://github.com/textileio/go-threads' 5 | repository: 'https://github.com/textileio/go-threads' 6 | environment: 7 | sdk: '>=2.12.0 <3.0.0' 8 | dependencies: 9 | async: ^2.2.0 10 | grpc: ^3.0.2 11 | protobuf: ^2.0.0 12 | dev_dependencies: 13 | pubspec_version: ^0.6.1 14 | 15 | -------------------------------------------------------------------------------- /api/pb/javascript/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@textile/threads-client-grpc", 3 | "version": "0.0.0", 4 | "lockfileVersion": 2, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "@textile/threads-client-grpc", 9 | "version": "0.0.0", 10 | "license": "MIT", 11 | "dependencies": { 12 | "@improbable-eng/grpc-web": "^0.14.1", 13 | "@types/google-protobuf": "^3.15.5", 14 | "google-protobuf": "^3.19.4" 15 | }, 16 | "devDependencies": { 17 | "ts-protoc-gen": "^0.15.0" 18 | } 19 | }, 20 | "node_modules/@improbable-eng/grpc-web": { 21 | "version": "0.14.1", 22 | "resolved": "https://registry.npmjs.org/@improbable-eng/grpc-web/-/grpc-web-0.14.1.tgz", 23 | "integrity": "sha512-XaIYuunepPxoiGVLLHmlnVminUGzBTnXr8Wv7khzmLWbNw4TCwJKX09GSMJlKhu/TRk6gms0ySFxewaETSBqgw==", 24 | "dependencies": { 25 | "browser-headers": "^0.4.1" 26 | }, 27 | "peerDependencies": { 28 | "google-protobuf": "^3.14.0" 29 | } 30 | }, 31 | "node_modules/@types/google-protobuf": { 32 | "version": "3.15.5", 33 | "resolved": "https://registry.npmjs.org/@types/google-protobuf/-/google-protobuf-3.15.5.tgz", 34 | "integrity": "sha512-6bgv24B+A2bo9AfzReeg5StdiijKzwwnRflA8RLd1V4Yv995LeTmo0z69/MPbBDFSiZWdZHQygLo/ccXhMEDgw==" 35 | }, 36 | "node_modules/browser-headers": { 37 | "version": "0.4.1", 38 | "resolved": "https://registry.npmjs.org/browser-headers/-/browser-headers-0.4.1.tgz", 39 | "integrity": "sha512-CA9hsySZVo9371qEHjHZtYxV2cFtVj5Wj/ZHi8ooEsrtm4vOnl9Y9HmyYWk9q+05d7K3rdoAE0j3MVEFVvtQtg==" 40 | }, 41 | "node_modules/google-protobuf": { 42 | "version": "3.19.4", 43 | "resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.19.4.tgz", 44 | "integrity": "sha512-OIPNCxsG2lkIvf+P5FNfJ/Km95CsXOBecS9ZcAU6m2Rq3svc0Apl9nB3GMDNKfQ9asNv4KjyAqGwPQFrVle3Yg==" 45 | }, 46 | "node_modules/ts-protoc-gen": { 47 | "version": "0.15.0", 48 | "resolved": "https://registry.npmjs.org/ts-protoc-gen/-/ts-protoc-gen-0.15.0.tgz", 49 | "integrity": "sha512-TycnzEyrdVDlATJ3bWFTtra3SCiEP0W0vySXReAuEygXCUr1j2uaVyL0DhzjwuUdQoW5oXPwk6oZWeA0955V+g==", 50 | "dev": true, 51 | "dependencies": { 52 | "google-protobuf": "^3.15.5" 53 | }, 54 | "bin": { 55 | "protoc-gen-ts": "bin/protoc-gen-ts" 56 | } 57 | } 58 | }, 59 | "dependencies": { 60 | "@improbable-eng/grpc-web": { 61 | "version": "0.14.1", 62 | "resolved": "https://registry.npmjs.org/@improbable-eng/grpc-web/-/grpc-web-0.14.1.tgz", 63 | "integrity": "sha512-XaIYuunepPxoiGVLLHmlnVminUGzBTnXr8Wv7khzmLWbNw4TCwJKX09GSMJlKhu/TRk6gms0ySFxewaETSBqgw==", 64 | "requires": { 65 | "browser-headers": "^0.4.1" 66 | } 67 | }, 68 | "@types/google-protobuf": { 69 | "version": "3.15.5", 70 | "resolved": "https://registry.npmjs.org/@types/google-protobuf/-/google-protobuf-3.15.5.tgz", 71 | "integrity": "sha512-6bgv24B+A2bo9AfzReeg5StdiijKzwwnRflA8RLd1V4Yv995LeTmo0z69/MPbBDFSiZWdZHQygLo/ccXhMEDgw==" 72 | }, 73 | "browser-headers": { 74 | "version": "0.4.1", 75 | "resolved": "https://registry.npmjs.org/browser-headers/-/browser-headers-0.4.1.tgz", 76 | "integrity": "sha512-CA9hsySZVo9371qEHjHZtYxV2cFtVj5Wj/ZHi8ooEsrtm4vOnl9Y9HmyYWk9q+05d7K3rdoAE0j3MVEFVvtQtg==" 77 | }, 78 | "google-protobuf": { 79 | "version": "3.19.4", 80 | "resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.19.4.tgz", 81 | "integrity": "sha512-OIPNCxsG2lkIvf+P5FNfJ/Km95CsXOBecS9ZcAU6m2Rq3svc0Apl9nB3GMDNKfQ9asNv4KjyAqGwPQFrVle3Yg==" 82 | }, 83 | "ts-protoc-gen": { 84 | "version": "0.15.0", 85 | "resolved": "https://registry.npmjs.org/ts-protoc-gen/-/ts-protoc-gen-0.15.0.tgz", 86 | "integrity": "sha512-TycnzEyrdVDlATJ3bWFTtra3SCiEP0W0vySXReAuEygXCUr1j2uaVyL0DhzjwuUdQoW5oXPwk6oZWeA0955V+g==", 87 | "dev": true, 88 | "requires": { 89 | "google-protobuf": "^3.15.5" 90 | } 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /api/pb/javascript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@textile/threads-client-grpc", 3 | "version": "0.0.0", 4 | "description": "A gRPC client for interacting with a remote ThreadDB service.", 5 | "repository": { 6 | "type": "git", 7 | "url": "git+https://github.com/textileio/go-threads.git" 8 | }, 9 | "author": "Textile", 10 | "license": "MIT", 11 | "files": [ 12 | "threads_pb.js", 13 | "threads_pb_service.js", 14 | "threads_pb.d.ts", 15 | "threads_pb_service.d.ts" 16 | ], 17 | "dependencies": { 18 | "@improbable-eng/grpc-web": "^0.14.1", 19 | "@types/google-protobuf": "^3.15.5", 20 | "google-protobuf": "^3.19.4" 21 | }, 22 | "devDependencies": { 23 | "ts-protoc-gen": "^0.15.0" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /broadcast/broadcast.go: -------------------------------------------------------------------------------- 1 | // Package broadcast implements multi-listener broadcast channels. 2 | // See https://godoc.org/github.com/tjgq/broadcast for original implementation. 3 | // 4 | // To create an un-buffered broadcast channel, just declare a Broadcaster: 5 | // 6 | // var b broadcast.Broadcaster 7 | // 8 | // To create a buffered broadcast channel with capacity n, call New: 9 | // 10 | // b := broadcast.New(n) 11 | // 12 | // To add a listener to a channel, call Listen and read from Channel(): 13 | // 14 | // l := b.Listen() 15 | // for v := range l.Channel() { 16 | // // ... 17 | // } 18 | // 19 | // 20 | // To send to the channel, call Send: 21 | // 22 | // b.Send("Hello world!") 23 | // v <- l.Channel() // returns interface{}("Hello world!") 24 | // 25 | // To remove a listener, call Discard. 26 | // 27 | // l.Discard() 28 | // 29 | // To close the broadcast channel, call Discard. Any existing or future listeners 30 | // will read from a closed channel: 31 | // 32 | // b.Discard() 33 | // v, ok <- l.Channel() // returns ok == false 34 | package broadcast 35 | 36 | import ( 37 | "errors" 38 | "fmt" 39 | "sync" 40 | "time" 41 | 42 | "github.com/hashicorp/go-multierror" 43 | ) 44 | 45 | // ErrClosedChannel means the caller attempted to send to one or more closed broadcast channels. 46 | const ErrClosedChannel = broadcastError("send after close") 47 | 48 | type broadcastError string 49 | 50 | func (e broadcastError) Error() string { return string(e) } 51 | 52 | // Broadcaster implements a Publisher. The zero value is a usable un-buffered channel. 53 | type Broadcaster struct { 54 | m sync.Mutex 55 | listeners map[uint]chan<- interface{} // lazy init 56 | nextID uint 57 | capacity int 58 | closed bool 59 | } 60 | 61 | // NewBroadcaster returns a new Broadcaster with the given capacity (0 means un-buffered). 62 | func NewBroadcaster(n int) *Broadcaster { 63 | return &Broadcaster{capacity: n} 64 | } 65 | 66 | // SendWithTimeout broadcasts a message to each listener's channel. 67 | // Sending on a closed channel causes a runtime panic. 68 | // This method blocks for a duration of up to `timeout` on each channel. 69 | // Returns error(s) if it is unable to send on a given listener's channel within `timeout` duration. 70 | func (b *Broadcaster) SendWithTimeout(v interface{}, timeout time.Duration) error { 71 | b.m.Lock() 72 | defer b.m.Unlock() 73 | if b.closed { 74 | return ErrClosedChannel 75 | } 76 | var result *multierror.Error 77 | for id, l := range b.listeners { 78 | select { 79 | case l <- v: 80 | // Success! 81 | case <-time.After(timeout): 82 | err := fmt.Sprintf("unable to send to listener '%d'", id) 83 | result = multierror.Append(result, errors.New(err)) 84 | } 85 | } 86 | if result != nil { 87 | return result.ErrorOrNil() 88 | } else { 89 | return nil 90 | } 91 | } 92 | 93 | // Send broadcasts a message to each listener's channel. 94 | // Sending on a closed channel causes a runtime panic. 95 | // This method is non-blocking, and will return errors if unable to send on a given listener's channel. 96 | func (b *Broadcaster) Send(v interface{}) error { 97 | return b.SendWithTimeout(v, 0) 98 | } 99 | 100 | // Discard closes the channel, disabling the sending of further messages. 101 | func (b *Broadcaster) Discard() { 102 | b.m.Lock() 103 | defer b.m.Unlock() 104 | if b.closed { 105 | return 106 | } 107 | b.closed = true 108 | for _, l := range b.listeners { 109 | close(l) 110 | } 111 | } 112 | 113 | // Listen returns a Listener for the broadcast channel. 114 | func (b *Broadcaster) Listen() *Listener { 115 | b.m.Lock() 116 | defer b.m.Unlock() 117 | if b.listeners == nil { 118 | b.listeners = make(map[uint]chan<- interface{}) 119 | } 120 | if b.listeners[b.nextID] != nil { 121 | b.nextID++ 122 | } 123 | ch := make(chan interface{}, b.capacity) 124 | if b.closed { 125 | close(ch) 126 | } 127 | b.listeners[b.nextID] = ch 128 | return &Listener{ch, b, b.nextID} 129 | } 130 | 131 | // Listener implements a Subscriber to broadcast channel. 132 | type Listener struct { 133 | ch <-chan interface{} 134 | b *Broadcaster 135 | id uint 136 | } 137 | 138 | // Discard closes the Listener, disabling the reception of further messages. 139 | func (l *Listener) Discard() { 140 | l.b.m.Lock() 141 | defer l.b.m.Unlock() 142 | delete(l.b.listeners, l.id) 143 | } 144 | 145 | // Channel returns the channel that receives broadcast messages. 146 | func (l *Listener) Channel() <-chan interface{} { 147 | return l.ch 148 | } 149 | -------------------------------------------------------------------------------- /broadcast/broadcast_test.go: -------------------------------------------------------------------------------- 1 | package broadcast 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | "time" 7 | 8 | "github.com/hashicorp/go-multierror" 9 | ) 10 | 11 | const ( 12 | N = 3 13 | testStr = "Test" 14 | timeout = time.Second * 4 15 | ) 16 | 17 | type ListenFunc func(int, *Broadcaster, *sync.WaitGroup) 18 | 19 | func setupN(f ListenFunc) (*Broadcaster, *sync.WaitGroup) { 20 | var b Broadcaster 21 | var wg sync.WaitGroup 22 | wg.Add(N) 23 | for i := 0; i < N; i++ { 24 | go f(i, &b, &wg) 25 | } 26 | wg.Wait() 27 | return &b, &wg 28 | } 29 | 30 | func TestSend(t *testing.T) { 31 | b, wg := setupN(func(i int, b *Broadcaster, wg *sync.WaitGroup) { 32 | l := b.Listen() 33 | wg.Done() 34 | select { 35 | case v := <-l.Channel(): 36 | if v.(string) != testStr { 37 | t.Error("bad value received") 38 | } 39 | case <-time.After(timeout): 40 | t.Error("receive timed out") 41 | } 42 | wg.Done() 43 | }) 44 | wg.Add(N) 45 | _ = b.Send(testStr) 46 | wg.Wait() 47 | } 48 | 49 | func TestSendError(t *testing.T) { 50 | var b Broadcaster 51 | // Register listeners, but do not consume 52 | b.Listen() 53 | b.Listen() 54 | b.Listen() 55 | if err := b.Send(testStr); err == nil { 56 | t.Error("should error when no consumers") 57 | if multi, ok := err.(*multierror.Error); ok { 58 | if len(multi.Errors) != 3 { 59 | t.Error("expected 3 errors") 60 | } 61 | } else { 62 | t.Error("expected a multi-error") 63 | } 64 | } 65 | if err := b.Send(testStr); err == nil { 66 | t.Error("should error when no consumers") 67 | } 68 | } 69 | 70 | func TestListenAndSendOnClosed(t *testing.T) { 71 | var b = NewBroadcaster(5) 72 | b.Discard() 73 | b.Listen() 74 | err := b.Send(testStr) 75 | if err != ErrClosedChannel { 76 | if err != nil { 77 | t.Errorf("Test should raise closed channel error: %s", err.Error()) 78 | } 79 | } 80 | if err == nil || err.Error() != "send after close" { 81 | t.Error("Test should raise `send after close`") 82 | } 83 | } 84 | 85 | func TestListenAndSendOnCloseWithTimeout(t *testing.T) { 86 | var b = NewBroadcaster(5) 87 | b.Discard() 88 | b.Listen() 89 | err := b.SendWithTimeout(testStr, 0) 90 | if err != ErrClosedChannel { 91 | if err != nil { 92 | t.Errorf("Test should raise closed channel error: %s", err.Error()) 93 | } 94 | } 95 | if err == nil || err.Error() != "send after close" { 96 | t.Error("Test should raise `send after close`") 97 | } 98 | } 99 | 100 | func TestSendWithTimeout(t *testing.T) { 101 | var b Broadcaster 102 | var wg sync.WaitGroup 103 | wg.Add(1) 104 | go func(i int, b *Broadcaster, wg *sync.WaitGroup) { 105 | l := b.Listen() 106 | wg.Done() 107 | time.Sleep(time.Second) 108 | select { 109 | case v := <-l.Channel(): 110 | if v.(string) != testStr { 111 | t.Error("bad value received") 112 | } 113 | case <-time.After(timeout): 114 | t.Error("receive timed out") 115 | } 116 | wg.Done() 117 | }(1, &b, &wg) 118 | wg.Wait() 119 | wg.Add(1) 120 | if err := b.Send(testStr); err == nil { 121 | t.Error("should error") 122 | } 123 | if err := b.SendWithTimeout(testStr, 0); err == nil { 124 | t.Error("should error within 1 second") 125 | } 126 | if err := b.SendWithTimeout(testStr, 2*time.Second); err != nil { 127 | t.Error("should not error within 2 seconds") 128 | } 129 | wg.Wait() 130 | } 131 | 132 | func TestBroadcasterClose(t *testing.T) { 133 | b, wg := setupN(func(i int, b *Broadcaster, wg *sync.WaitGroup) { 134 | l := b.Listen() 135 | wg.Done() 136 | select { 137 | case _, ok := <-l.Channel(): 138 | if ok { 139 | t.Error("receive after close") 140 | } 141 | case <-time.After(timeout): 142 | t.Error("receive timed out") 143 | } 144 | wg.Done() 145 | }) 146 | wg.Add(N) 147 | b.Discard() 148 | wg.Wait() 149 | } 150 | 151 | func TestListenerClose(t *testing.T) { 152 | b, wg := setupN(func(i int, b *Broadcaster, wg *sync.WaitGroup) { 153 | l := b.Listen() 154 | if i == 0 { 155 | l.Discard() 156 | } 157 | wg.Done() 158 | select { 159 | case <-l.Channel(): 160 | if i == 0 { 161 | t.Error("receive after close") 162 | } 163 | case <-time.After(timeout): 164 | if i != 0 { 165 | t.Error("receive timed out") 166 | } 167 | } 168 | wg.Done() 169 | }) 170 | wg.Add(N) 171 | _ = b.Send(testStr) 172 | wg.Wait() 173 | } 174 | -------------------------------------------------------------------------------- /cbor/coding.go: -------------------------------------------------------------------------------- 1 | package cbor 2 | 3 | import ( 4 | blocks "github.com/ipfs/go-block-format" 5 | "github.com/ipfs/go-ipld-cbor" 6 | "github.com/ipfs/go-ipld-format" 7 | mh "github.com/multiformats/go-multihash" 8 | "github.com/textileio/crypto" 9 | ) 10 | 11 | // EncodeBlock returns a node by encrypting the block's raw bytes with key. 12 | func EncodeBlock(block blocks.Block, key crypto.EncryptionKey) (format.Node, error) { 13 | coded, err := key.Encrypt(block.RawData()) 14 | if err != nil { 15 | return nil, err 16 | } 17 | return cbornode.WrapObject(coded, mh.SHA2_256, -1) 18 | } 19 | 20 | // DecodeBlock returns a node by decrypting the block's raw bytes with key. 21 | func DecodeBlock(block blocks.Block, key crypto.DecryptionKey) (format.Node, error) { 22 | var raw []byte 23 | err := cbornode.DecodeInto(block.RawData(), &raw) 24 | if err != nil { 25 | return nil, err 26 | } 27 | decoded, err := key.Decrypt(raw) 28 | if err != nil { 29 | return nil, err 30 | } 31 | return cbornode.Decode(decoded, mh.SHA2_256, -1) 32 | } 33 | -------------------------------------------------------------------------------- /cbor/event.go: -------------------------------------------------------------------------------- 1 | package cbor 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/ipfs/go-cid" 8 | cbornode "github.com/ipfs/go-ipld-cbor" 9 | "github.com/ipfs/go-ipld-format" 10 | mh "github.com/multiformats/go-multihash" 11 | "github.com/textileio/go-threads/core/net" 12 | "github.com/textileio/crypto" 13 | sym "github.com/textileio/crypto/symmetric" 14 | ) 15 | 16 | func init() { 17 | cbornode.RegisterCborType(event{}) 18 | cbornode.RegisterCborType(eventHeader{}) 19 | } 20 | 21 | // event defines the node structure of an event. 22 | type event struct { 23 | Body cid.Cid 24 | Header cid.Cid 25 | } 26 | 27 | // eventHeader defines the node structure of an event header. 28 | type eventHeader struct { 29 | Key []byte `refmt:",omitempty"` 30 | } 31 | 32 | // CreateEvent create a new event by wrapping the body node. 33 | func CreateEvent(ctx context.Context, dag format.DAGService, body format.Node, rkey crypto.EncryptionKey) (net.Event, error) { 34 | key, err := sym.NewRandom() 35 | if err != nil { 36 | return nil, err 37 | } 38 | codedBody, err := EncodeBlock(body, key) 39 | if err != nil { 40 | return nil, err 41 | } 42 | keyb, err := key.MarshalBinary() 43 | if err != nil { 44 | return nil, err 45 | } 46 | eventHeader := &eventHeader{ 47 | Key: keyb, 48 | } 49 | header, err := cbornode.WrapObject(eventHeader, mh.SHA2_256, -1) 50 | if err != nil { 51 | return nil, err 52 | } 53 | codedHeader, err := EncodeBlock(header, rkey) 54 | if err != nil { 55 | return nil, err 56 | } 57 | obj := &event{ 58 | Body: codedBody.Cid(), 59 | Header: codedHeader.Cid(), 60 | } 61 | node, err := cbornode.WrapObject(obj, mh.SHA2_256, -1) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | if dag != nil { 67 | if err = dag.AddMany(ctx, []format.Node{node, codedHeader, codedBody}); err != nil { 68 | return nil, err 69 | } 70 | } 71 | 72 | return &Event{ 73 | Node: node, 74 | obj: obj, 75 | header: &EventHeader{ 76 | Node: codedHeader, 77 | obj: eventHeader, 78 | }, 79 | body: codedBody, 80 | }, nil 81 | } 82 | 83 | // GetEvent returns the event node for the given cid. 84 | func GetEvent(ctx context.Context, dag format.DAGService, id cid.Cid) (net.Event, error) { 85 | node, err := dag.Get(ctx, id) 86 | if err != nil { 87 | return nil, err 88 | } 89 | return EventFromNode(node) 90 | } 91 | 92 | // EventFromNode decodes the given node into an event. 93 | func EventFromNode(node format.Node) (*Event, error) { 94 | obj := new(event) 95 | if err := cbornode.DecodeInto(node.RawData(), obj); err != nil { 96 | return nil, err 97 | } 98 | return &Event{ 99 | Node: node, 100 | obj: obj, 101 | }, nil 102 | } 103 | 104 | // EventFromRecord returns the event within the given node. 105 | func EventFromRecord(ctx context.Context, dag format.DAGService, rec net.Record) (*Event, error) { 106 | block, err := rec.GetBlock(ctx, dag) 107 | if err != nil { 108 | return nil, err 109 | } 110 | 111 | event, ok := block.(*Event) 112 | if !ok { 113 | return EventFromNode(block) 114 | } 115 | return event, nil 116 | } 117 | 118 | // RemoveEvent removes an event from the dag service. 119 | func RemoveEvent(ctx context.Context, dag format.DAGService, e *Event) error { 120 | return dag.RemoveMany(ctx, []cid.Cid{e.Cid(), e.HeaderID(), e.BodyID()}) 121 | } 122 | 123 | // Event is a IPLD node representing an event. 124 | type Event struct { 125 | format.Node 126 | 127 | obj *event 128 | header *EventHeader 129 | body format.Node 130 | } 131 | 132 | func (e *Event) HeaderID() cid.Cid { 133 | return e.obj.Header 134 | } 135 | 136 | func (e *Event) GetHeader(ctx context.Context, dag format.DAGService, key crypto.DecryptionKey) (net.EventHeader, error) { 137 | if e.header == nil { 138 | coded, err := dag.Get(ctx, e.obj.Header) 139 | if err != nil { 140 | return nil, err 141 | } 142 | e.header = &EventHeader{ 143 | Node: coded, 144 | } 145 | } 146 | 147 | if e.header.obj != nil { 148 | return e.header, nil 149 | } 150 | 151 | header := new(eventHeader) 152 | if key != nil { 153 | node, err := DecodeBlock(e.header, key) 154 | if err != nil { 155 | return nil, err 156 | } 157 | if err = cbornode.DecodeInto(node.RawData(), header); err != nil { 158 | return nil, err 159 | } 160 | e.header.obj = header 161 | } 162 | return e.header, nil 163 | } 164 | 165 | func (e *Event) BodyID() cid.Cid { 166 | return e.obj.Body 167 | } 168 | 169 | func (e *Event) GetBody(ctx context.Context, dag format.DAGService, key crypto.DecryptionKey) (format.Node, error) { 170 | var k crypto.DecryptionKey 171 | if key != nil { 172 | header, err := e.GetHeader(ctx, dag, key) 173 | if err != nil { 174 | return nil, err 175 | } 176 | k, err = header.Key() 177 | if err != nil { 178 | return nil, err 179 | } 180 | } 181 | 182 | var err error 183 | if e.body == nil { 184 | e.body, err = dag.Get(ctx, e.obj.Body) 185 | if err != nil { 186 | return nil, err 187 | } 188 | } 189 | 190 | if k == nil { 191 | return e.body, nil 192 | } else { 193 | return DecodeBlock(e.body, k) 194 | } 195 | } 196 | 197 | // EventHeader is an IPLD node representing an event header. 198 | type EventHeader struct { 199 | format.Node 200 | 201 | obj *eventHeader 202 | } 203 | 204 | func (h *EventHeader) Key() (crypto.DecryptionKey, error) { 205 | if h.obj == nil { 206 | return nil, fmt.Errorf("obj not loaded") 207 | } 208 | return crypto.DecryptionKeyFromBytes(h.obj.Key) 209 | } 210 | -------------------------------------------------------------------------------- /core/app/app.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "time" 8 | 9 | format "github.com/ipfs/go-ipld-format" 10 | "github.com/textileio/go-threads/broadcast" 11 | "github.com/textileio/go-threads/core/net" 12 | "github.com/textileio/go-threads/core/thread" 13 | "github.com/textileio/go-threads/util" 14 | ) 15 | 16 | var ( 17 | // ErrThreadInUse indicates an operation could not be completed because the 18 | // thread is bound to an app. 19 | ErrThreadInUse = errors.New("thread is in use") 20 | 21 | // ErrInvalidNetRecordBody indicates the app determined the record body should not be accepted. 22 | ErrInvalidNetRecordBody = errors.New("app denied net record body") 23 | ) 24 | 25 | const busTimeout = time.Second * 10 26 | 27 | // App provides a bidirectional hook for thread-based apps. 28 | type App interface { 29 | // ValidateNetRecordBody provides the app an opportunity to validate the contents 30 | // of a record before it's committed to a thread log. 31 | // identity is the author's public key. 32 | ValidateNetRecordBody(ctx context.Context, body format.Node, identity thread.PubKey) error 33 | 34 | // HandleNetRecord handles an inbound thread record from net. 35 | HandleNetRecord(ctx context.Context, rec net.ThreadRecord, key thread.Key) error 36 | } 37 | 38 | // LocalEventsBus wraps a broadcaster for local events. 39 | type LocalEventsBus struct { 40 | bus *broadcast.Broadcaster 41 | } 42 | 43 | // NewLocalEventsBus returns a new bus for local event. 44 | func NewLocalEventsBus() *LocalEventsBus { 45 | return &LocalEventsBus{bus: broadcast.NewBroadcaster(0)} 46 | } 47 | 48 | // Send an IPLD node and thread auth into the bus. 49 | // These are received by the app connector and written to the underlying thread. 50 | func (leb *LocalEventsBus) Send(event *LocalEvent) error { 51 | return leb.bus.SendWithTimeout(event, busTimeout) 52 | } 53 | 54 | // Listen returns a local event listener. 55 | func (leb *LocalEventsBus) Listen() *LocalEventListener { 56 | l := &LocalEventListener{ 57 | listener: leb.bus.Listen(), 58 | c: make(chan *LocalEvent), 59 | } 60 | go func() { 61 | for v := range l.listener.Channel() { 62 | events := v.(*LocalEvent) 63 | l.c <- events 64 | } 65 | close(l.c) 66 | }() 67 | return l 68 | } 69 | 70 | // Discard the bus, closing all listeners. 71 | func (leb *LocalEventsBus) Discard() { 72 | leb.bus.Discard() 73 | } 74 | 75 | // LocalEvent wraps an IPLD node and auth for delivery to a thread. 76 | type LocalEvent struct { 77 | Node format.Node 78 | Token thread.Token 79 | } 80 | 81 | // LocalEventListener notifies about new locally generated events. 82 | type LocalEventListener struct { 83 | listener *broadcast.Listener 84 | c chan *LocalEvent 85 | } 86 | 87 | // Channel returns an unbuffered channel to receive local events. 88 | func (l *LocalEventListener) Channel() <-chan *LocalEvent { 89 | return l.c 90 | } 91 | 92 | // Discard indicates that no further events will be received 93 | // and ready for being garbage collected. 94 | func (l *LocalEventListener) Discard() { 95 | l.listener.Discard() 96 | } 97 | 98 | // Net adds the ability to connect an app to a thread. 99 | type Net interface { 100 | net.Net 101 | 102 | // ConnectApp returns an app<->thread connector. 103 | ConnectApp(App, thread.ID) (*Connector, error) 104 | 105 | // Validate thread ID and token against the net host. 106 | // If token is present and was issued the net host (is valid), the embedded public key is returned. 107 | // If token is not present, both the returned public key and error will be nil. 108 | Validate(id thread.ID, token thread.Token, readOnly bool) (thread.PubKey, error) 109 | } 110 | 111 | // Connector connects an app to a thread. 112 | type Connector struct { 113 | Net Net 114 | 115 | app App 116 | token net.Token 117 | threadID thread.ID 118 | threadKey thread.Key 119 | } 120 | 121 | // Connection receives new thread records, which are pumped to the app. 122 | type Connection func(context.Context, thread.ID) (<-chan net.ThreadRecord, error) 123 | 124 | // NewConnector creates bidirectional connection between an app and a thread. 125 | func NewConnector(app App, net Net, tinfo thread.Info) (*Connector, error) { 126 | if !tinfo.Key.CanRead() { 127 | return nil, fmt.Errorf("read key not found for thread %s", tinfo.ID) 128 | } 129 | return &Connector{ 130 | Net: net, 131 | app: app, 132 | token: util.GenerateRandomBytes(32), 133 | threadID: tinfo.ID, 134 | threadKey: tinfo.Key, 135 | }, nil 136 | } 137 | 138 | // ThreadID returns the underlying thread's ID. 139 | func (c *Connector) ThreadID() thread.ID { 140 | return c.threadID 141 | } 142 | 143 | // Token returns the net token. 144 | func (c *Connector) Token() net.Token { 145 | return c.token 146 | } 147 | 148 | // CreateNetRecord calls net.CreateRecord while supplying thread ID and API token. 149 | func (c *Connector) CreateNetRecord(ctx context.Context, body format.Node, token thread.Token) (net.ThreadRecord, error) { 150 | return c.Net.CreateRecord(ctx, c.threadID, body, net.WithThreadToken(token), net.WithAPIToken(c.token)) 151 | } 152 | 153 | // Validate thread token against the net host. 154 | func (c *Connector) Validate(token thread.Token, readOnly bool) error { 155 | _, err := c.Net.Validate(c.threadID, token, readOnly) 156 | return err 157 | } 158 | 159 | // ValidateNetRecordBody calls the connection app's ValidateNetRecordBody. 160 | func (c *Connector) ValidateNetRecordBody(ctx context.Context, body format.Node, identity thread.PubKey) error { 161 | return c.app.ValidateNetRecordBody(ctx, body, identity) 162 | } 163 | 164 | // HandleNetRecord calls the connection app's HandleNetRecord while supplying thread key. 165 | func (c *Connector) HandleNetRecord(ctx context.Context, rec net.ThreadRecord) error { 166 | return c.app.HandleNetRecord(ctx, rec, c.threadKey) 167 | } 168 | -------------------------------------------------------------------------------- /core/db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "crypto/rand" 5 | "strings" 6 | 7 | ds "github.com/ipfs/go-datastore" 8 | "github.com/ipfs/go-ipld-format" 9 | ulid "github.com/oklog/ulid/v2" 10 | ) 11 | 12 | const ( 13 | // EmptyInstanceID represents an empty InstanceID. 14 | EmptyInstanceID = InstanceID("") 15 | ) 16 | 17 | // InstanceID is the type used in instance identities. 18 | type InstanceID string 19 | 20 | // NewInstanceID generates a new identity for an instance. 21 | func NewInstanceID() InstanceID { 22 | id := ulid.MustNew(ulid.Now(), rand.Reader) 23 | return InstanceID(strings.ToLower(id.String())) 24 | } 25 | 26 | func (e InstanceID) String() string { 27 | return string(e) 28 | } 29 | 30 | // Event is a local or remote event generated in collection and dispatcher 31 | // by Dispatcher. 32 | type Event interface { 33 | // Time (wall-clock) the event was created. 34 | Time() []byte 35 | // InstanceID is the associated instance's unique identifier. 36 | InstanceID() InstanceID 37 | // Collection is the associated instance's collection name. 38 | Collection() string 39 | // Marshal the event to JSON. 40 | Marshal() ([]byte, error) 41 | } 42 | 43 | // ActionType is the type used by actions done in a txn. 44 | type ActionType int 45 | 46 | const ( 47 | // Create indicates the creation of an instance in a txn. 48 | Create ActionType = iota 49 | // Save indicates the mutation of an instance in a txn. 50 | Save 51 | // Delete indicates the deletion of an instance by ID in a txn. 52 | Delete 53 | ) 54 | 55 | // Action is a operation done in the collection. 56 | type Action struct { 57 | // Type of the action. 58 | Type ActionType 59 | // InstanceID of the instance in action. 60 | InstanceID InstanceID 61 | // CollectionName of the instance in action. 62 | CollectionName string 63 | // Previous is the instance before the action. 64 | Previous []byte 65 | // Current is the instance after the action was done. 66 | Current []byte 67 | } 68 | 69 | type ReduceAction struct { 70 | // Type of the reduced action. 71 | Type ActionType 72 | // Collection in which action was made. 73 | Collection string 74 | // InstanceID of the instance in reduced action. 75 | InstanceID InstanceID 76 | } 77 | 78 | // IndexFunc handles index updates. 79 | type IndexFunc func(collection string, key ds.Key, oldData, newData []byte, txn ds.Txn) error 80 | 81 | // EventCodec transforms actions generated in collections to 82 | // events dispatched to thread logs, and viceversa. 83 | type EventCodec interface { 84 | // Reduce applies generated events into state. 85 | Reduce(events []Event, store ds.TxnDatastore, baseKey ds.Key, indexFunc IndexFunc) ([]ReduceAction, error) 86 | // Create corresponding events to be dispatched. 87 | Create(ops []Action) ([]Event, format.Node, error) 88 | // EventsFromBytes deserializes a format.Node bytes payload into Events. 89 | EventsFromBytes(data []byte) ([]Event, error) 90 | } 91 | -------------------------------------------------------------------------------- /core/doc.go: -------------------------------------------------------------------------------- 1 | // Package core defines the core types, constants, data structures and interfaces for go-threads. 2 | package core 3 | -------------------------------------------------------------------------------- /core/net/event.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ipfs/go-cid" 7 | "github.com/ipfs/go-ipld-format" 8 | "github.com/textileio/crypto" 9 | ) 10 | 11 | // Event is the Block format used by threads 12 | type Event interface { 13 | format.Node 14 | 15 | // HeaderID returns the cid of the event header. 16 | HeaderID() cid.Cid 17 | 18 | // GetHeader loads and optionally decrypts the event header. 19 | // If no key is given, the header time and key methods will return an error. 20 | GetHeader(context.Context, format.DAGService, crypto.DecryptionKey) (EventHeader, error) 21 | 22 | // BodyID returns the cid of the event body. 23 | BodyID() cid.Cid 24 | 25 | // GetBody loads and optionally decrypts the event body. 26 | GetBody(context.Context, format.DAGService, crypto.DecryptionKey) (format.Node, error) 27 | } 28 | 29 | // EventHeader is the format of the event's header object 30 | type EventHeader interface { 31 | format.Node 32 | 33 | // Key returns a single-use decryption key for the event body. 34 | Key() (crypto.DecryptionKey, error) 35 | } 36 | -------------------------------------------------------------------------------- /core/net/net.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | 8 | "github.com/ipfs/go-cid" 9 | "github.com/ipfs/go-ipld-format" 10 | "github.com/libp2p/go-libp2p-core/host" 11 | "github.com/libp2p/go-libp2p-core/peer" 12 | ma "github.com/multiformats/go-multiaddr" 13 | "github.com/textileio/go-threads/core/thread" 14 | ) 15 | 16 | // Net wraps API with a DAGService and libp2p host. 17 | type Net interface { 18 | API 19 | 20 | // DAGService provides a DAG API to the network. 21 | format.DAGService 22 | 23 | // Host provides a network identity. 24 | Host() host.Host 25 | } 26 | 27 | // API is the network interface for thread orchestration. 28 | type API interface { 29 | io.Closer 30 | 31 | // GetHostID returns the host's peer id. 32 | GetHostID(ctx context.Context) (peer.ID, error) 33 | 34 | // GetToken returns a signed token representing an identity that can be used with other API methods, e.g., 35 | // CreateThread, AddThread, etc. 36 | GetToken(ctx context.Context, identity thread.Identity) (thread.Token, error) 37 | 38 | // CreateThread creates and adds a new thread with id and opts. 39 | CreateThread(ctx context.Context, id thread.ID, opts ...NewThreadOption) (thread.Info, error) 40 | 41 | // AddThread adds an existing thread from a multiaddress and opts. 42 | AddThread(ctx context.Context, addr ma.Multiaddr, opts ...NewThreadOption) (thread.Info, error) 43 | 44 | // GetThread returns thread info by id. 45 | GetThread(ctx context.Context, id thread.ID, opts ...ThreadOption) (thread.Info, error) 46 | 47 | // PullThread requests new records from each known thread host. 48 | // This method is called internally on an interval as part of the orchestration protocol. 49 | // Calling it manually can be useful when new records are known to be available. 50 | PullThread(ctx context.Context, id thread.ID, opts ...ThreadOption) error 51 | 52 | // DeleteThread removes a thread by id and opts. 53 | DeleteThread(ctx context.Context, id thread.ID, opts ...ThreadOption) error 54 | 55 | // AddReplicator replicates a thread by id on a different host. 56 | // All logs and records are pushed to the new host. 57 | AddReplicator(ctx context.Context, id thread.ID, paddr ma.Multiaddr, opts ...ThreadOption) (peer.ID, error) 58 | 59 | // CreateRecord creates and adds a new record with body to a thread by id. 60 | CreateRecord(ctx context.Context, id thread.ID, body format.Node, opts ...ThreadOption) (ThreadRecord, error) 61 | 62 | // AddRecord add an existing record to a thread by id and lid. 63 | AddRecord(ctx context.Context, id thread.ID, lid peer.ID, rec Record, opts ...ThreadOption) error 64 | 65 | // GetRecord returns a record by thread id and cid. 66 | GetRecord(ctx context.Context, id thread.ID, rid cid.Cid, opts ...ThreadOption) (Record, error) 67 | 68 | // Subscribe returns a read-only channel that receives newly created / added thread records. 69 | // Cancelling the context effectively unsubscribes and releases the resources. 70 | Subscribe(ctx context.Context, opts ...SubOption) (<-chan ThreadRecord, error) 71 | } 72 | 73 | // Token is used to restrict network APIs to a single app.App. 74 | // In other words, a net token protects against writes and deletes 75 | // which are external to an app. 76 | type Token []byte 77 | 78 | // Equal returns whether or not the token is equal to the given value. 79 | func (t Token) Equal(b Token) bool { 80 | return bytes.Equal(t, b) 81 | } 82 | -------------------------------------------------------------------------------- /core/net/options.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "github.com/libp2p/go-libp2p-core/crypto" 5 | "github.com/textileio/go-threads/core/thread" 6 | ) 7 | 8 | // NewThreadOptions defines options to be used when creating / adding a thread. 9 | type NewThreadOptions struct { 10 | ThreadKey thread.Key 11 | LogKey crypto.Key 12 | Token thread.Token 13 | } 14 | 15 | // NewThreadOption specifies new thread options. 16 | type NewThreadOption func(*NewThreadOptions) 17 | 18 | // WithThreadKey handles log encryption. 19 | func WithThreadKey(key thread.Key) NewThreadOption { 20 | return func(args *NewThreadOptions) { 21 | args.ThreadKey = key 22 | } 23 | } 24 | 25 | // WithLogKey is the public or private key used to write log records. 26 | // If this is just a public key, the service itself won't be able to create records. 27 | // In other words, all records must be pre-created and added with AddRecord. 28 | // If no log key is provided, one will be created internally. 29 | func WithLogKey(key crypto.Key) NewThreadOption { 30 | return func(args *NewThreadOptions) { 31 | args.LogKey = key 32 | } 33 | } 34 | 35 | // WithNewThreadToken provides authorization for creating a new thread. 36 | func WithNewThreadToken(t thread.Token) NewThreadOption { 37 | return func(args *NewThreadOptions) { 38 | args.Token = t 39 | } 40 | } 41 | 42 | // ThreadOptions defines options for interacting with a thread. 43 | type ThreadOptions struct { 44 | Token thread.Token 45 | APIToken Token 46 | } 47 | 48 | // ThreadOption specifies thread options. 49 | type ThreadOption func(*ThreadOptions) 50 | 51 | // WithThreadToken provides authorization for interacting with a thread. 52 | func WithThreadToken(t thread.Token) ThreadOption { 53 | return func(args *ThreadOptions) { 54 | args.Token = t 55 | } 56 | } 57 | 58 | // WithAPIToken provides additional authorization for interacting 59 | // with a thread as an application. 60 | // For example, this is used by a db.DB to ensure that only it can 61 | // create records or delete the underlying thread. 62 | func WithAPIToken(t Token) ThreadOption { 63 | return func(args *ThreadOptions) { 64 | args.APIToken = t 65 | } 66 | } 67 | 68 | // SubOptions defines options for a thread subscription. 69 | type SubOptions struct { 70 | ThreadIDs thread.IDSlice 71 | Token thread.Token 72 | } 73 | 74 | // SubOption is a thread subscription option. 75 | type SubOption func(*SubOptions) 76 | 77 | // WithSubFilter restricts the subscription to a given thread. 78 | // Use this option multiple times to subscribe to multiple threads. 79 | func WithSubFilter(id thread.ID) SubOption { 80 | return func(args *SubOptions) { 81 | args.ThreadIDs = append(args.ThreadIDs, id) 82 | } 83 | } 84 | 85 | // WithSubToken provides authorization for a subscription. 86 | func WithSubToken(t thread.Token) SubOption { 87 | return func(args *SubOptions) { 88 | args.Token = t 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /core/net/record.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ipfs/go-cid" 7 | "github.com/ipfs/go-ipld-format" 8 | "github.com/libp2p/go-libp2p-core/crypto" 9 | "github.com/libp2p/go-libp2p-core/peer" 10 | "github.com/textileio/go-threads/core/thread" 11 | ) 12 | 13 | // Record is the most basic component of a log. 14 | type Record interface { 15 | format.Node 16 | 17 | // BlockID returns the cid of the inner block. 18 | BlockID() cid.Cid 19 | 20 | // GetBlock loads the inner block. 21 | GetBlock(context.Context, format.DAGService) (format.Node, error) 22 | 23 | // PrevID returns the cid of the previous record. 24 | PrevID() cid.Cid 25 | 26 | // Sig returns a signature from the log key. 27 | Sig() []byte 28 | 29 | // PubKey of the identity used to author this record. 30 | PubKey() []byte 31 | 32 | // Verify returns a nil error if the node signature is valid. 33 | Verify(key crypto.PubKey) error 34 | } 35 | 36 | // ThreadRecord wraps Record within a thread and log context. 37 | type ThreadRecord interface { 38 | // Value returns the underlying record. 39 | Value() Record 40 | 41 | // ThreadID returns the record's thread ID. 42 | ThreadID() thread.ID 43 | 44 | // LogID returns the record's log ID. 45 | LogID() peer.ID 46 | } 47 | -------------------------------------------------------------------------------- /core/thread/head.go: -------------------------------------------------------------------------------- 1 | package thread 2 | 3 | import "github.com/ipfs/go-cid" 4 | 5 | // Head represents the log head (including the number of records in the log and the id of the head) 6 | type Head struct { 7 | // ID of the head 8 | ID cid.Cid 9 | // Counter is the number of logs in the head 10 | Counter int64 11 | } 12 | 13 | const CounterUndef int64 = 0 14 | 15 | var HeadUndef = Head{ 16 | ID: cid.Undef, 17 | Counter: CounterUndef, 18 | } 19 | -------------------------------------------------------------------------------- /core/thread/id_test.go: -------------------------------------------------------------------------------- 1 | package thread 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/binary" 6 | "testing" 7 | 8 | mbase "github.com/multiformats/go-multibase" 9 | ) 10 | 11 | func TestCast(t *testing.T) { 12 | i := NewIDV1(Raw, 32) 13 | j, err := Cast(i.Bytes()) 14 | if err != nil { 15 | t.Errorf("failed to cast ID %s: %s", i.String(), err) 16 | } 17 | if i != j { 18 | t.Errorf("id %v not equal to id %v", i.String(), j.String()) 19 | } 20 | } 21 | 22 | func TestDecode(t *testing.T) { 23 | i := NewIDV1(Raw, 32) 24 | t.Logf("New ID: %s", i.String()) 25 | 26 | j, err := Decode(i.String()) 27 | if err != nil { 28 | t.Errorf("failed to decode ID %s: %s", i.String(), err) 29 | } 30 | 31 | t.Logf("Decoded ID: %s", j.String()) 32 | } 33 | 34 | func TestExtractEncoding(t *testing.T) { 35 | i := NewIDV1(Raw, 16) 36 | 37 | e, err := ExtractEncoding(i.String()) 38 | if err != nil { 39 | t.Errorf("failed to extract encoding from %s: %s", i.String(), err) 40 | } 41 | 42 | t.Logf("Encoding: %s", mbase.EncodingToStr[e]) 43 | } 44 | 45 | func TestID_Version(t *testing.T) { 46 | i := NewIDV1(Raw, 16) 47 | 48 | v := i.Version() 49 | if v != V1 { 50 | t.Errorf("got wrong version from %s: %d", i.String(), v) 51 | } 52 | 53 | t.Logf("Version: %d", v) 54 | } 55 | 56 | func TestID_Variant(t *testing.T) { 57 | i := NewIDV1(Raw, 16) 58 | 59 | v := i.Variant() 60 | if v != Raw { 61 | t.Errorf("got wrong variant from %s: %d", i.String(), v) 62 | } 63 | 64 | t.Logf("Variant: %s", v) 65 | 66 | i = NewIDV1(AccessControlled, 32) 67 | 68 | v = i.Variant() 69 | if v != AccessControlled { 70 | t.Errorf("got wrong variant from %s: %d", i.String(), v) 71 | } 72 | 73 | t.Logf("Variant: %s", v) 74 | } 75 | 76 | func TestID_Valid(t *testing.T) { 77 | i := NewIDV1(Raw, 16) 78 | if err := i.Validate(); err != nil { 79 | t.Errorf("id %s is invalid", i.String()) 80 | } 81 | } 82 | 83 | func TestID_Invalid(t *testing.T) { 84 | i := makeID(t, 5, int64(Raw), 16) 85 | if err := i.Validate(); err == nil { 86 | t.Errorf("id %s is valid but it has an invalid version", i.String()) 87 | } 88 | 89 | i = makeID(t, V1, 50, 16) 90 | if err := i.Validate(); err == nil { 91 | t.Errorf("id %s is valid but it has an invalid variant", i.String()) 92 | } 93 | 94 | i = makeID(t, V1, int64(Raw), 0) 95 | if err := i.Validate(); err == nil { 96 | t.Errorf("id %s is valid but it has no random bytes", i.String()) 97 | } 98 | } 99 | 100 | func makeID(t *testing.T, version uint64, variant int64, size uint8) ID { 101 | num := make([]byte, size) 102 | _, err := rand.Read(num) 103 | if err != nil { 104 | t.Errorf("failed to generate random data: %v", err) 105 | } 106 | 107 | numlen := len(num) 108 | // two 8 bytes (max) numbers plus num 109 | buf := make([]byte, 2*binary.MaxVarintLen64+numlen) 110 | n := binary.PutUvarint(buf, version) 111 | n += binary.PutUvarint(buf[n:], uint64(variant)) 112 | cn := copy(buf[n:], num) 113 | if cn != numlen { 114 | t.Errorf("copy length is inconsistent") 115 | } 116 | 117 | return ID(buf[:n+numlen]) 118 | } 119 | -------------------------------------------------------------------------------- /core/thread/key.go: -------------------------------------------------------------------------------- 1 | package thread 2 | 3 | import ( 4 | "fmt" 5 | 6 | mbase "github.com/multiformats/go-multibase" 7 | sym "github.com/textileio/crypto/symmetric" 8 | ) 9 | 10 | var ( 11 | // ErrInvalidKey indicates an invalid byte slice was given to KeyFromBytes. 12 | ErrInvalidKey = fmt.Errorf("invalid key") 13 | ) 14 | 15 | // Key is a thread encryption key with two components. 16 | // Service key is used to encrypt outer log record linkages. 17 | // Read key is used to encrypt inner record events. 18 | type Key struct { 19 | sk *sym.Key 20 | rk *sym.Key 21 | } 22 | 23 | // NewKey wraps service and read keys. 24 | func NewKey(sk, rk *sym.Key) Key { 25 | if sk == nil { 26 | panic("service-key must not be nil") 27 | } 28 | return Key{sk: sk, rk: rk} 29 | } 30 | 31 | // NewServiceKey wraps a service-only key. 32 | func NewServiceKey(sk *sym.Key) Key { 33 | return Key{sk: sk} 34 | } 35 | 36 | // NewRandomKey returns a random key, which includes a service and read key. 37 | func NewRandomKey() Key { 38 | return Key{sk: sym.New(), rk: sym.New()} 39 | } 40 | 41 | // NewRandomServiceKey returns a random service-only key. 42 | func NewRandomServiceKey() Key { 43 | return Key{sk: sym.New()} 44 | } 45 | 46 | // KeyFromBytes returns a key by wrapping k. 47 | func KeyFromBytes(b []byte) (k Key, err error) { 48 | if len(b) != sym.KeyBytes && len(b) != sym.KeyBytes*2 { 49 | return k, ErrInvalidKey 50 | } 51 | sk, err := sym.FromBytes(b[:sym.KeyBytes]) 52 | if err != nil { 53 | return k, err 54 | } 55 | var rk *sym.Key 56 | if len(b) == sym.KeyBytes*2 { 57 | rk, err = sym.FromBytes(b[sym.KeyBytes:]) 58 | if err != nil { 59 | return k, err 60 | } 61 | } 62 | return Key{sk: sk, rk: rk}, nil 63 | } 64 | 65 | // KeyFromString returns a key by decoding a base32-encoded string. 66 | func KeyFromString(s string) (k Key, err error) { 67 | _, b, err := mbase.Decode(s) 68 | if err != nil { 69 | return k, err 70 | } 71 | return KeyFromBytes(b) 72 | } 73 | 74 | // Service returns the service key. 75 | func (k Key) Service() *sym.Key { 76 | return k.sk 77 | } 78 | 79 | // Read returns the read key. 80 | func (k Key) Read() *sym.Key { 81 | return k.rk 82 | } 83 | 84 | // Defined returns whether or not key has any defined components. 85 | // Since it's not possible to have a read key w/o a service key, 86 | // we just need to check service key. 87 | func (k Key) Defined() bool { 88 | return k.sk != nil 89 | } 90 | 91 | // CanRead returns whether or not read key is available. 92 | func (k Key) CanRead() bool { 93 | return k.rk != nil 94 | } 95 | 96 | // MarshalBinary implements BinaryMarshaler. 97 | func (k Key) MarshalBinary() ([]byte, error) { 98 | return k.Bytes(), nil 99 | } 100 | 101 | // Bytes returns raw key bytes. 102 | func (k Key) Bytes() []byte { 103 | if k.rk != nil { 104 | return append(k.sk.Bytes(), k.rk.Bytes()...) 105 | } else if k.sk != nil { 106 | return k.sk.Bytes() 107 | } else { 108 | return nil 109 | } 110 | } 111 | 112 | // String returns the base32-encoded string representation of raw key bytes. 113 | // For example, 114 | // Full: "brv7t5l2h55uklz5qwpntcat26csaasfchzof3emmdy6povabcd3a2to2qdkqdkto2prfhizerqqudqsdvwherbiy4nazqxjejgdr4oy" 115 | // Service: "bp2vvqody5zm6yqycsnazb4kpqvycbdosos352zvpsorxce5koh7q" 116 | func (k Key) String() string { 117 | str, err := mbase.Encode(mbase.Base32, k.Bytes()) 118 | if err != nil { 119 | panic("should not error with hardcoded mbase: " + err.Error()) 120 | } 121 | return str 122 | } 123 | -------------------------------------------------------------------------------- /core/thread/key_test.go: -------------------------------------------------------------------------------- 1 | package thread 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | func TestNewRandomKey(t *testing.T) { 9 | k := NewRandomKey() 10 | if k.sk == nil { 11 | t.Fatal("service key should not be nil") 12 | } 13 | if k.rk == nil { 14 | t.Fatal("read key should not be nil") 15 | } 16 | } 17 | 18 | func TestNewRandomServiceKey(t *testing.T) { 19 | k := NewRandomServiceKey() 20 | if k.sk == nil { 21 | t.Fatal("service key should not be nil") 22 | } 23 | if k.rk != nil { 24 | t.Fatal("read key should be nil") 25 | } 26 | } 27 | 28 | func TestKey_FromBytes(t *testing.T) { 29 | t.Run("full", func(t *testing.T) { 30 | k1 := NewRandomKey() 31 | b := k1.Bytes() 32 | k2, err := KeyFromBytes(b) 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | if !bytes.Equal(k2.sk.Bytes(), k1.sk.Bytes()) { 37 | t.Fatal("service keys are not equal") 38 | } 39 | if !bytes.Equal(k2.rk.Bytes(), k1.rk.Bytes()) { 40 | t.Fatal("read keys are not equal") 41 | } 42 | }) 43 | t.Run("service", func(t *testing.T) { 44 | k1 := NewRandomServiceKey() 45 | b := k1.Bytes() 46 | k2, err := KeyFromBytes(b) 47 | if err != nil { 48 | t.Fatal(err) 49 | } 50 | if !bytes.Equal(k2.sk.Bytes(), k1.sk.Bytes()) { 51 | t.Fatal("service keys are not equal") 52 | } 53 | if k2.rk != nil { 54 | t.Fatal("read key should be nil") 55 | } 56 | }) 57 | } 58 | 59 | func TestKey_FromString(t *testing.T) { 60 | t.Run("full", func(t *testing.T) { 61 | k1 := NewRandomKey() 62 | s := k1.String() 63 | k2, err := KeyFromString(s) 64 | if err != nil { 65 | t.Fatal(err) 66 | } 67 | if !bytes.Equal(k2.sk.Bytes(), k1.sk.Bytes()) { 68 | t.Fatal("service keys are not equal") 69 | } 70 | if !bytes.Equal(k2.rk.Bytes(), k1.rk.Bytes()) { 71 | t.Fatal("read keys are not equal") 72 | } 73 | }) 74 | t.Run("service", func(t *testing.T) { 75 | k1 := NewRandomServiceKey() 76 | s := k1.String() 77 | k2, err := KeyFromString(s) 78 | if err != nil { 79 | t.Fatal(err) 80 | } 81 | if !bytes.Equal(k2.sk.Bytes(), k1.sk.Bytes()) { 82 | t.Fatal("service keys are not equal") 83 | } 84 | if k2.rk != nil { 85 | t.Fatal("read key should be nil") 86 | } 87 | }) 88 | } 89 | -------------------------------------------------------------------------------- /core/thread/protocol.go: -------------------------------------------------------------------------------- 1 | package thread 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/libp2p/go-libp2p-core/protocol" 7 | ma "github.com/multiformats/go-multiaddr" 8 | mb "github.com/multiformats/go-multibase" 9 | ) 10 | 11 | const ( 12 | // Name is the protocol slug. 13 | Name = "thread" 14 | // Code is the protocol code. 15 | Code = 406 16 | // Version is the current protocol version. 17 | Version = "0.0.1" 18 | // Protocol is the threads protocol tag. 19 | Protocol protocol.ID = "/" + Name + "/" + Version 20 | ) 21 | 22 | var addrProtocol = ma.Protocol{ 23 | Name: Name, 24 | Code: Code, 25 | VCode: ma.CodeToVarint(Code), 26 | Size: ma.LengthPrefixedVarSize, 27 | Transcoder: ma.NewTranscoderFromFunctions(threadStB, threadBtS, threadVal), 28 | } 29 | 30 | func threadStB(s string) ([]byte, error) { 31 | _, data, err := mb.Decode(s) 32 | if err != nil { 33 | return nil, fmt.Errorf("failed to parse thread addr: %s %s", s, err) 34 | } 35 | return data, nil 36 | } 37 | 38 | func threadVal(b []byte) error { 39 | _, err := Cast(b) 40 | return err 41 | } 42 | 43 | func threadBtS(b []byte) (string, error) { 44 | m, err := Cast(b) 45 | if err != nil { 46 | return "", err 47 | } 48 | if err := m.Validate(); err != nil { 49 | return "", err 50 | } 51 | return m.String(), nil 52 | } 53 | 54 | func init() { 55 | if err := ma.AddProtocol(addrProtocol); err != nil { 56 | panic(err) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /core/thread/varint.go: -------------------------------------------------------------------------------- 1 | package thread 2 | 3 | // Version of varint function that work with a string rather than 4 | // []byte to avoid unnecessary allocation 5 | 6 | // Copyright 2011 The Go Authors. All rights reserved. 7 | // Use of this source code is governed by a BSD-style 8 | // license as given at https://golang.org/LICENSE 9 | 10 | // uvarint decodes a uint64 from buf and returns that value and the 11 | // number of characters read (> 0). If an error occurred, the value is 0 12 | // and the number of bytes n is <= 0 meaning: 13 | // 14 | // n == 0: buf too small 15 | // n < 0: value larger than 64 bits (overflow) 16 | // and -n is the number of bytes read 17 | // 18 | func uvarint(buf string) (uint64, int) { 19 | var x uint64 20 | var s uint 21 | // we have a binary string so we can't use a range loope 22 | for i := 0; i < len(buf); i++ { 23 | b := buf[i] 24 | if b < 0x80 { 25 | if i > 9 || i == 9 && b > 1 { 26 | return 0, -(i + 1) // overflow 27 | } 28 | return x | uint64(b)< == operations. 14 | 15 | func TestCompare(t *testing.T) { 16 | t.Parallel() 17 | tests := []struct { 18 | // a < b < c 19 | a interface{} 20 | b interface{} 21 | c interface{} 22 | }{ 23 | {a: time.Now().Add(-time.Hour), b: time.Now(), c: time.Now().Add(time.Hour)}, 24 | {a: *big.NewFloat(1.01), b: *big.NewFloat(2.2), c: *big.NewFloat(5.4)}, 25 | {a: *big.NewInt(1), b: *big.NewInt(2), c: *big.NewInt(5)}, 26 | {a: *big.NewRat(1, 3), b: *big.NewRat(1, 2), c: *big.NewRat(3, 2)}, 27 | {a: 0, b: 1, c: 2}, 28 | {a: int8(-2), b: int8(1), c: int8(2)}, 29 | {a: int16(-2), b: int16(1), c: int16(2)}, 30 | {a: int32(-2), b: int32(1), c: int32(2)}, 31 | {a: int64(-2), b: int64(1), c: int64(2)}, 32 | {a: uint(0), b: uint(1), c: uint(2)}, 33 | {a: uint8(0), b: uint8(1), c: uint8(2)}, 34 | {a: uint16(0), b: uint16(1), c: uint16(2)}, 35 | {a: uint32(0), b: uint32(1), c: uint32(2)}, 36 | {a: uint64(0), b: uint64(1), c: uint64(2)}, 37 | {a: float32(-0.3), b: float32(0.6), c: float32(2.21)}, 38 | {a: float64(-0.3), b: float64(0.6), c: float64(2.21)}, 39 | {a: "a", b: "b", c: "c"}, 40 | {a: &tstComparer{Val: -1}, b: &tstComparer{Val: 2}, c: &tstComparer{Val: 3}}, 41 | } 42 | 43 | for _, tc := range tests { 44 | tc := tc 45 | t.Run(fmt.Sprintf("%T", tc.a), func(t *testing.T) { 46 | t.Parallel() 47 | if compareTst(t, tc.a, tc.b) != -1 || 48 | compareTst(t, tc.a, tc.c) != -1 || 49 | compareTst(t, tc.b, tc.c) != -1 || 50 | compareTst(t, tc.c, tc.a) != 1 || 51 | compareTst(t, tc.c, tc.b) != 1 || 52 | compareTst(t, tc.b, tc.a) != 1 || 53 | compareTst(t, tc.a, tc.a) != 0 || 54 | compareTst(t, tc.b, tc.b) != 0 || 55 | compareTst(t, tc.c, tc.c) != 0 { 56 | t.Fatalf("compare incorrect result: %v", tc) 57 | } 58 | }) 59 | } 60 | } 61 | 62 | func compareTst(t *testing.T, v1, v2 interface{}) int { 63 | r, err := compare(v1, v2) 64 | if err != nil { 65 | t.Fatalf("error when calling compare: %v", err) 66 | } 67 | return r 68 | } 69 | 70 | type tstComparer struct { 71 | Val int 72 | } 73 | 74 | func (t *tstComparer) Compare(other interface{}) (int, error) { 75 | v := other.(*tstComparer) 76 | if t.Val == v.Val { 77 | return 0, nil 78 | } 79 | if t.Val < v.Val { 80 | return -1, nil 81 | } 82 | return 1, nil 83 | } 84 | -------------------------------------------------------------------------------- /db/design.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/textileio/go-threads/a715c240241320492cdd81aa4f7003168784c212/db/design.png -------------------------------------------------------------------------------- /db/dispatcher.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/binary" 7 | "encoding/gob" 8 | "strconv" 9 | "sync" 10 | 11 | ds "github.com/ipfs/go-datastore" 12 | "github.com/ipfs/go-datastore/query" 13 | core "github.com/textileio/go-threads/core/db" 14 | kt "github.com/textileio/go-threads/db/keytransform" 15 | "golang.org/x/sync/errgroup" 16 | ) 17 | 18 | var ( 19 | dsDispatcherPrefix = dsPrefix.ChildString("dispatcher") 20 | ) 21 | 22 | // Reducer applies an event to an existing state. 23 | type Reducer interface { 24 | Reduce(events []core.Event) error 25 | } 26 | 27 | // dispatcher is used to dispatch events to registered reducers. 28 | // 29 | // This is different from generic pub-sub systems because reducers are not subscribed to particular events. 30 | // Every event is dispatched to every registered reducer. When a given reducer is registered, it returns a `token`, 31 | // which can be used to deregister the reducer later. 32 | type dispatcher struct { 33 | store kt.TxnDatastoreExtended 34 | reducers []Reducer 35 | lock sync.RWMutex 36 | lastID int 37 | } 38 | 39 | // NewDispatcher creates a new EventDispatcher. 40 | func newDispatcher(store kt.TxnDatastoreExtended) *dispatcher { 41 | return &dispatcher{ 42 | store: store, 43 | } 44 | } 45 | 46 | // Store returns the internal event store. 47 | func (d *dispatcher) Store() kt.TxnDatastoreExtended { 48 | return d.store 49 | } 50 | 51 | // Register takes a reducer to be invoked with each dispatched event. 52 | func (d *dispatcher) Register(reducer Reducer) { 53 | d.lock.Lock() 54 | defer d.lock.Unlock() 55 | d.lastID++ 56 | d.reducers = append(d.reducers, reducer) 57 | } 58 | 59 | // Dispatch dispatches a payload to all registered reducers. 60 | // The logic is separated in two parts: 61 | // 1. Save all txn events with transaction guarantees. 62 | // 2. Notify all reducers about the known events. 63 | func (d *dispatcher) Dispatch(events []core.Event) error { 64 | d.lock.Lock() 65 | defer d.lock.Unlock() 66 | 67 | txn, err := d.store.NewTransaction(false) 68 | if err != nil { 69 | return err 70 | } 71 | defer txn.Discard() 72 | for _, event := range events { 73 | key, err := getKey(event) 74 | if err != nil { 75 | return err 76 | } 77 | // Encode and add an Event to event store 78 | b := bytes.Buffer{} 79 | e := gob.NewEncoder(&b) 80 | if err := e.Encode(event); err != nil { 81 | return err 82 | } 83 | if err := txn.Put(key, b.Bytes()); err != nil { 84 | return err 85 | } 86 | } 87 | if err := txn.Commit(); err != nil { 88 | return err 89 | } 90 | // Safe to fire off reducers now that event is persisted 91 | g, _ := errgroup.WithContext(context.Background()) 92 | for _, reducer := range d.reducers { 93 | reducer := reducer 94 | // Launch each reducer in a separate goroutine 95 | g.Go(func() error { 96 | return reducer.Reduce(events) 97 | }) 98 | } 99 | // Wait for all reducers to complete or error out 100 | if err := g.Wait(); err != nil { 101 | return err 102 | } 103 | return nil 104 | } 105 | 106 | // Query searches the internal event store and returns a query result. 107 | // This is a synchronous version of github.com/ipfs/go-datastore's Query method. 108 | func (d *dispatcher) Query(query query.Query) ([]query.Entry, error) { 109 | result, err := d.store.Query(query) 110 | if err != nil { 111 | return nil, err 112 | } 113 | return result.Rest() 114 | } 115 | 116 | // Key format: // 117 | // @todo: This is up for debate, its a 'fake' Event struct right now anyway 118 | func getKey(event core.Event) (key ds.Key, err error) { 119 | buf := bytes.NewBuffer(event.Time()) 120 | var unix int64 121 | if err = binary.Read(buf, binary.BigEndian, &unix); err != nil { 122 | return 123 | } 124 | time := strconv.FormatInt(unix, 10) 125 | key = dsDispatcherPrefix.ChildString(time). 126 | ChildString(event.Collection()). 127 | Instance(event.InstanceID().String()) 128 | 129 | return key, nil 130 | } 131 | -------------------------------------------------------------------------------- /db/dispatcher_test.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "sync" 7 | "testing" 8 | "time" 9 | 10 | ds "github.com/ipfs/go-datastore" 11 | "github.com/ipfs/go-datastore/query" 12 | core "github.com/textileio/go-threads/core/db" 13 | ) 14 | 15 | func TestNewEventDispatcher(t *testing.T) { 16 | t.Parallel() 17 | eventstore := NewTxMapDatastore() 18 | dispatcher := newDispatcher(eventstore) 19 | event := newNullEvent(time.Now()) 20 | _ = dispatcher.Dispatch([]core.Event{event}) 21 | } 22 | 23 | func TestRegister(t *testing.T) { 24 | t.Parallel() 25 | eventstore := NewTxMapDatastore() 26 | dispatcher := newDispatcher(eventstore) 27 | dispatcher.Register(&nullReducer{}) 28 | if len(dispatcher.reducers) < 1 { 29 | t.Error("expected callbacks map to have non-zero length") 30 | } 31 | } 32 | 33 | func TestDispatchLock(t *testing.T) { 34 | t.Parallel() 35 | eventstore := NewTxMapDatastore() 36 | dispatcher := newDispatcher(eventstore) 37 | dispatcher.Register(&slowReducer{}) 38 | event := newNullEvent(time.Now()) 39 | t1 := time.Now() 40 | wg := &sync.WaitGroup{} 41 | wg.Add(1) 42 | go func() { 43 | defer wg.Done() 44 | if err := dispatcher.Dispatch([]core.Event{event}); err != nil { 45 | t.Error("unexpected error in dispatch call") 46 | } 47 | }() 48 | if err := dispatcher.Dispatch([]core.Event{event}); err != nil { 49 | t.Error("unexpected error in dispatch call") 50 | } 51 | wg.Wait() 52 | t2 := time.Now() 53 | if t2.Sub(t1) < (4 * time.Second) { 54 | t.Error("reached this point too soon") 55 | } 56 | } 57 | 58 | func TestDispatch(t *testing.T) { 59 | t.Parallel() 60 | eventstore := NewTxMapDatastore() 61 | dispatcher := newDispatcher(eventstore) 62 | event := newNullEvent(time.Now()) 63 | if err := dispatcher.Dispatch([]core.Event{event}); err != nil { 64 | t.Error("unexpected error in dispatch call") 65 | } 66 | results, err := dispatcher.Query(query.Query{}) 67 | if err != nil { 68 | t.Fatalf("query failed: %v", err) 69 | } 70 | if len(results) != 1 { 71 | t.Errorf("expected 1 result, got %d", len(results)) 72 | } 73 | dispatcher.Register(&errorReducer{}) 74 | err = dispatcher.Dispatch([]core.Event{event}) 75 | if err == nil { 76 | t.Error("expected error in dispatch call") 77 | } else { 78 | if err.Error() != "error" { 79 | t.Errorf("`%s` should be `error`", err) 80 | } 81 | } 82 | results, err = dispatcher.Query(query.Query{}) 83 | if err != nil { 84 | t.Fatalf("query failed: %v", err) 85 | } 86 | if len(results) > 1 { 87 | t.Errorf("expected 1 result, got %d", len(results)) 88 | } 89 | } 90 | 91 | func TestValidStore(t *testing.T) { 92 | t.Parallel() 93 | eventstore := NewTxMapDatastore() 94 | dispatcher := newDispatcher(eventstore) 95 | store := dispatcher.Store() 96 | if store == nil { 97 | t.Error("store should not be nil") 98 | } else { 99 | if ok, _ := store.Has(ds.NewKey("blah")); ok { 100 | t.Error("store should be empty") 101 | } 102 | } 103 | } 104 | 105 | func TestDispatcherQuery(t *testing.T) { 106 | t.Parallel() 107 | eventstore := NewTxMapDatastore() 108 | dispatcher := newDispatcher(eventstore) 109 | var events []core.Event 110 | n := 100 111 | for i := 1; i <= n; i++ { 112 | events = append(events, newNullEvent(time.Now())) 113 | time.Sleep(time.Millisecond) 114 | } 115 | for _, event := range events { 116 | if err := dispatcher.Dispatch([]core.Event{event}); err != nil { 117 | t.Error("unexpected error in dispatch call") 118 | } 119 | } 120 | results, err := dispatcher.Query(query.Query{ 121 | Orders: []query.Order{query.OrderByKey{}}, 122 | }) 123 | if err != nil { 124 | t.Errorf("unexpected error: %s", err.Error()) 125 | } 126 | if len(results) != n { 127 | t.Errorf("expected %d result, got %d", n, len(results)) 128 | } 129 | } 130 | 131 | func newNullEvent(t time.Time) core.Event { 132 | return &nullEvent{Timestamp: t} 133 | } 134 | 135 | type nullEvent struct { 136 | Timestamp time.Time 137 | } 138 | 139 | func (n *nullEvent) Time() []byte { 140 | t := n.Timestamp.UnixNano() 141 | buf := new(bytes.Buffer) 142 | // Use big endian to preserve lexicographic sorting 143 | _ = binary.Write(buf, binary.BigEndian, t) 144 | return buf.Bytes() 145 | } 146 | 147 | func (n *nullEvent) InstanceID() core.InstanceID { 148 | return "null" 149 | } 150 | 151 | func (n *nullEvent) Collection() string { 152 | return "null" 153 | } 154 | 155 | func (n *nullEvent) Marshal() ([]byte, error) { 156 | return nil, nil 157 | } 158 | 159 | // Sanity check 160 | var _ core.Event = (*nullEvent)(nil) 161 | -------------------------------------------------------------------------------- /db/encode.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Tim Shannon. All rights reserved. 2 | // Use of this source code is governed by the MIT license 3 | // that can be found in the LICENSE file. 4 | 5 | package db 6 | 7 | import ( 8 | "bytes" 9 | "encoding/gob" 10 | ) 11 | 12 | // EncodeFunc is a function for encoding a value into bytes. 13 | type EncodeFunc func(value interface{}) ([]byte, error) 14 | 15 | // DecodeFunc is a function for decoding a value from bytes. 16 | type DecodeFunc func(data []byte, value interface{}) error 17 | 18 | // DefaultEncode is the default encoding func from badgerhold (Gob). 19 | func DefaultEncode(value interface{}) ([]byte, error) { 20 | var buff bytes.Buffer 21 | 22 | en := gob.NewEncoder(&buff) 23 | 24 | err := en.Encode(value) 25 | if err != nil { 26 | return nil, err 27 | } 28 | 29 | return buff.Bytes(), nil 30 | } 31 | 32 | // DefaultDecode is the default decoding func from badgerhold (Gob). 33 | func DefaultDecode(data []byte, value interface{}) error { 34 | var buff bytes.Buffer 35 | de := gob.NewDecoder(&buff) 36 | 37 | _, err := buff.Write(data) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | return de.Decode(value) 43 | } 44 | -------------------------------------------------------------------------------- /db/listeners.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | format "github.com/ipfs/go-ipld-format" 8 | "github.com/textileio/go-threads/core/app" 9 | core "github.com/textileio/go-threads/core/db" 10 | "github.com/textileio/go-threads/core/thread" 11 | ) 12 | 13 | // Listen returns a Listener which notifies about actions applying the 14 | // defined filters. The DB *won't* wait for slow receivers, so if the 15 | // channel is full, the action will be dropped. 16 | func (d *DB) Listen(los ...ListenOption) (Listener, error) { 17 | d.txnlock.Lock() 18 | defer d.txnlock.Unlock() 19 | if d.closed { 20 | return nil, fmt.Errorf("can't listen on closed DB") 21 | } 22 | 23 | sl := &listener{ 24 | scn: d.stateChangedNotifee, 25 | filters: los, 26 | c: make(chan Action, 1), 27 | } 28 | d.stateChangedNotifee.addListener(sl) 29 | return sl, nil 30 | } 31 | 32 | func (d *DB) notifyStateChanged(actions []Action) { 33 | d.stateChangedNotifee.notify(actions) 34 | } 35 | 36 | func (d *DB) notifyTxnEvents(node format.Node, token thread.Token) error { 37 | return d.localEventsBus.Send(&app.LocalEvent{ 38 | Node: node, 39 | Token: token, 40 | }) 41 | } 42 | 43 | type ActionType int 44 | type ListenActionType int 45 | 46 | const ( 47 | ActionCreate ActionType = iota + 1 48 | ActionSave 49 | ActionDelete 50 | ) 51 | 52 | const ( 53 | ListenAll ListenActionType = iota 54 | ListenCreate 55 | ListenSave 56 | ListenDelete 57 | ) 58 | 59 | type Action struct { 60 | Collection string 61 | Type ActionType 62 | ID core.InstanceID 63 | } 64 | 65 | type ListenOption struct { 66 | Type ListenActionType 67 | Collection string 68 | ID core.InstanceID 69 | } 70 | 71 | type Listener interface { 72 | Channel() <-chan Action 73 | Close() 74 | } 75 | 76 | type stateChangedNotifee struct { 77 | lock sync.RWMutex 78 | listeners []*listener 79 | } 80 | 81 | type listener struct { 82 | scn *stateChangedNotifee 83 | filters []ListenOption 84 | c chan Action 85 | } 86 | 87 | var _ Listener = (*listener)(nil) 88 | 89 | func (scn *stateChangedNotifee) notify(actions []Action) { 90 | scn.lock.RLock() 91 | defer scn.lock.RUnlock() 92 | for _, a := range actions { 93 | for _, l := range scn.listeners { 94 | if l.evaluate(a) { 95 | select { 96 | case l.c <- a: 97 | default: 98 | log.Warnf("dropped action %v for reducer with filters %v", a, l.filters) 99 | } 100 | } 101 | } 102 | } 103 | } 104 | 105 | func (scn *stateChangedNotifee) addListener(sl *listener) { 106 | scn.lock.Lock() 107 | defer scn.lock.Unlock() 108 | scn.listeners = append(scn.listeners, sl) 109 | } 110 | 111 | func (scn *stateChangedNotifee) remove(sl *listener) bool { 112 | scn.lock.Lock() 113 | defer scn.lock.Unlock() 114 | for i := range scn.listeners { 115 | if scn.listeners[i] == sl { 116 | scn.listeners[i] = scn.listeners[len(scn.listeners)-1] 117 | scn.listeners[len(scn.listeners)-1] = nil 118 | scn.listeners = scn.listeners[:len(scn.listeners)-1] 119 | return true 120 | } 121 | } 122 | return false 123 | } 124 | 125 | func (scn *stateChangedNotifee) close() { 126 | scn.lock.Lock() 127 | defer scn.lock.Unlock() 128 | for i := range scn.listeners { 129 | close(scn.listeners[i].c) 130 | scn.listeners[i] = nil 131 | } 132 | scn.listeners = nil 133 | } 134 | 135 | // Channel returns an unbuffered channel to receive 136 | // db change notifications 137 | func (sl *listener) Channel() <-chan Action { 138 | return sl.c 139 | } 140 | 141 | // Close indicates that no further notifications will be received 142 | // and ready for being garbage collected 143 | func (sl *listener) Close() { 144 | if ok := sl.scn.remove(sl); ok { 145 | close(sl.c) 146 | } 147 | } 148 | 149 | func (sl *listener) evaluate(a Action) bool { 150 | if len(sl.filters) == 0 { 151 | return true 152 | } 153 | for _, f := range sl.filters { 154 | switch f.Type { 155 | case ListenAll: 156 | case ListenCreate: 157 | if a.Type != ActionCreate { 158 | continue 159 | } 160 | case ListenSave: 161 | if a.Type != ActionSave { 162 | continue 163 | } 164 | case ListenDelete: 165 | if a.Type != ActionDelete { 166 | continue 167 | } 168 | default: 169 | panic("unknown action type") 170 | } 171 | 172 | if f.Collection != "" && f.Collection != a.Collection { 173 | continue 174 | } 175 | 176 | if f.ID != core.EmptyInstanceID && f.ID != a.ID { 177 | continue 178 | } 179 | return true 180 | } 181 | return false 182 | } 183 | -------------------------------------------------------------------------------- /db/options.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "github.com/libp2p/go-libp2p-core/crypto" 5 | core "github.com/textileio/go-threads/core/db" 6 | "github.com/textileio/go-threads/core/thread" 7 | "github.com/textileio/go-threads/jsonpatcher" 8 | ) 9 | 10 | func newDefaultEventCodec() core.EventCodec { 11 | return jsonpatcher.New() 12 | } 13 | 14 | // NewOptions defines options for creating a new db. 15 | type NewOptions struct { 16 | Name string 17 | Key thread.Key 18 | LogKey crypto.Key 19 | Collections []CollectionConfig 20 | Block bool 21 | EventCodec core.EventCodec 22 | Token thread.Token 23 | Debug bool 24 | } 25 | 26 | // NewOption specifies a new db option. 27 | type NewOption func(*NewOptions) 28 | 29 | // WithNewKey provides control over thread keys to use with a db. 30 | func WithNewKey(key thread.Key) NewOption { 31 | return func(o *NewOptions) { 32 | o.Key = key 33 | } 34 | } 35 | 36 | // WithNewLogKey is the public or private key used to write log records. 37 | // If this is just a public key, the service itself won't be able to create records. 38 | // In other words, all records must be pre-created and added with AddRecord. 39 | // If no log key is provided, one will be created internally. 40 | func WithNewLogKey(key crypto.Key) NewOption { 41 | return func(o *NewOptions) { 42 | o.LogKey = key 43 | } 44 | } 45 | 46 | // WithNewName sets the db name. 47 | func WithNewName(name string) NewOption { 48 | return func(o *NewOptions) { 49 | o.Name = name 50 | } 51 | } 52 | 53 | // WithNewCollections is used to specify collections that 54 | // will be created. 55 | func WithNewCollections(cs ...CollectionConfig) NewOption { 56 | return func(o *NewOptions) { 57 | o.Collections = cs 58 | } 59 | } 60 | 61 | // WithNewBackfillBlock makes the caller of NewDBFromAddr block until the 62 | // underlying thread is completely backfilled. 63 | // Without this, NewDBFromAddr returns immediately and thread backfilling 64 | // happens in the background. 65 | func WithNewBackfillBlock(block bool) NewOption { 66 | return func(o *NewOptions) { 67 | o.Block = block 68 | } 69 | } 70 | 71 | // WithNewEventCodec configure to use ec as the EventCodec 72 | // for transforming actions in events, and viceversa. 73 | func WithNewEventCodec(ec core.EventCodec) NewOption { 74 | return func(o *NewOptions) { 75 | o.EventCodec = ec 76 | } 77 | } 78 | 79 | // WithNewToken provides authorization for interacting with a db. 80 | func WithNewToken(t thread.Token) NewOption { 81 | return func(o *NewOptions) { 82 | o.Token = t 83 | } 84 | } 85 | 86 | // WithNewDebug indicate to output debug information. 87 | func WithNewDebug(enable bool) NewOption { 88 | return func(o *NewOptions) { 89 | o.Debug = enable 90 | } 91 | } 92 | 93 | // Options defines options for interacting with a db. 94 | type Options struct { 95 | Token thread.Token 96 | } 97 | 98 | // Option specifies a db option. 99 | type Option func(*Options) 100 | 101 | // WithToken provides authorization for interacting with a db. 102 | func WithToken(t thread.Token) Option { 103 | return func(o *Options) { 104 | o.Token = t 105 | } 106 | } 107 | 108 | // TxnOptions defines options for a transaction. 109 | type TxnOptions struct { 110 | Token thread.Token 111 | } 112 | 113 | // TxnOption specifies a transaction option. 114 | type TxnOption func(*TxnOptions) 115 | 116 | // WithTxnToken provides authorization for the transaction. 117 | func WithTxnToken(t thread.Token) TxnOption { 118 | return func(o *TxnOptions) { 119 | o.Token = t 120 | } 121 | } 122 | 123 | // NewManagedOptions defines options for creating a new managed db. 124 | type NewManagedOptions struct { 125 | Name string 126 | Key thread.Key 127 | LogKey crypto.Key 128 | Token thread.Token 129 | Collections []CollectionConfig 130 | Block bool 131 | } 132 | 133 | // NewManagedOption specifies a new managed db option. 134 | type NewManagedOption func(*NewManagedOptions) 135 | 136 | // WithNewManagedName assigns a name to a new managed db. 137 | func WithNewManagedName(name string) NewManagedOption { 138 | return func(o *NewManagedOptions) { 139 | o.Name = name 140 | } 141 | } 142 | 143 | // WithNewManagedToken provides authorization for creating a new managed db. 144 | func WithNewManagedToken(t thread.Token) NewManagedOption { 145 | return func(o *NewManagedOptions) { 146 | o.Token = t 147 | } 148 | } 149 | 150 | // WithNewManagedKey provides control over thread keys to use with a managed db. 151 | func WithNewManagedKey(key thread.Key) NewManagedOption { 152 | return func(o *NewManagedOptions) { 153 | o.Key = key 154 | } 155 | } 156 | 157 | // WithNewManagedLogKey is the public or private key used to write log records. 158 | // If this is just a public key, the service itself won't be able to create records. 159 | // In other words, all records must be pre-created and added with AddRecord. 160 | // If no log key is provided, one will be created internally. 161 | func WithNewManagedLogKey(key crypto.Key) NewManagedOption { 162 | return func(o *NewManagedOptions) { 163 | o.LogKey = key 164 | } 165 | } 166 | 167 | // WithNewManagedCollections is used to specify collections that 168 | // will be created in a managed db. 169 | func WithNewManagedCollections(cs ...CollectionConfig) NewManagedOption { 170 | return func(o *NewManagedOptions) { 171 | o.Collections = cs 172 | } 173 | } 174 | 175 | // WithNewBackfillBlock makes the caller of NewDBFromAddr block until the 176 | // underlying thread is completely backfilled. 177 | // Without this, NewDBFromAddr returns immediately and thread backfilling 178 | // happens in the background. 179 | func WithNewManagedBackfillBlock(block bool) NewManagedOption { 180 | return func(o *NewManagedOptions) { 181 | o.Block = block 182 | } 183 | } 184 | 185 | // ManagedOptions defines options for interacting with a managed db. 186 | type ManagedOptions struct { 187 | Token thread.Token 188 | } 189 | 190 | // ManagedOption specifies a managed db option. 191 | type ManagedOption func(*ManagedOptions) 192 | 193 | // WithManagedToken provides authorization for interacting with a managed db. 194 | func WithManagedToken(t thread.Token) ManagedOption { 195 | return func(o *ManagedOptions) { 196 | o.Token = t 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /db/testutils_test.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | "time" 9 | 10 | "github.com/textileio/go-threads/common" 11 | "github.com/textileio/go-threads/core/thread" 12 | "github.com/textileio/go-threads/util" 13 | ) 14 | 15 | func checkErr(t *testing.T, err error) { 16 | t.Helper() 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | } 21 | 22 | func createTestDB(t *testing.T, opts ...NewOption) (*DB, func()) { 23 | dir, err := ioutil.TempDir("", "") 24 | checkErr(t, err) 25 | n, err := common.DefaultNetwork( 26 | common.WithNetBadgerPersistence(dir), 27 | common.WithNetHostAddr(util.FreeLocalAddr()), 28 | common.WithNetDebug(true), 29 | ) 30 | checkErr(t, err) 31 | store, err := util.NewBadgerDatastore(dir, "eventstore", false) 32 | checkErr(t, err) 33 | d, err := NewDB(context.Background(), store, n, thread.NewIDV1(thread.Raw, 32), opts...) 34 | checkErr(t, err) 35 | return d, func() { 36 | time.Sleep(time.Second) // Give threads a chance to finish work 37 | if err := n.Close(); err != nil { 38 | panic(err) 39 | } 40 | if err := store.Close(); err != nil { 41 | panic(err) 42 | } 43 | _ = os.RemoveAll(dir) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /dist/README.md: -------------------------------------------------------------------------------- 1 | # `threadsd` 2 | 3 | This is the [ThreadDB](https://github.com/textileio/go-threads/tree/master/threadsd) daemon. 4 | 5 | ## Install 6 | 7 | To install it, move the binary somewhere in your `$PATH`: 8 | 9 | ```sh 10 | sudo mv threadsd /usr/local/bin/threadsd 11 | ``` 12 | 13 | Or run `sudo ./install` which does this for you. 14 | 15 | ## Usage 16 | 17 | ```sh 18 | threadsd --help 19 | ``` -------------------------------------------------------------------------------- /dist/install: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -Eeuo pipefail 3 | 4 | # From https://github.com/ipfs/go-ipfs/blob/ccef991a194beaedf009b1d0702b1150db3da0a6/cmd/ipfs/dist/install.sh 5 | # 6 | # Installation script for textile. It tries to move $bin in one of the 7 | # directories stored in $binpaths. 8 | 9 | INSTALL_DIR="$(dirname "$0")" 10 | 11 | threadsd="$INSTALL_DIR/threadsd" 12 | binpaths="/usr/local/bin /usr/bin" 13 | 14 | # This variable contains a nonzero length string in case the script fails 15 | # because of missing write permissions. 16 | is_write_perm_missing="" 17 | 18 | for binpath in $binpaths; do 19 | if mv "$threadsd" "$binpath/$threadsd" 2>/dev/null; then 20 | echo "Moved $threadsd to $binpath" 21 | exit 0 22 | else 23 | if test -d "$binpath" && ! test -w "$binpath"; then 24 | is_write_perm_missing=1 25 | fi 26 | fi 27 | done 28 | 29 | echo "We cannot install $threadsd in one of the directories $binpaths" 30 | 31 | if test -n "$is_write_perm_missing"; then 32 | echo "It seems that we do not have the necessary write permissions." 33 | echo "Perhaps try running this script as a privileged user:" 34 | echo 35 | echo " sudo $0" 36 | echo 37 | fi 38 | 39 | exit 1 -------------------------------------------------------------------------------- /docker-compose-dev.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | threaddb: 4 | build: 5 | context: . 6 | environment: 7 | - THRDS_REPO 8 | - THRDS_HOSTADDR 9 | - THRDS_APIADDR=/ip4/0.0.0.0/tcp/6006 10 | - THRDS_APIPROXYADDR=/ip4/0.0.0.0/tcp/6007 11 | - THRDS_CONNLOWWATER 12 | - THRDS_CONNHIGHWATER 13 | - THRDS_CONNGRACEPERIOD 14 | - THRDS_KEEPALIVEINTERVAL 15 | - THRDS_ENABLENETPUBSUB 16 | - THRDS_MONGOURI=mongodb://mongo:27017 17 | - THRDS_MONGODATABASE=threaddb 18 | - THRDS_DEBUG=true 19 | ports: 20 | - "4006:4006" 21 | - "127.0.0.1:6006:6006" 22 | - "127.0.0.1:6007:6007" 23 | depends_on: 24 | - mongo 25 | mongo: 26 | image: mongo:latest 27 | ports: 28 | - "127.0.0.1:27017:27017" 29 | command: 30 | - /bin/bash 31 | - -c 32 | - | 33 | /usr/bin/mongod --fork --logpath /var/log/mongod.log --bind_ip_all --replSet rs0 34 | mongo --eval 'rs.initiate({_id: "rs0", version: 1, members: [{ _id: 0, host: "mongo:27017" }]})' 35 | tail -f /var/log/mongod.log 36 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | threads: 4 | image: textile/go-threads:latest 5 | restart: always 6 | volumes: 7 | - "${THRDS_REPO}/threaddb:/data/threaddb" 8 | environment: 9 | - THRDS_REPO 10 | - THRDS_HOSTADDR 11 | - THRDS_APIADDR=/ip4/0.0.0.0/tcp/6006 12 | - THRDS_APIPROXYADDR=/ip4/0.0.0.0/tcp/6007 13 | - THRDS_CONNLOWWATER 14 | - THRDS_CONNHIGHWATER 15 | - THRDS_CONNGRACEPERIOD 16 | - THRDS_KEEPALIVEINTERVAL 17 | - THRDS_ENABLENETPUBSUB 18 | - THRDS_MONGOURI=mongodb://mongo:27017 19 | - THRDS_MONGODATABASE=threaddb 20 | - THRDS_DEBUG 21 | ports: 22 | - "4006:4006" 23 | - "127.0.0.1:6006:6006" 24 | - "127.0.0.1:6007:6007" 25 | depends_on: 26 | - mongo 27 | mongo: 28 | image: mongo:latest 29 | restart: always 30 | volumes: 31 | - "${THRDS_REPO}/mongodb:/data/mongodb" 32 | ports: 33 | - "127.0.0.1:27017:27017" 34 | command: 35 | - /bin/bash 36 | - -c 37 | - | 38 | /usr/bin/mongod --fork --logpath /var/log/mongod.log --bind_ip_all --replSet rs0 39 | mongo --eval 'rs.initiate({_id: "rs0", version: 1, members: [{ _id: 0, host: "mongo:27017" }]})' 40 | tail -f /var/log/mongod.log 41 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/textileio/go-threads 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/alecthomas/jsonschema v0.0.0-20191017121752-4bb6e3fae4f2 7 | github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect 8 | github.com/dgraph-io/badger v1.6.2 9 | github.com/dgtony/collections v0.1.6 10 | github.com/dlclark/regexp2 v1.2.0 // indirect 11 | github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 12 | github.com/evanphx/json-patch v4.5.0+incompatible 13 | github.com/fsnotify/fsnotify v1.4.9 14 | github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect 15 | github.com/gogo/googleapis v1.3.1 // indirect 16 | github.com/gogo/protobuf v1.3.2 17 | github.com/gogo/status v1.1.0 18 | github.com/golang-jwt/jwt v3.2.2+incompatible 19 | github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 20 | github.com/hashicorp/go-multierror v1.1.1 21 | github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d 22 | github.com/hsanjuan/ipfs-lite v1.1.21 23 | github.com/improbable-eng/grpc-web v0.14.1 24 | github.com/ipfs/go-block-format v0.0.3 25 | github.com/ipfs/go-blockservice v0.1.4 26 | github.com/ipfs/go-cid v0.0.7 27 | github.com/ipfs/go-datastore v0.4.5 28 | github.com/ipfs/go-ipfs-blockstore v1.0.4 29 | github.com/ipfs/go-ipfs-exchange-offline v0.0.1 30 | github.com/ipfs/go-ipfs-util v0.0.2 31 | github.com/ipfs/go-ipld-cbor v0.0.5 32 | github.com/ipfs/go-ipld-format v0.2.0 33 | github.com/ipfs/go-log/v2 v2.3.0 34 | github.com/ipfs/go-merkledag v0.3.2 35 | github.com/libp2p/go-libp2p v0.14.4 36 | github.com/libp2p/go-libp2p-connmgr v0.2.4 37 | github.com/libp2p/go-libp2p-core v0.8.6 38 | github.com/libp2p/go-libp2p-gostream v0.3.1 39 | github.com/libp2p/go-libp2p-peerstore v0.2.8 40 | github.com/libp2p/go-libp2p-pubsub v0.5.4 41 | github.com/multiformats/go-multiaddr v0.3.3 42 | github.com/multiformats/go-multibase v0.0.3 43 | github.com/multiformats/go-multihash v0.0.15 44 | github.com/multiformats/go-varint v0.0.6 45 | github.com/namsral/flag v1.7.4-pre 46 | github.com/oklog/ulid/v2 v2.0.2 47 | github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 48 | github.com/rs/cors v1.7.0 // indirect 49 | github.com/textileio/crypto v0.0.0-20210928200545-9b5a55171e1b 50 | github.com/textileio/go-datastore-extensions v1.0.1 51 | github.com/textileio/go-ds-badger v0.2.7-0.20201204225019-4ee78c4a40e2 52 | github.com/textileio/go-ds-mongo v0.1.5 53 | github.com/textileio/go-libp2p-pubsub-rpc v0.0.5 54 | github.com/tidwall/gjson v1.10.2 55 | github.com/tidwall/sjson v1.2.3 56 | github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc 57 | github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect 58 | github.com/xeipuuv/gojsonschema v1.2.0 59 | go.uber.org/zap v1.19.0 60 | golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5 61 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 62 | google.golang.org/grpc v1.39.0 63 | google.golang.org/protobuf v1.27.1 64 | nhooyr.io/websocket v1.8.7 // indirect 65 | ) 66 | -------------------------------------------------------------------------------- /integrationtests/foldersync/watcher/watcher.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/fsnotify/fsnotify" 7 | logging "github.com/ipfs/go-log/v2" 8 | ) 9 | 10 | var ( 11 | log = logging.Logger("watcher") 12 | ) 13 | 14 | type Handler func(fileName string) error 15 | 16 | type FolderWatcher struct { 17 | w *fsnotify.Watcher 18 | onCreate Handler 19 | 20 | stopWatch chan struct{} 21 | done chan struct{} 22 | 23 | lock sync.Mutex 24 | started bool 25 | closed bool 26 | } 27 | 28 | func New(path string, onCreate Handler) (*FolderWatcher, error) { 29 | watcher, err := fsnotify.NewWatcher() 30 | if err != nil { 31 | log.Fatal(err) 32 | } 33 | if err = watcher.Add(path); err != nil { 34 | return nil, err 35 | } 36 | 37 | return &FolderWatcher{ 38 | w: watcher, 39 | onCreate: onCreate, 40 | stopWatch: make(chan struct{}), 41 | done: make(chan struct{}), 42 | }, nil 43 | } 44 | 45 | func (fw *FolderWatcher) Close() { 46 | fw.lock.Lock() 47 | defer fw.lock.Unlock() 48 | 49 | if !fw.started || fw.closed { 50 | return 51 | } 52 | fw.closed = true 53 | 54 | close(fw.stopWatch) 55 | <-fw.done 56 | _ = fw.w.Close() 57 | } 58 | 59 | func (fw *FolderWatcher) Watch() { 60 | fw.lock.Lock() 61 | defer fw.lock.Unlock() 62 | if fw.started { 63 | return 64 | } 65 | 66 | fw.started = true 67 | go func() { 68 | for { 69 | select { 70 | case <-fw.stopWatch: 71 | log.Info("grafceful shutdown") 72 | close(fw.done) 73 | return 74 | case event, ok := <-fw.w.Events: 75 | if !ok { 76 | return 77 | } 78 | if event.Op&fsnotify.Create == fsnotify.Create { 79 | log.Debug("created file:", event.Name) 80 | 81 | if err := fw.onCreate(event.Name); err != nil { 82 | log.Errorf("error when calling onCreate for %s", event.Name) 83 | } 84 | } 85 | case err, ok := <-fw.w.Errors: 86 | if !ok { 87 | return 88 | } 89 | log.Error(err) 90 | } 91 | } 92 | }() 93 | } 94 | -------------------------------------------------------------------------------- /integrationtests/migration/Makefile: -------------------------------------------------------------------------------- 1 | .EXPORT_ALL_VARIABLES: 2 | 3 | RUN_MIGRATION_TEST="true" 4 | ENV_FILENAME="thread_ids" 5 | ENV_NUM_RECORDS="10" 6 | ENV_NUM_THREADS="10" 7 | 8 | test-migration: 9 | cd generate && ENV_REPO_PATH="../generated" go run main.go 10 | ENV_REPO_PATH="generated" go test ./... || rm -rf generated 11 | rm -rf generated 12 | -------------------------------------------------------------------------------- /integrationtests/migration/generate/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/textileio/go-threads/integrationtests/migration 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/google/gofuzz v1.0.0 7 | github.com/ipfs/go-ipld-cbor v0.0.5 8 | github.com/multiformats/go-multiaddr v0.3.3 // indirect 9 | github.com/multiformats/go-multihash v0.0.15 10 | github.com/smartystreets/goconvey v1.6.4 // indirect 11 | github.com/textileio/go-threads v1.1.0-rc1.0.20210609142634-18b4ea73088f 12 | google.golang.org/grpc v1.37.0 // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /integrationtests/migration/generate/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "strconv" 8 | 9 | fuzz "github.com/google/gofuzz" 10 | ipldcbor "github.com/ipfs/go-ipld-cbor" 11 | "github.com/multiformats/go-multihash" 12 | "github.com/textileio/go-threads/common" 13 | "github.com/textileio/go-threads/core/thread" 14 | "github.com/textileio/go-threads/util" 15 | ) 16 | 17 | var fuzzer = fuzz.New() 18 | 19 | const DefaultRepoPath = "../generated" 20 | const ThreadIdsFilename = "thread_ids" 21 | const NumRecords = 10 22 | const NumThreads = 10 23 | 24 | func main() { 25 | repoPath := os.Getenv("ENV_REPO_PATH") 26 | if repoPath == "" { 27 | repoPath = DefaultRepoPath 28 | } 29 | numRecords, err := getEnvInt("ENV_NUM_RECORDS") 30 | if err != nil { 31 | numRecords = NumRecords 32 | } 33 | numThreads, err := getEnvInt("ENV_NUM_THREADS") 34 | if err != nil { 35 | numThreads = NumThreads 36 | } 37 | fileName := os.Getenv("ENV_FILENAME") 38 | if fileName == "" { 39 | fileName = ThreadIdsFilename 40 | } 41 | 42 | os.RemoveAll(repoPath) 43 | err = genStore(repoPath, fileName, numThreads, numRecords) 44 | if err != nil { 45 | fmt.Println(err) 46 | } 47 | } 48 | 49 | func genStore(repoPath string, fileName string, numThreads int, numRecords int) error { 50 | net, err := newNetwork(repoPath) 51 | if err != nil { 52 | return err 53 | } 54 | ctx := context.Background() 55 | 56 | filePath := repoPath + "/" + fileName 57 | file, err := os.Create(filePath) 58 | if err != nil { 59 | return err 60 | } 61 | defer file.Close() 62 | 63 | for i := 0; i < numThreads; i++ { 64 | id, err := createThreadWithRecords(ctx, net, numRecords) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | _, err = file.WriteString(id.String() + "\n") 70 | if err != nil { 71 | return err 72 | } 73 | 74 | err = file.Sync() 75 | if err != nil { 76 | return err 77 | } 78 | } 79 | 80 | return nil 81 | } 82 | 83 | func createThreadWithRecords(ctx context.Context, net common.NetBoostrapper, numRecords int) (thread.ID, error) { 84 | id := thread.NewIDV1(thread.Raw, 32) 85 | 86 | _, err := net.CreateThread(ctx, id) 87 | if err != nil { 88 | return "", err 89 | } 90 | 91 | for i := 0; i < numRecords; i++ { 92 | obj := make(map[string][]byte) 93 | fuzzer.Fuzz(&obj) 94 | body, err := ipldcbor.WrapObject(obj, multihash.SHA2_256, -1) 95 | if err != nil { 96 | return "", err 97 | } 98 | 99 | _, err = net.CreateRecord(ctx, id, body) 100 | if err != nil { 101 | return "", err 102 | } 103 | } 104 | 105 | return id, nil 106 | } 107 | 108 | func newNetwork(repoPath string) (common.NetBoostrapper, error) { 109 | network, err := common.DefaultNetwork( 110 | common.WithNetBadgerPersistence(repoPath), 111 | common.WithNetHostAddr(util.FreeLocalAddr()), 112 | common.WithNetDebug(true), 113 | ) 114 | if err != nil { 115 | return nil, err 116 | } 117 | return network, nil 118 | } 119 | 120 | func getEnvInt(key string) (int, error) { 121 | s := os.Getenv(key) 122 | if s == "" { 123 | return 0, fmt.Errorf("no such variable") 124 | } 125 | 126 | v, err := strconv.Atoi(s) 127 | if err != nil { 128 | return 0, err 129 | } 130 | return v, nil 131 | } 132 | -------------------------------------------------------------------------------- /integrationtests/migration/migration_test.go: -------------------------------------------------------------------------------- 1 | package migration 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | "os" 9 | "strconv" 10 | "testing" 11 | 12 | "github.com/textileio/go-threads/common" 13 | "github.com/textileio/go-threads/core/thread" 14 | "github.com/textileio/go-threads/util" 15 | ) 16 | 17 | const DefaultRepoPath = "generated" 18 | const ThreadIdsFilename = "thread_ids" 19 | const NumRecords = 10 20 | 21 | func TestMigration(t *testing.T) { 22 | if os.Getenv("RUN_MIGRATION_TEST") == "" { 23 | t.Skip("Skipping migration test") 24 | } 25 | // reading environment 26 | repoPath := os.Getenv("ENV_REPO_PATH") 27 | if repoPath == "" { 28 | repoPath = DefaultRepoPath 29 | } 30 | numRecords, err := getEnvInt("ENV_NUM_RECORDS") 31 | if err != nil { 32 | numRecords = NumRecords 33 | } 34 | fileName := os.Getenv("ENV_FILENAME") 35 | if fileName == "" { 36 | fileName = ThreadIdsFilename 37 | } 38 | net, err := newNetwork(repoPath) 39 | if err != nil { 40 | t.Fatalf("error in creating network") 41 | } 42 | 43 | // reading all thread ids from file 44 | file, err := os.Open(repoPath + "/" + fileName) 45 | if err != nil { 46 | t.Fatalf("error in opening file") 47 | } 48 | defer file.Close() 49 | 50 | reader := bufio.NewReader(file) 51 | var line string 52 | var threadIds []thread.ID 53 | for { 54 | line, err = reader.ReadString('\n') 55 | if err != nil { 56 | break 57 | } 58 | line = line[:len(line)-1] 59 | id, err := thread.Decode(line) 60 | if err != nil { 61 | t.Fatalf("error while decoding string %s", err) 62 | } 63 | threadIds = append(threadIds, id) 64 | } 65 | if err != io.EOF { 66 | t.Fatalf("got error while reading from file") 67 | } 68 | 69 | ctx := context.Background() 70 | for _, id := range threadIds { 71 | info, err := net.GetThread(ctx, id) 72 | if err != nil { 73 | t.Fatalf("could not get thread with id: %s", err) 74 | } 75 | 76 | for _, log := range info.Logs { 77 | if log.Head.Counter != int64(numRecords) { 78 | t.Errorf("incorrect number of records, expected %d, but got %d", numRecords, log.Head.Counter) 79 | } 80 | } 81 | } 82 | } 83 | 84 | func newNetwork(repoPath string) (common.NetBoostrapper, error) { 85 | network, err := common.DefaultNetwork( 86 | common.WithNetBadgerPersistence(repoPath), 87 | common.WithNetHostAddr(util.FreeLocalAddr()), 88 | common.WithNetDebug(true), 89 | ) 90 | if err != nil { 91 | return nil, err 92 | } 93 | return network, nil 94 | } 95 | 96 | func getEnvInt(key string) (int, error) { 97 | s := os.Getenv(key) 98 | if s == "" { 99 | return 0, fmt.Errorf("no such variable") 100 | } 101 | 102 | v, err := strconv.Atoi(s) 103 | if err != nil { 104 | return 0, err 105 | } 106 | return v, nil 107 | } 108 | -------------------------------------------------------------------------------- /integrationtests/testground/README.md: -------------------------------------------------------------------------------- 1 | Here contains test plans to be run on [Testground](https://docs.testground.ai/). 2 | 3 | # Setup 4 | 5 | ``` 6 | git clone https://github.com/testground/testground.git 7 | cd testground 8 | # compile Testground and all related dependencies 9 | make install 10 | # start the Testground daemon, listening by default on localhost:8042 11 | testground daemon 12 | ``` 13 | 14 | Run below from another terminal window. This will add tests here to the testground home directory, which is `$HOME/testground` by default, unless you explicitly define `$TESTGROUND_HOME`. 15 | ``` 16 | testground plan import --from . --name go-threads 17 | ``` 18 | 19 | # Run 20 | 21 | Then you can run the prepared composition plans as follows: 22 | ``` 23 | # This one sticks to an early version of go-threads 24 | testground run composition --wait -f baseline-docker.toml 25 | # This one always test against the current head of go-threads 26 | testground run composition --wait -f head-docker.toml 27 | # This one uses whatever version specified by go.mod in the current directory. Do check/update dependencies before running this. 28 | testground run composition --wait -f current-docker.toml 29 | # This one is good for local test while developing. 30 | testground run composition --wait -f current-exec.toml 31 | ``` 32 | 33 | You can also run individual test case and pass test parameters in command line. For example, below builds the test as native executables, runs 2 instances, with very verbose logs printed to the console. See manifest.toml for all test parameters. 34 | ``` 35 | testground run single --wait -p go-threads -t sync-threads -b exec:go -r local:exec -i 2 -tp verbose=2 36 | ``` 37 | 38 | # Analysize 39 | 40 | 1. Check for errors running the tests 41 | 1. Check the recorded metrics and compare them with a baseline run. `run_id` is shown at both the begining and the end of the run. 42 | ``` 43 | cd $HOME/testground/data/outputs/local_docker/go-threads/ 44 | find . -name "results.out" | xargs grep elapsed-seconds | sort -t ':' -k 4 45 | ``` 46 | 47 | 48 | # Develop 49 | 50 | To run these tests after making some changes to go-threads, do the following: 51 | 52 | 1. Commit you changes and push to a branch 53 | 1. `GOPRIVATE=* go get github.com/textileio/go-threads@` 54 | 1. `testground run composition --wait -f current-docker.toml` and have a look at the results (See [section Analysize](#analysize)) 55 | 1. Commit changes to the branch 56 | -------------------------------------------------------------------------------- /integrationtests/testground/baseline-docker.toml: -------------------------------------------------------------------------------- 1 | [global] 2 | plan = "go-threads" 3 | case = "sync-threads" 4 | builder = "docker:go" 5 | runner = "local:docker" 6 | total_instances = 5 7 | [global.build] 8 | dependencies = [ 9 | { module = "github.com/textileio/go-threads", version = "4c90b2d32403eac31871637dc0922ef5ee6371a9"}, 10 | ] 11 | 12 | [[groups]] 13 | id = "always-on" 14 | instances = { count = 3 } 15 | 16 | [[groups]] 17 | id = "early-stopper" 18 | instances = { count = 1 } 19 | [groups.run] 20 | test_params = {early-stop="true"} 21 | 22 | [[groups]] 23 | id = "late-joiner" 24 | instances = { count = 1 } 25 | [groups.run] 26 | test_params = {late-start="true"} 27 | -------------------------------------------------------------------------------- /integrationtests/testground/current-docker.toml: -------------------------------------------------------------------------------- 1 | [global] 2 | plan = "go-threads" 3 | case = "sync-threads" 4 | builder = "docker:go" 5 | runner = "local:docker" 6 | total_instances = 5 7 | [global.run] 8 | test_params = {records="10", verbose="1", test-timeout="2m"} 9 | 10 | [[groups]] 11 | id = "always-on" 12 | instances = { count = 3 } 13 | 14 | [[groups]] 15 | id = "early-stopper" 16 | instances = { count = 1 } 17 | [groups.run] 18 | test_params = {early-stop="true"} 19 | 20 | [[groups]] 21 | id = "late-joiner" 22 | instances = { count = 1 } 23 | [groups.run] 24 | test_params = {late-start="true"} 25 | -------------------------------------------------------------------------------- /integrationtests/testground/current-exec-bitswap-sync-race.toml: -------------------------------------------------------------------------------- 1 | [global] 2 | plan = "go-threads" 3 | case = "bitswap-sync-race" 4 | builder = "exec:go" 5 | runner = "local:exec" 6 | total_instances = 2 7 | [global.run] 8 | test_params = {records="200", verbose="2"} 9 | 10 | [[groups]] 11 | id = "first-instance" 12 | instances = { count = 1 } 13 | [groups.run] 14 | test_params = {first="true"} 15 | 16 | [[groups]] 17 | id = "second-instance" 18 | instances = { count = 1 } 19 | [groups.run] 20 | test_params = {second="true"} 21 | -------------------------------------------------------------------------------- /integrationtests/testground/current-exec-sync-threads.toml: -------------------------------------------------------------------------------- 1 | [global] 2 | plan = "go-threads" 3 | case = "sync-threads" 4 | builder = "exec:go" 5 | runner = "local:exec" 6 | total_instances = 3 7 | [global.run] 8 | test_params = {records="3", verbose="2"} 9 | 10 | [[groups]] 11 | id = "always-on" 12 | instances = { count = 1 } 13 | 14 | [[groups]] 15 | id = "early-stopper" 16 | instances = { count = 1 } 17 | [groups.run] 18 | test_params = {early-stop="true"} 19 | 20 | [[groups]] 21 | id = "late-joiner" 22 | instances = { count = 1 } 23 | [groups.run] 24 | test_params = {late-start="true"} 25 | -------------------------------------------------------------------------------- /integrationtests/testground/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/textileio/go-threads/integrationtests/testground 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/go-kit/kit v0.10.0 7 | github.com/google/gofuzz v1.0.0 8 | github.com/guptarohit/asciigraph v0.5.2 9 | github.com/ipfs/go-cid v0.0.7 10 | github.com/ipfs/go-ipld-cbor v0.0.5 11 | github.com/ipfs/go-log/v2 v2.3.0 12 | github.com/libp2p/go-libp2p-core v0.8.6 13 | github.com/multiformats/go-multiaddr v0.3.3 14 | github.com/multiformats/go-multihash v0.0.15 15 | github.com/prometheus/client_golang v1.11.0 16 | github.com/prometheus/client_model v0.2.0 17 | github.com/prometheus/node_exporter v1.1.2 18 | github.com/testground/sdk-go v0.2.7 19 | github.com/textileio/go-threads v1.1.0-rc1.0.20210821175230-b925bde5b101 20 | google.golang.org/grpc v1.39.0 21 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 22 | ) 23 | -------------------------------------------------------------------------------- /integrationtests/testground/head-docker.toml: -------------------------------------------------------------------------------- 1 | [global] 2 | plan = "go-threads" 3 | case = "sync-threads" 4 | builder = "docker:go" 5 | runner = "local:docker" 6 | total_instances = 5 7 | [global.build] 8 | dependencies = [ 9 | { module = "github.com/textileio/go-threads", version = "master"}, 10 | ] 11 | 12 | [[groups]] 13 | id = "always-on" 14 | instances = { count = 3 } 15 | 16 | [[groups]] 17 | id = "early-stopper" 18 | instances = { count = 1 } 19 | [groups.run] 20 | test_params = {early-stop="true"} 21 | 22 | [[groups]] 23 | id = "late-joiner" 24 | instances = { count = 1 } 25 | [groups.run] 26 | test_params = {late-start="true"} 27 | -------------------------------------------------------------------------------- /integrationtests/testground/manifest.toml: -------------------------------------------------------------------------------- 1 | name = "go-threads" 2 | 3 | [defaults] 4 | builder = "exec:go" 5 | runner = "local:exec" 6 | 7 | [builders] 8 | "docker:go" = {enabled = true, enable_go_build_cache = true} 9 | "exec:go" = { enabled = true } 10 | 11 | [runners] 12 | "local:docker" = { enabled = true } 13 | "local:exec" = { enabled = true } 14 | "cluster:k8s" = { enabled = true } 15 | 16 | [global.build_config] 17 | enable_go_build_cache = true 18 | 19 | [[testcases]] 20 | name = "sync-threads" 21 | instances = { min = 1, max = 500, default = 1 } 22 | [testcases.params] 23 | records = { type = "int", desc = "number of random records to be created for each thread", default = 10 } 24 | verbose = { type = "int", desc = "verbose level of on screen logs, 0-3", default = 0 } 25 | test-timeout = { type = "string", desc = "how long each test is allowed to run", default = "1m" } 26 | late-start = { type = "boolean", desc = "start client in a later stage, used by composition", default = false } 27 | early-stop = { type = "boolean", desc = "stop client in an early stage, used by composition", default = false } 28 | 29 | [[testcases]] 30 | name = "bitswap-sync-race" 31 | instances = { min = 1, max = 500, default = 1 } 32 | [testcases.params] 33 | records = { type = "int", desc = "number of random records to be created for each thread", default = 10 } 34 | verbose = { type = "int", desc = "verbose level of on screen logs, 0-3", default = 0 } 35 | test-timeout = { type = "string", desc = "how long each test is allowed to run", default = "1m" } 36 | first = { type = "boolean", desc = "start client in a later stage, used by composition", default = false } 37 | second = { type = "boolean", desc = "stop client in an early stage, used by composition", default = false } 38 | -------------------------------------------------------------------------------- /integrationtests/testground/measurement.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | goruntime "runtime" 6 | "time" 7 | 8 | "github.com/go-kit/kit/log" 9 | "github.com/guptarohit/asciigraph" 10 | "github.com/prometheus/client_golang/prometheus" 11 | dto "github.com/prometheus/client_model/go" 12 | "github.com/prometheus/node_exporter/collector" 13 | "github.com/testground/sdk-go/runtime" 14 | kingpin "gopkg.in/alecthomas/kingpin.v2" 15 | ) 16 | 17 | type measure struct { 18 | runenv *runtime.RunEnv 19 | registry *prometheus.Registry 20 | chStop chan struct{} 21 | metrics map[string][]float64 22 | lastRecv float64 23 | lastTransmit float64 24 | } 25 | 26 | const ( 27 | metricGoroutines = "goroutines" 28 | metricHeapAllocMiBs = "heap-alloc-mibs" 29 | metricRecvBytes = "receive-bytes" 30 | metricTransmitBytes = "transmit-bytes" 31 | ) 32 | 33 | // startMeasure starts collecting number of goroutines, golang heap allocation and transmit/receive bytes every second, until stopAndPrint is called on the returned measure, at which point it sends all the recorded metrics as test result to InfluxDB, and prints them as line graphs for inspection. 34 | func startMeasure(runenv *runtime.RunEnv) (*measure, error) { 35 | // have to do this because node_exporter requires it being called to properly initialize global variables. 36 | kingpin.Parse() 37 | logger := log.NewNopLogger() 38 | registry := prometheus.NewRegistry() 39 | collector.DisableDefaultCollectors() 40 | nodeCollector, err := collector.NewNodeCollector(logger) 41 | if err != nil { 42 | return nil, err 43 | } 44 | netCollector, err := collector.NewNetDevCollector(logger) 45 | if err != nil { 46 | return nil, err 47 | } 48 | nodeCollector.Collectors["net"] = netCollector 49 | registry.MustRegister(nodeCollector) 50 | p := &measure{runenv: runenv, registry: registry, 51 | chStop: make(chan struct{}), 52 | metrics: make(map[string][]float64), 53 | } 54 | go func() { 55 | p.Collect() 56 | }() 57 | 58 | return p, nil 59 | } 60 | 61 | func (p *measure) Collect() { 62 | tk := time.NewTicker(time.Second) 63 | for { 64 | select { 65 | case <-tk.C: 66 | mf, err := p.registry.Gather() 67 | if err != nil { 68 | panic(err) 69 | } 70 | for _, m := range mf { 71 | switch *m.Name { 72 | case "node_network_receive_bytes_total": 73 | p.collectRecv(m.Metric) 74 | case "node_network_transmit_bytes_total": 75 | p.collectTransmit(m.Metric) 76 | } 77 | } 78 | p.collectGoroutines() 79 | p.collectMemStats() 80 | case <-p.chStop: 81 | return 82 | } 83 | } 84 | } 85 | 86 | func (p *measure) collectGoroutines() { 87 | p.metrics[metricGoroutines] = append(p.metrics[metricGoroutines], float64(goruntime.NumGoroutine())) 88 | } 89 | 90 | func (p *measure) collectMemStats() { 91 | var m goruntime.MemStats 92 | goruntime.ReadMemStats(&m) 93 | p.metrics[metricHeapAllocMiBs] = append(p.metrics[metricHeapAllocMiBs], float64(m.HeapAlloc)/1048576.0) 94 | } 95 | 96 | func (p *measure) collectRecv(metrics []*dto.Metric) { 97 | total := p.collectBytes(metrics) 98 | usage := total - p.lastRecv 99 | if p.lastRecv > 0 { 100 | p.metrics[metricRecvBytes] = append(p.metrics[metricRecvBytes], usage) 101 | p.runenv.D().Gauge(metricRecvBytes).Update(usage) 102 | } 103 | p.lastRecv = total 104 | } 105 | 106 | func (p *measure) collectTransmit(metrics []*dto.Metric) { 107 | total := p.collectBytes(metrics) 108 | usage := total - p.lastTransmit 109 | if p.lastTransmit > 0 { 110 | p.metrics[metricTransmitBytes] = append(p.metrics[metricTransmitBytes], usage) 111 | p.runenv.D().Gauge(metricTransmitBytes).Update(usage) 112 | } 113 | p.lastTransmit = total 114 | } 115 | 116 | func (p *measure) collectBytes(metrics []*dto.Metric) float64 { 117 | var total, exclude float64 118 | for _, m := range metrics { 119 | total += *m.Counter.Value 120 | for _, label := range m.Label { 121 | if *label.Name == "device" && *label.Value == "lo0" { 122 | exclude += *m.Counter.Value 123 | } 124 | } 125 | } 126 | return total - exclude 127 | } 128 | 129 | func (p *measure) stopAndPrint() { 130 | close(p.chStop) 131 | output := fmt.Sprintf("Test params: %v", p.runenv.TestInstanceParams) 132 | for _, name := range []string{ 133 | metricGoroutines, 134 | metricHeapAllocMiBs, 135 | metricRecvBytes, 136 | metricTransmitBytes, 137 | } { 138 | if len(p.metrics[name]) == 0 { 139 | p.runenv.RecordMessage("WARNING: No metrics for %s!", name) 140 | continue 141 | } 142 | output += "\n" 143 | output += asciigraph.Plot( 144 | p.metrics[name], 145 | asciigraph.Caption(name), 146 | asciigraph.Width(100), 147 | asciigraph.Height(10), 148 | asciigraph.Offset(10)) 149 | } 150 | p.runenv.RecordMessage(output) 151 | } 152 | -------------------------------------------------------------------------------- /jsonpatcher/jsonpatcher_test.go: -------------------------------------------------------------------------------- 1 | package jsonpatcher 2 | 3 | import ( 4 | "bytes" 5 | "encoding/gob" 6 | "testing" 7 | "time" 8 | 9 | cbornode "github.com/ipfs/go-ipld-cbor" 10 | mh "github.com/multiformats/go-multihash" 11 | core "github.com/textileio/go-threads/core/db" 12 | ) 13 | 14 | type patchEventOld struct { 15 | Timestamp time.Time 16 | ID core.InstanceID 17 | CollectionName string 18 | Patch operation 19 | } 20 | 21 | func init() { 22 | cbornode.RegisterCborType(patchEventOld{}) 23 | cbornode.RegisterCborType(time.Time{}) 24 | gob.Register(map[string]interface {}{}) 25 | } 26 | 27 | func TestJsonPatcher_Migration(t *testing.T) { 28 | in1 := patchEventOld{Timestamp: time.Now(), ID: "123", CollectionName: "abc"} 29 | node1, err := cbornode.WrapObject(in1, mh.SHA2_256, -1) 30 | if err != nil { 31 | t.Fatal(err) 32 | } 33 | b := bytes.Buffer{} 34 | e := gob.NewEncoder(&b) 35 | if err := e.Encode(in1); err != nil { 36 | t.Errorf("failed to gob encode event with time.Time timestamp: %v", err) 37 | } 38 | 39 | out := new(patchEvent) 40 | err = cbornode.DecodeInto(node1.RawData(), &out) 41 | if err != nil { 42 | t.Errorf("failed to decode event with time.Time timestamp: %v", err) 43 | } 44 | if !out.time().Equal(time.Time{}) { 45 | t.Error("non-encodable time should have zero-value") 46 | } 47 | b.Reset() 48 | e = gob.NewEncoder(&b) 49 | if err := e.Encode(out); err != nil { 50 | t.Errorf("failed to gob encode event with empty time.Time timestamp: %v", err) 51 | } 52 | 53 | ts := time.Now() 54 | in2 := patchEvent{Timestamp: ts.UnixNano(), ID: "123", CollectionName: "abc"} 55 | node2, err := cbornode.WrapObject(in2, mh.SHA2_256, -1) 56 | if err != nil { 57 | t.Fatal(err) 58 | } 59 | 60 | b.Reset() 61 | e = gob.NewEncoder(&b) 62 | if err := e.Encode(in2); err != nil { 63 | t.Errorf("failed to gob encode event with int64 timestamp: %v", err) 64 | } 65 | 66 | err = cbornode.DecodeInto(node2.RawData(), &out) 67 | if err != nil { 68 | t.Errorf("failed to decode event with int64 timestamp: %v", err) 69 | } 70 | tsout := out.time() 71 | 72 | b.Reset() 73 | e = gob.NewEncoder(&b) 74 | if err := e.Encode(out); err != nil { 75 | t.Errorf("failed to gob encode event with int64 timestamp: %v", err) 76 | } 77 | 78 | if !tsout.Equal(ts) { 79 | t.Error("encodable time should be equal to input") 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /jwt/jwt.go: -------------------------------------------------------------------------------- 1 | package jwted25519 2 | 3 | import ( 4 | "github.com/golang-jwt/jwt" 5 | "github.com/libp2p/go-libp2p-core/crypto" 6 | ) 7 | 8 | // Implements the Ed25519 signing method. 9 | // Expects *crypto.Ed25519PublicKey for signing and *crypto.Ed25519PublicKey for validation. 10 | type SigningMethodEd25519 struct { 11 | Name string 12 | } 13 | 14 | // Specific instance for Ed25519. 15 | var SigningMethodEd25519i *SigningMethodEd25519 16 | 17 | func init() { 18 | SigningMethodEd25519i = &SigningMethodEd25519{"Ed25519"} 19 | jwt.RegisterSigningMethod(SigningMethodEd25519i.Alg(), func() jwt.SigningMethod { 20 | return SigningMethodEd25519i 21 | }) 22 | } 23 | 24 | // Alg returns the name of this signing method. 25 | func (m *SigningMethodEd25519) Alg() string { 26 | return m.Name 27 | } 28 | 29 | // Implements the Verify method from SigningMethod. 30 | // For this signing method, must be a *crypto.Ed25519PublicKey structure. 31 | func (m *SigningMethodEd25519) Verify(signingString, signature string, key interface{}) error { 32 | var err error 33 | 34 | // Decode the signature 35 | var sig []byte 36 | if sig, err = jwt.DecodeSegment(signature); err != nil { 37 | return err 38 | } 39 | 40 | var ed25519Key *crypto.Ed25519PublicKey 41 | var ok bool 42 | 43 | if ed25519Key, ok = key.(*crypto.Ed25519PublicKey); !ok { 44 | return jwt.ErrInvalidKeyType 45 | } 46 | 47 | // verify the signature 48 | valid, err := ed25519Key.Verify([]byte(signingString), sig) 49 | if err != nil { 50 | return err 51 | } 52 | if !valid { 53 | return jwt.ErrSignatureInvalid 54 | } 55 | 56 | return nil 57 | } 58 | 59 | // Implements the Sign method from SigningMethod. 60 | // For this signing method, must be a *crypto.Ed25519PublicKey structure. 61 | func (m *SigningMethodEd25519) Sign(signingString string, key interface{}) (string, error) { 62 | var ed25519Key *crypto.Ed25519PrivateKey 63 | var ok bool 64 | 65 | // validate type of key 66 | if ed25519Key, ok = key.(*crypto.Ed25519PrivateKey); !ok { 67 | return "", jwt.ErrInvalidKey 68 | } 69 | 70 | sigBytes, err := ed25519Key.Sign([]byte(signingString)) 71 | if err != nil { 72 | return "", err 73 | } 74 | return jwt.EncodeSegment(sigBytes), nil 75 | } 76 | -------------------------------------------------------------------------------- /jwt/jwt_test.go: -------------------------------------------------------------------------------- 1 | package jwted25519_test 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/golang-jwt/jwt" 8 | "github.com/libp2p/go-libp2p-core/crypto" 9 | . "github.com/textileio/go-threads/jwt" 10 | ) 11 | 12 | var publicKey = "CAESIP1G8uGFpX+iduqgJfKLt0nw870MI9ydHcKg9gDIr5Tb" 13 | var privateKey = "CAESYKcFG4UOHb1fyF+GlyGWfjfX47DH3y/K9fYMMMdy3Ow2/Uby4YWlf6J26qAl8ou3SfDzvQwj3J0dwqD2AMivlNv9RvLhhaV/onbqoCXyi7dJ8PO9DCPcnR3CoPYAyK+U2w==" 14 | 15 | var ed25519TestData = []struct { 16 | name string 17 | tokenString string 18 | alg string 19 | claims map[string]interface{} 20 | valid bool 21 | }{ 22 | { 23 | "Ed25519", 24 | "eyJhbGciOiJFZDI1NTE5IiwidHlwIjoiSldUIn0.eyJqdGkiOiJmb28iLCJzdWIiOiJiYXIifQ.E73qcBjcCSsYto_Pa5CpwZUu9lA3ecCVkZ8pJiFYNaOe2x-uZCDmZnx52AByO78oxft09GosVcJtqYNv1VBxDQ", 25 | "Ed25519", 26 | map[string]interface{}{"jti": "foo", "sub": "bar"}, 27 | true, 28 | }, 29 | { 30 | "invalid key", 31 | "eyJhbGciOiJFZDI1NTE5IiwidHlwIjoiSldUIn0.eyJqdGkiOiJmb28iLCJzdWIiOiJiYXIifQ.7FSQFedbbRl42nvUWJqBswvjmyMaBBLKk0opiARjxtZmQ86dVMYs5wcZ0gItVV8YLVu6F5065IFD699tVcacBA", 32 | "Ed25519", 33 | map[string]interface{}{"jti": "foo", "sub": "bar"}, 34 | false, 35 | }, 36 | } 37 | 38 | func TestSigningMethodEd25519_Alg(t *testing.T) { 39 | if SigningMethodEd25519i.Alg() != "Ed25519" { 40 | t.Fatal("wrong alg") 41 | } 42 | } 43 | 44 | func TestSigningMethodEd25519_Sign(t *testing.T) { 45 | sk, err := unmarshalPrivateKeyFromString(privateKey) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | for _, data := range ed25519TestData { 50 | if data.valid { 51 | parts := strings.Split(data.tokenString, ".") 52 | method := jwt.GetSigningMethod(data.alg) 53 | sig, err := method.Sign(strings.Join(parts[0:2], "."), sk) 54 | if err != nil { 55 | t.Errorf("[%v] error signing token: %v", data.name, err) 56 | } 57 | if sig != parts[2] { 58 | t.Errorf("[%v] incorrect signature.\nwas:\n%v\nexpecting:\n%v", data.name, sig, parts[2]) 59 | } 60 | } 61 | } 62 | } 63 | 64 | func TestSigningMethodEd25519_Verify(t *testing.T) { 65 | pk, err := unmarshalPublicKeyFromString(publicKey) 66 | if err != nil { 67 | t.Fatal(err) 68 | } 69 | for _, data := range ed25519TestData { 70 | parts := strings.Split(data.tokenString, ".") 71 | 72 | method := jwt.GetSigningMethod(data.alg) 73 | err := method.Verify(strings.Join(parts[0:2], "."), parts[2], pk) 74 | if data.valid && err != nil { 75 | t.Errorf("[%v] error while verifying key: %v", data.name, err) 76 | } 77 | if !data.valid && err == nil { 78 | t.Errorf("[%v] invalid key passed validation", data.name) 79 | } 80 | } 81 | } 82 | 83 | func TestGenerateEd25519Token(t *testing.T) { 84 | sk, err := unmarshalPrivateKeyFromString(privateKey) 85 | if err != nil { 86 | t.Fatal(err) 87 | } 88 | claims := &jwt.StandardClaims{ 89 | Id: "bar", 90 | Subject: "foo", 91 | } 92 | _, err = jwt.NewWithClaims(SigningMethodEd25519i, claims).SignedString(sk) 93 | if err != nil { 94 | t.Fatal(err) 95 | } 96 | } 97 | 98 | func unmarshalPrivateKeyFromString(key string) (crypto.PrivKey, error) { 99 | keyb, err := crypto.ConfigDecodeKey(key) 100 | if err != nil { 101 | return nil, err 102 | } 103 | return crypto.UnmarshalPrivateKey(keyb) 104 | } 105 | 106 | func unmarshalPublicKeyFromString(key string) (crypto.PubKey, error) { 107 | keyb, err := crypto.ConfigDecodeKey(key) 108 | if err != nil { 109 | return nil, err 110 | } 111 | return crypto.UnmarshalPublicKey(keyb) 112 | } 113 | -------------------------------------------------------------------------------- /logstore/lstoreds/README.md: -------------------------------------------------------------------------------- 1 | # Datastore backed Logstore 2 | 3 | > Work pretty much inspired by `go-libp2p-peerstore` great implementation. 4 | 5 | This is a `go-datastore` backed implementation of [`go-threads/core/logstore`](https://github.com/textileio/go-threads/blob/master/core/logstore/logstore.go): 6 | * AddrBook 7 | * HeadBook 8 | * KeyBook 9 | 10 | For testing, two `go-datastore` implementation are table-tested: 11 | * [badger](github.com/ipfs/go-ds-badger) 12 | * [leveldb](github.com/ipfs/go-ds-leveldb) 13 | 14 | Tests leverage the `/test` suites used also for in-memory implementation. -------------------------------------------------------------------------------- /logstore/lstoreds/addr_book_gc.go: -------------------------------------------------------------------------------- 1 | package lstoreds 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | query "github.com/ipfs/go-datastore/query" 9 | pb "github.com/textileio/go-threads/net/pb" 10 | ) 11 | 12 | var ( 13 | purgeStoreQuery = query.Query{ 14 | Prefix: logBookBase.String(), 15 | Orders: []query.Order{query.OrderByKey{}}, 16 | KeysOnly: false, 17 | } 18 | ) 19 | 20 | // dsAddrBookGc is responsible for garbage collection in a datastore-backed address book. 21 | type dsAddrBookGc struct { 22 | ctx context.Context 23 | ab *DsAddrBook 24 | running chan struct{} 25 | purgeFunc func() 26 | } 27 | 28 | func newAddressBookGc(ctx context.Context, ab *DsAddrBook) (*dsAddrBookGc, error) { 29 | if ab.opts.GCPurgeInterval < 0 { 30 | return nil, fmt.Errorf("negative GC purge interval provided: %s", ab.opts.GCPurgeInterval) 31 | } 32 | 33 | gc := &dsAddrBookGc{ 34 | ctx: ctx, 35 | ab: ab, 36 | running: make(chan struct{}, 1), 37 | } 38 | 39 | gc.purgeFunc = gc.purgeStore 40 | 41 | // do not start GC timers if purge is disabled; this GC can only be triggered manually. 42 | if ab.opts.GCPurgeInterval > 0 { 43 | gc.ab.childrenDone.Add(1) 44 | go gc.background() 45 | } 46 | 47 | return gc, nil 48 | } 49 | 50 | // gc prunes expired addresses from the datastore at regular intervals. It should be spawned as a goroutine. 51 | func (gc *dsAddrBookGc) background() { 52 | defer gc.ab.childrenDone.Done() 53 | 54 | select { 55 | case <-time.After(gc.ab.opts.GCInitialDelay): 56 | case <-gc.ab.ctx.Done(): 57 | // yield if we have been cancelled/closed before the delay elapses. 58 | return 59 | } 60 | 61 | purgeTimer := time.NewTicker(gc.ab.opts.GCPurgeInterval) 62 | defer purgeTimer.Stop() 63 | 64 | for { 65 | select { 66 | case <-purgeTimer.C: 67 | gc.purgeFunc() 68 | case <-gc.ctx.Done(): 69 | return 70 | } 71 | } 72 | } 73 | 74 | func (gc *dsAddrBookGc) purgeStore() { 75 | select { 76 | case gc.running <- struct{}{}: 77 | defer func() { <-gc.running }() 78 | default: 79 | // yield if lookahead is running. 80 | return 81 | } 82 | 83 | batch, err := newCyclicBatch(gc.ab.ds, defaultOpsPerCyclicBatch) 84 | if err != nil { 85 | log.Warnf("failed while creating batch to purge GC entries: %v", err) 86 | } 87 | 88 | results, err := gc.ab.ds.Query(purgeStoreQuery) 89 | if err != nil { 90 | log.Warnf("failed while opening iterator: %v", err) 91 | return 92 | } 93 | defer results.Close() 94 | 95 | record := &addrsRecord{AddrBookRecord: &pb.AddrBookRecord{}} // empty record to reuse and avoid allocs. 96 | // keys: /thread/addrs/ 97 | for result := range results.Next() { 98 | record.Reset() 99 | if err = record.Unmarshal(result.Value); err != nil { 100 | log.Warnf("key %v has an unmarshable record", result.Key) 101 | continue 102 | } 103 | 104 | if !record.clean() { 105 | continue 106 | } 107 | 108 | id := genCacheKey(record.ThreadID.ID, record.PeerID.ID) 109 | if err := record.flush(batch); err != nil { 110 | log.Warnf("failed to flush entry modified by GC for peer: &v, err: %v", id, err) 111 | } 112 | gc.ab.cache.Remove(id) 113 | } 114 | 115 | if err = batch.Commit(); err != nil { 116 | log.Warnf("failed to commit GC purge batch: %v", err) 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /logstore/lstoreds/cache.go: -------------------------------------------------------------------------------- 1 | package lstoreds 2 | 3 | // cache abstracts all methods we access from ARCCache, to enable alternate 4 | // implementations such as a no-op one. 5 | type cache interface { 6 | Get(key interface{}) (value interface{}, ok bool) 7 | Add(key, value interface{}) 8 | Remove(key interface{}) 9 | Contains(key interface{}) bool 10 | Peek(key interface{}) (value interface{}, ok bool) 11 | Keys() []interface{} 12 | } 13 | 14 | // noopCache is a dummy implementation that's used when the cache is disabled. 15 | type noopCache struct { 16 | } 17 | 18 | var _ cache = (*noopCache)(nil) 19 | 20 | func (*noopCache) Get(_ interface{}) (value interface{}, ok bool) { 21 | return nil, false 22 | } 23 | 24 | func (*noopCache) Add(_, _ interface{}) { 25 | } 26 | 27 | func (*noopCache) Remove(_ interface{}) { 28 | } 29 | 30 | func (*noopCache) Contains(_ interface{}) bool { 31 | return false 32 | } 33 | 34 | func (*noopCache) Peek(_ interface{}) (value interface{}, ok bool) { 35 | return nil, false 36 | } 37 | 38 | func (*noopCache) Keys() (keys []interface{}) { 39 | return keys 40 | } 41 | -------------------------------------------------------------------------------- /logstore/lstoreds/cyclic_batch.go: -------------------------------------------------------------------------------- 1 | package lstoreds 2 | 3 | import ( 4 | "fmt" 5 | 6 | ds "github.com/ipfs/go-datastore" 7 | ) 8 | 9 | // how many operations are queued in a cyclic batch before we flush it. 10 | var defaultOpsPerCyclicBatch = 20 11 | 12 | // cyclicBatch buffers ds write operations and automatically flushes them after defaultOpsPerCyclicBatch (20) have been 13 | // queued. An explicit `Commit()` closes this cyclic batch, erroring all further operations. 14 | // 15 | // It is similar to go-ds autobatch, but it's driven by an actual Batch facility offered by the 16 | // ds. 17 | type cyclicBatch struct { 18 | threshold int 19 | ds.Batch 20 | ds ds.Batching 21 | pending int 22 | } 23 | 24 | func newCyclicBatch(ds ds.Batching, _ int) (ds.Batch, error) { 25 | batch, err := ds.Batch() 26 | if err != nil { 27 | return nil, err 28 | } 29 | return &cyclicBatch{Batch: batch, ds: ds}, nil 30 | } 31 | 32 | func (cb *cyclicBatch) cycle() (err error) { 33 | if cb.Batch == nil { 34 | return fmt.Errorf("cyclic batch is closed") 35 | } 36 | if cb.pending < cb.threshold { 37 | // we haven't reached the threshold yet. 38 | return nil 39 | } 40 | // commit and renew the batch. 41 | if err = cb.Batch.Commit(); err != nil { 42 | return fmt.Errorf("failed while committing cyclic batch: %s", err) 43 | } 44 | if cb.Batch, err = cb.ds.Batch(); err != nil { 45 | return fmt.Errorf("failed while renewing cyclic batch: %s", err) 46 | } 47 | return nil 48 | } 49 | 50 | func (cb *cyclicBatch) Put(key ds.Key, val []byte) error { 51 | if err := cb.cycle(); err != nil { 52 | return err 53 | } 54 | cb.pending++ 55 | return cb.Batch.Put(key, val) 56 | } 57 | 58 | func (cb *cyclicBatch) Delete(key ds.Key) error { 59 | if err := cb.cycle(); err != nil { 60 | return err 61 | } 62 | cb.pending++ 63 | return cb.Batch.Delete(key) 64 | } 65 | 66 | func (cb *cyclicBatch) Commit() error { 67 | if cb.Batch == nil { 68 | return fmt.Errorf("cyclic batch is closed") 69 | } 70 | if err := cb.Batch.Commit(); err != nil { 71 | return err 72 | } 73 | cb.pending = 0 74 | cb.Batch = nil 75 | return nil 76 | } 77 | -------------------------------------------------------------------------------- /logstore/lstoreds/ds_test.go: -------------------------------------------------------------------------------- 1 | package lstoreds 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | "time" 9 | 10 | ds "github.com/ipfs/go-datastore" 11 | badger "github.com/textileio/go-ds-badger" 12 | core "github.com/textileio/go-threads/core/logstore" 13 | pt "github.com/textileio/go-threads/test" 14 | ) 15 | 16 | type datastoreFactory func(tb testing.TB) (ds.Batching, func()) 17 | 18 | var dstores = map[string]datastoreFactory{ 19 | "Badger": badgerStore, 20 | // "Leveldb": leveldbStore, 21 | } 22 | 23 | func TestDatastoreLogstore(t *testing.T) { 24 | for name, dsFactory := range dstores { 25 | t.Run(name, func(t *testing.T) { 26 | t.Parallel() 27 | pt.LogstoreTest(t, logstoreFactory(t, dsFactory, DefaultOpts())) 28 | }) 29 | } 30 | } 31 | 32 | func TestDatastoreAddrBook(t *testing.T) { 33 | for name, dsFactory := range dstores { 34 | t.Run(name+" Cacheful", func(t *testing.T) { 35 | t.Parallel() 36 | opts := DefaultOpts() 37 | opts.GCPurgeInterval = 1 * time.Second 38 | opts.CacheSize = 1024 39 | 40 | pt.AddrBookTest(t, addressBookFactory(t, dsFactory, opts)) 41 | }) 42 | 43 | t.Run(name+" Cacheless", func(t *testing.T) { 44 | t.Parallel() 45 | opts := DefaultOpts() 46 | opts.GCPurgeInterval = 1 * time.Second 47 | opts.CacheSize = 0 48 | 49 | pt.AddrBookTest(t, addressBookFactory(t, dsFactory, opts)) 50 | }) 51 | } 52 | } 53 | 54 | func TestDatastoreKeyBook(t *testing.T) { 55 | for name, dsFactory := range dstores { 56 | t.Run(name, func(t *testing.T) { 57 | t.Parallel() 58 | pt.KeyBookTest(t, keyBookFactory(t, dsFactory)) 59 | }) 60 | } 61 | } 62 | 63 | func TestDatastoreHeadBook(t *testing.T) { 64 | for name, dsFactory := range dstores { 65 | t.Run(name, func(t *testing.T) { 66 | t.Parallel() 67 | pt.HeadBookTest(t, headBookFactory(t, dsFactory)) 68 | }) 69 | } 70 | } 71 | 72 | func TestDatastoreMetadataBook(t *testing.T) { 73 | for name, dsFactory := range dstores { 74 | t.Run(name, func(t *testing.T) { 75 | t.Parallel() 76 | pt.MetadataBookTest(t, metadataBookFactory(t, dsFactory)) 77 | }) 78 | } 79 | } 80 | 81 | func logstoreFactory(tb testing.TB, storeFactory datastoreFactory, opts Options) pt.LogstoreFactory { 82 | return func() (core.Logstore, func()) { 83 | store, closeFunc := storeFactory(tb) 84 | ls, err := NewLogstore(context.Background(), store, opts) 85 | if err != nil { 86 | tb.Fatal(err) 87 | } 88 | closer := func() { 89 | _ = ls.Close() 90 | closeFunc() 91 | } 92 | return ls, closer 93 | } 94 | } 95 | 96 | func addressBookFactory(tb testing.TB, storeFactory datastoreFactory, opts Options) pt.AddrBookFactory { 97 | return func() (core.AddrBook, func()) { 98 | store, closeFunc := storeFactory(tb) 99 | ab, err := NewAddrBook(context.Background(), store, opts) 100 | if err != nil { 101 | tb.Fatal(err) 102 | } 103 | closer := func() { 104 | _ = ab.Close() 105 | closeFunc() 106 | } 107 | return ab, closer 108 | } 109 | } 110 | 111 | func keyBookFactory(tb testing.TB, storeFactory datastoreFactory) pt.KeyBookFactory { 112 | return func() (core.KeyBook, func()) { 113 | store, closeFunc := storeFactory(tb) 114 | kb, err := NewKeyBook(store) 115 | if err != nil { 116 | tb.Fatal(err) 117 | } 118 | closer := func() { 119 | closeFunc() 120 | } 121 | return kb, closer 122 | } 123 | } 124 | 125 | func headBookFactory(tb testing.TB, storeFactory datastoreFactory) pt.HeadBookFactory { 126 | return func() (core.HeadBook, func()) { 127 | store, closeFunc := storeFactory(tb) 128 | hb := NewHeadBook(store.(ds.TxnDatastore)) 129 | closer := func() { 130 | closeFunc() 131 | } 132 | return hb, closer 133 | } 134 | } 135 | 136 | func metadataBookFactory(tb testing.TB, storeFactory datastoreFactory) pt.MetadataBookFactory { 137 | return func() (core.ThreadMetadata, func()) { 138 | store, closeFunc := storeFactory(tb) 139 | tm := NewThreadMetadata(store) 140 | closer := func() { 141 | closeFunc() 142 | } 143 | return tm, closer 144 | } 145 | } 146 | 147 | func badgerStore(tb testing.TB) (ds.Batching, func()) { 148 | dataPath, err := ioutil.TempDir(os.TempDir(), "badger") 149 | if err != nil { 150 | tb.Fatal(err) 151 | } 152 | store, err := badger.NewDatastore(dataPath, nil) 153 | if err != nil { 154 | tb.Fatal(err) 155 | } 156 | closer := func() { 157 | _ = store.Close() 158 | _ = os.RemoveAll(dataPath) 159 | } 160 | return store, closer 161 | } 162 | 163 | // func leveldbStore(tb testing.TB) (ds.Datastore, func()) { 164 | // dataPath, err := ioutil.TempDir(os.TempDir(), "leveldb") 165 | // if err != nil { 166 | // tb.Fatal(err) 167 | // } 168 | // store, err := leveldb.NewDatastore(dataPath, nil) 169 | // if err != nil { 170 | // tb.Fatal(err) 171 | // } 172 | // closer := func() { 173 | // _ = store.Close() 174 | // _ = os.RemoveAll(dataPath) 175 | // } 176 | // return store, closer 177 | // } 178 | -------------------------------------------------------------------------------- /logstore/lstoreds/logstore.go: -------------------------------------------------------------------------------- 1 | package lstoreds 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | ds "github.com/ipfs/go-datastore" 8 | "github.com/ipfs/go-datastore/query" 9 | "github.com/libp2p/go-libp2p-core/peer" 10 | core "github.com/textileio/go-threads/core/logstore" 11 | "github.com/textileio/go-threads/core/thread" 12 | lstore "github.com/textileio/go-threads/logstore" 13 | "github.com/whyrusleeping/base32" 14 | ) 15 | 16 | // Define if storage will accept empty dumps. 17 | var AllowEmptyRestore = false 18 | 19 | const EmptyEdgeValue uint64 = 0 20 | 21 | // Configuration object for datastores 22 | type Options struct { 23 | // The size of the in-memory cache. A value of 0 or lower disables the cache. 24 | CacheSize uint 25 | 26 | // Sweep interval to purge expired addresses from the datastore. If this is a zero value, GC will not run 27 | // automatically, but it'll be available on demand via explicit calls. 28 | GCPurgeInterval time.Duration 29 | 30 | // Initial delay before GC processes start. Intended to give the system breathing room to fully boot 31 | // before starting GC. 32 | GCInitialDelay time.Duration 33 | } 34 | 35 | // DefaultOpts returns the default options for a persistent peerstore, with the full-purge GC algorithm: 36 | // 37 | // * Cache size: 1024. 38 | // * GC purge interval: 2 hours. 39 | // * GC initial delay: 60 seconds. 40 | func DefaultOpts() Options { 41 | return Options{ 42 | CacheSize: 1024, 43 | GCPurgeInterval: 2 * time.Hour, 44 | GCInitialDelay: 60 * time.Second, 45 | } 46 | } 47 | 48 | // NewLogstore creates a logstore backed by the provided persistent datastore. 49 | func NewLogstore(ctx context.Context, store ds.Batching, opts Options) (core.Logstore, error) { 50 | addrBook, err := NewAddrBook(ctx, store, opts) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | keyBook, err := NewKeyBook(store) 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | threadMetadata := NewThreadMetadata(store) 61 | 62 | headBook := NewHeadBook(store.(ds.TxnDatastore)) 63 | 64 | ps := lstore.NewLogstore(keyBook, addrBook, headBook, threadMetadata) 65 | return ps, nil 66 | } 67 | 68 | // uniqueThreadIds extracts and returns unique thread IDs from database keys. 69 | func uniqueThreadIds(ds ds.Datastore, prefix ds.Key, extractor func(result query.Result) string) (thread.IDSlice, error) { 70 | var ( 71 | q = query.Query{Prefix: prefix.String(), KeysOnly: true} 72 | results query.Results 73 | err error 74 | ) 75 | 76 | if results, err = ds.Query(q); err != nil { 77 | log.Error(err) 78 | return nil, err 79 | } 80 | 81 | defer results.Close() 82 | 83 | idset := make(map[string]struct{}) 84 | for result := range results.Next() { 85 | k := extractor(result) 86 | idset[k] = struct{}{} 87 | } 88 | 89 | if len(idset) == 0 { 90 | return thread.IDSlice{}, nil 91 | } 92 | 93 | ids := make(thread.IDSlice, 0, len(idset)) 94 | for id := range idset { 95 | id, err := parseThreadID(id) 96 | if err == nil { 97 | ids = append(ids, id) 98 | } 99 | } 100 | return ids, nil 101 | } 102 | 103 | // uniqueLogIds extracts and returns unique thread IDs from database keys. 104 | func uniqueLogIds(ds ds.Datastore, prefix ds.Key, extractor func(result query.Result) string) (peer.IDSlice, error) { 105 | var ( 106 | q = query.Query{Prefix: prefix.String(), KeysOnly: true} 107 | results query.Results 108 | err error 109 | ) 110 | 111 | if results, err = ds.Query(q); err != nil { 112 | log.Error(err) 113 | return nil, err 114 | } 115 | 116 | defer results.Close() 117 | 118 | idset := make(map[string]struct{}) 119 | for result := range results.Next() { 120 | k := extractor(result) 121 | idset[k] = struct{}{} 122 | } 123 | 124 | if len(idset) == 0 { 125 | return peer.IDSlice{}, nil 126 | } 127 | 128 | ids := make(peer.IDSlice, 0, len(idset)) 129 | for id := range idset { 130 | id, err := parseLogID(id) 131 | if err == nil { 132 | ids = append(ids, id) 133 | } 134 | } 135 | return ids, nil 136 | } 137 | 138 | func dsThreadKey(t thread.ID, baseKey ds.Key) ds.Key { 139 | key := baseKey.ChildString(base32.RawStdEncoding.EncodeToString(t.Bytes())) 140 | return key 141 | } 142 | 143 | func dsLogKey(t thread.ID, p peer.ID, baseKey ds.Key) ds.Key { 144 | key := baseKey.ChildString(base32.RawStdEncoding.EncodeToString(t.Bytes())) 145 | key = key.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))) 146 | return key 147 | } 148 | 149 | func parseThreadID(id string) (thread.ID, error) { 150 | pid, err := base32.RawStdEncoding.DecodeString(id) 151 | if err != nil { 152 | return thread.Undef, err 153 | } 154 | 155 | return thread.Cast(pid) 156 | } 157 | 158 | func parseLogID(id string) (peer.ID, error) { 159 | pid, err := base32.RawStdEncoding.DecodeString(id) 160 | if err != nil { 161 | return "", err 162 | } 163 | 164 | return peer.IDFromBytes(pid) 165 | } 166 | -------------------------------------------------------------------------------- /logstore/lstorehybrid/hybrid_test.go: -------------------------------------------------------------------------------- 1 | package lstorehybrid 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | badger "github.com/textileio/go-ds-badger" 10 | core "github.com/textileio/go-threads/core/logstore" 11 | "github.com/textileio/go-threads/logstore/lstoreds" 12 | m "github.com/textileio/go-threads/logstore/lstoremem" 13 | pt "github.com/textileio/go-threads/test" 14 | ) 15 | 16 | type storeFactory func(tb testing.TB) (core.Logstore, func()) 17 | 18 | var ( 19 | persist = map[string]storeFactory{ 20 | "lstoreds:Badger": lstoredsBadgerF, 21 | } 22 | 23 | inMem = map[string]storeFactory{ 24 | "lstoremem": lstorememF, 25 | } 26 | ) 27 | 28 | func TestHybridLogstore(t *testing.T) { 29 | for psName, psF := range persist { 30 | for msName, msF := range inMem { 31 | t.Run(psName+"+"+msName, func(t *testing.T) { 32 | t.Parallel() 33 | pt.LogstoreTest(t, logstoreFactory(t, psF, msF)) 34 | }) 35 | } 36 | } 37 | } 38 | 39 | func TestHybridAddrBook(t *testing.T) { 40 | for psName, psF := range persist { 41 | for msName, msF := range inMem { 42 | t.Run(psName+"+"+msName, func(t *testing.T) { 43 | t.Parallel() 44 | pt.AddrBookTest(t, adapterAddrBook(logstoreFactory(t, psF, msF))) 45 | }) 46 | } 47 | } 48 | } 49 | 50 | func TestHybridKeyBook(t *testing.T) { 51 | for psName, psF := range persist { 52 | for msName, msF := range inMem { 53 | t.Run(psName+"+"+msName, func(t *testing.T) { 54 | t.Parallel() 55 | pt.KeyBookTest(t, adapterKeyBook(logstoreFactory(t, psF, msF))) 56 | }) 57 | } 58 | } 59 | } 60 | 61 | func TestHybridHeadBook(t *testing.T) { 62 | for psName, psF := range persist { 63 | for msName, msF := range inMem { 64 | t.Run(psName+"+"+msName, func(t *testing.T) { 65 | t.Parallel() 66 | pt.HeadBookTest(t, adapterHeadBook(logstoreFactory(t, psF, msF))) 67 | }) 68 | } 69 | } 70 | } 71 | 72 | func TestHybridMetadataBook(t *testing.T) { 73 | for psName, psF := range persist { 74 | for msName, msF := range inMem { 75 | t.Run(psName+"+"+msName, func(t *testing.T) { 76 | t.Parallel() 77 | pt.MetadataBookTest(t, adapterMetaBook(logstoreFactory(t, psF, msF))) 78 | }) 79 | } 80 | } 81 | } 82 | 83 | /* store factories */ 84 | 85 | func logstoreFactory(tb testing.TB, persistF, memF storeFactory) pt.LogstoreFactory { 86 | return func() (core.Logstore, func()) { 87 | ps, psClose := persistF(tb) 88 | ms, msClose := memF(tb) 89 | 90 | ls, err := NewLogstore(ps, ms) 91 | if err != nil { 92 | tb.Fatal(err) 93 | } 94 | 95 | closer := func() { 96 | _ = ls.Close() 97 | psClose() 98 | msClose() 99 | } 100 | 101 | return ls, closer 102 | } 103 | } 104 | 105 | func lstoredsBadgerF(tb testing.TB) (core.Logstore, func()) { 106 | dataPath, err := ioutil.TempDir(os.TempDir(), "badger") 107 | if err != nil { 108 | tb.Fatal(err) 109 | } 110 | 111 | backend, err := badger.NewDatastore(dataPath, nil) 112 | if err != nil { 113 | tb.Fatal(err) 114 | } 115 | 116 | lstore, err := lstoreds.NewLogstore( 117 | context.Background(), 118 | backend, 119 | lstoreds.DefaultOpts(), 120 | ) 121 | 122 | closer := func() { 123 | _ = lstore.Close() 124 | _ = backend.Close() 125 | _ = os.RemoveAll(dataPath) 126 | } 127 | 128 | return lstore, closer 129 | } 130 | 131 | func lstorememF(_ testing.TB) (core.Logstore, func()) { 132 | store := m.NewLogstore() 133 | return store, func() { _ = store.Close() } 134 | } 135 | 136 | /* component adapters */ 137 | 138 | func adapterAddrBook(f pt.LogstoreFactory) pt.AddrBookFactory { 139 | return func() (core.AddrBook, func()) { return f() } 140 | } 141 | 142 | func adapterKeyBook(f pt.LogstoreFactory) pt.KeyBookFactory { 143 | return func() (core.KeyBook, func()) { return f() } 144 | } 145 | 146 | func adapterHeadBook(f pt.LogstoreFactory) pt.HeadBookFactory { 147 | return func() (core.HeadBook, func()) { return f() } 148 | } 149 | 150 | func adapterMetaBook(f pt.LogstoreFactory) pt.MetadataBookFactory { 151 | return func() (core.ThreadMetadata, func()) { return f() } 152 | } 153 | -------------------------------------------------------------------------------- /logstore/lstoremem/headbook.go: -------------------------------------------------------------------------------- 1 | package lstoremem 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/ipfs/go-cid" 7 | "github.com/libp2p/go-libp2p-core/peer" 8 | core "github.com/textileio/go-threads/core/logstore" 9 | "github.com/textileio/go-threads/core/thread" 10 | "github.com/textileio/go-threads/util" 11 | ) 12 | 13 | type memoryHeadBook struct { 14 | sync.RWMutex 15 | threads map[thread.ID]struct { 16 | heads map[peer.ID]map[cid.Cid]int64 17 | edge uint64 18 | } 19 | } 20 | 21 | func (mhb *memoryHeadBook) getHeads(t thread.ID, p peer.ID, createEmpty bool) map[cid.Cid]int64 { 22 | lmap := mhb.threads[t] 23 | if lmap.heads == nil { 24 | if !createEmpty { 25 | return nil 26 | } 27 | lmap.heads = make(map[peer.ID]map[cid.Cid]int64, 1) 28 | mhb.threads[t] = lmap 29 | } 30 | hmap := lmap.heads[p] 31 | if hmap == nil && createEmpty { 32 | hmap = make(map[cid.Cid]int64) 33 | lmap.heads[p] = hmap 34 | } 35 | return hmap 36 | } 37 | 38 | var _ core.HeadBook = (*memoryHeadBook)(nil) 39 | 40 | func NewHeadBook() core.HeadBook { 41 | return &memoryHeadBook{ 42 | threads: make(map[thread.ID]struct { 43 | heads map[peer.ID]map[cid.Cid]int64 44 | edge uint64 45 | }), 46 | } 47 | } 48 | 49 | func (mhb *memoryHeadBook) AddHead(t thread.ID, p peer.ID, head thread.Head) error { 50 | return mhb.AddHeads(t, p, []thread.Head{head}) 51 | } 52 | 53 | func (mhb *memoryHeadBook) AddHeads(t thread.ID, p peer.ID, heads []thread.Head) error { 54 | mhb.Lock() 55 | defer mhb.Unlock() 56 | defer mhb.updateEdge(t) 57 | 58 | hmap := mhb.getHeads(t, p, true) 59 | for _, h := range heads { 60 | if !h.ID.Defined() { 61 | log.Warnf("was passed nil head for %s", p) 62 | continue 63 | } 64 | hmap[h.ID] = h.Counter 65 | } 66 | return nil 67 | } 68 | 69 | func (mhb *memoryHeadBook) SetHead(t thread.ID, p peer.ID, head thread.Head) error { 70 | return mhb.SetHeads(t, p, []thread.Head{head}) 71 | } 72 | 73 | func (mhb *memoryHeadBook) SetHeads(t thread.ID, p peer.ID, heads []thread.Head) error { 74 | mhb.Lock() 75 | defer mhb.Unlock() 76 | defer mhb.updateEdge(t) 77 | 78 | var hset = make(map[cid.Cid]int64, len(heads)) 79 | for _, h := range heads { 80 | if !h.ID.Defined() { 81 | log.Warnf("was passed nil head for %s", p) 82 | continue 83 | } 84 | hset[h.ID] = h.Counter 85 | } 86 | // replace heads 87 | if mhb.threads[t].heads == nil { 88 | mhb.threads[t] = struct { 89 | heads map[peer.ID]map[cid.Cid]int64 90 | edge uint64 91 | }{heads: make(map[peer.ID]map[cid.Cid]int64)} 92 | } 93 | mhb.threads[t].heads[p] = hset 94 | return nil 95 | } 96 | 97 | func (mhb *memoryHeadBook) Heads(t thread.ID, p peer.ID) ([]thread.Head, error) { 98 | mhb.RLock() 99 | defer mhb.RUnlock() 100 | 101 | hset := mhb.getHeads(t, p, false) 102 | if hset == nil { 103 | return nil, nil 104 | } 105 | 106 | var heads = make([]thread.Head, 0, len(hset)) 107 | for h, c := range hset { 108 | heads = append(heads, thread.Head{h, c}) 109 | } 110 | return heads, nil 111 | } 112 | 113 | func (mhb *memoryHeadBook) ClearHeads(t thread.ID, p peer.ID) error { 114 | mhb.Lock() 115 | defer mhb.Unlock() 116 | 117 | var lset = mhb.threads[t].heads 118 | if lset == nil { 119 | return nil 120 | } 121 | delete(lset, p) 122 | if len(lset) == 0 { 123 | delete(mhb.threads, t) 124 | } else { 125 | mhb.updateEdge(t) 126 | } 127 | return nil 128 | } 129 | 130 | func (mhb *memoryHeadBook) HeadsEdge(t thread.ID) (uint64, error) { 131 | mhb.RLock() 132 | defer mhb.RUnlock() 133 | 134 | lset, found := mhb.threads[t] 135 | if !found { 136 | return 0, core.ErrThreadNotFound 137 | } 138 | // invariant: edge always precomputed for existing thread 139 | return lset.edge, nil 140 | } 141 | 142 | func (mhb *memoryHeadBook) updateEdge(t thread.ID) { 143 | // invariant: requested thread exist 144 | var ( 145 | lset = mhb.threads[t] 146 | heads = make([]util.LogHead, 0, len(lset.heads)) 147 | ) 148 | for lid, hs := range lset.heads { 149 | for head, counter := range hs { 150 | heads = append(heads, util.LogHead{ 151 | LogID: lid, 152 | Head: thread.Head{head, counter}, 153 | }) 154 | } 155 | } 156 | lset.edge = util.ComputeHeadsEdge(heads) 157 | mhb.threads[t] = lset 158 | } 159 | 160 | func (mhb *memoryHeadBook) DumpHeads() (core.DumpHeadBook, error) { 161 | var dump = core.DumpHeadBook{ 162 | Data: make(map[thread.ID]map[peer.ID][]thread.Head, len(mhb.threads)), 163 | } 164 | 165 | for tid, lset := range mhb.threads { 166 | lm := make(map[peer.ID][]thread.Head, len(lset.heads)) 167 | for lid, hs := range lset.heads { 168 | heads := make([]thread.Head, 0, len(hs)) 169 | for head, counter := range hs { 170 | heads = append(heads, thread.Head{head, counter}) 171 | } 172 | lm[lid] = heads 173 | } 174 | dump.Data[tid] = lm 175 | } 176 | 177 | return dump, nil 178 | } 179 | 180 | func (mhb *memoryHeadBook) RestoreHeads(dump core.DumpHeadBook) error { 181 | if !AllowEmptyRestore && len(dump.Data) == 0 { 182 | return core.ErrEmptyDump 183 | } 184 | 185 | // reset stored 186 | mhb.threads = make(map[thread.ID]struct { 187 | heads map[peer.ID]map[cid.Cid]int64 188 | edge uint64 189 | }, len(dump.Data)) 190 | 191 | for tid, logs := range dump.Data { 192 | lm := make(map[peer.ID]map[cid.Cid]int64, len(logs)) 193 | for lid, hs := range logs { 194 | hm := make(map[cid.Cid]int64, len(hs)) 195 | for _, head := range hs { 196 | hm[head.ID] = head.Counter 197 | } 198 | lm[lid] = hm 199 | } 200 | mhb.threads[tid] = struct { 201 | heads map[peer.ID]map[cid.Cid]int64 202 | edge uint64 203 | }{heads: lm} 204 | mhb.updateEdge(tid) 205 | } 206 | 207 | return nil 208 | } 209 | -------------------------------------------------------------------------------- /logstore/lstoremem/inmem_test.go: -------------------------------------------------------------------------------- 1 | package lstoremem_test 2 | 3 | import ( 4 | "testing" 5 | 6 | core "github.com/textileio/go-threads/core/logstore" 7 | m "github.com/textileio/go-threads/logstore/lstoremem" 8 | pt "github.com/textileio/go-threads/test" 9 | ) 10 | 11 | func TestInMemoryLogstore(t *testing.T) { 12 | pt.LogstoreTest(t, func() (core.Logstore, func()) { 13 | return m.NewLogstore(), nil 14 | }) 15 | } 16 | 17 | func TestInMemoryAddrBook(t *testing.T) { 18 | pt.AddrBookTest(t, func() (core.AddrBook, func()) { 19 | return m.NewAddrBook(), nil 20 | }) 21 | } 22 | 23 | func TestInMemoryKeyBook(t *testing.T) { 24 | pt.KeyBookTest(t, func() (core.KeyBook, func()) { 25 | return m.NewKeyBook(), nil 26 | }) 27 | } 28 | 29 | func TestInMemoryHeadBook(t *testing.T) { 30 | pt.HeadBookTest(t, func() (core.HeadBook, func()) { 31 | return m.NewHeadBook(), nil 32 | }) 33 | } 34 | 35 | func TestInMemoryMetadataBook(t *testing.T) { 36 | pt.MetadataBookTest(t, func() (core.ThreadMetadata, func()) { 37 | return m.NewThreadMetadata(), nil 38 | }) 39 | } 40 | 41 | func BenchmarkInMemoryLogstore(b *testing.B) { 42 | pt.BenchmarkLogstore(b, func() (core.Logstore, func()) { 43 | return m.NewLogstore(), nil 44 | }, "InMem") 45 | } 46 | 47 | func BenchmarkInMemoryKeyBook(b *testing.B) { 48 | pt.BenchmarkKeyBook(b, func() (core.KeyBook, func()) { 49 | return m.NewKeyBook(), nil 50 | }) 51 | } 52 | 53 | func BenchmarkInMemoryHeadBook(b *testing.B) { 54 | pt.BenchmarkHeadBook(b, func() (core.HeadBook, func()) { 55 | return m.NewHeadBook(), nil 56 | }) 57 | } 58 | -------------------------------------------------------------------------------- /logstore/lstoremem/logstore.go: -------------------------------------------------------------------------------- 1 | package lstoremem 2 | 3 | import ( 4 | core "github.com/textileio/go-threads/core/logstore" 5 | lstore "github.com/textileio/go-threads/logstore" 6 | ) 7 | 8 | // Define if storage will accept empty dumps. 9 | var AllowEmptyRestore = true 10 | 11 | // NewLogstore creates an in-memory threadsafe collection of thread logs. 12 | func NewLogstore() core.Logstore { 13 | return lstore.NewLogstore( 14 | NewKeyBook(), 15 | NewAddrBook(), 16 | NewHeadBook(), 17 | NewThreadMetadata()) 18 | } 19 | -------------------------------------------------------------------------------- /logstore/lstoremem/metadata.go: -------------------------------------------------------------------------------- 1 | package lstoremem 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | core "github.com/textileio/go-threads/core/logstore" 8 | "github.com/textileio/go-threads/core/thread" 9 | ) 10 | 11 | type memoryThreadMetadata struct { 12 | ds map[core.MetadataKey]interface{} 13 | dslock sync.RWMutex 14 | } 15 | 16 | var _ core.ThreadMetadata = (*memoryThreadMetadata)(nil) 17 | 18 | func NewThreadMetadata() core.ThreadMetadata { 19 | return &memoryThreadMetadata{ 20 | ds: make(map[core.MetadataKey]interface{}), 21 | } 22 | } 23 | 24 | func (m *memoryThreadMetadata) PutInt64(t thread.ID, key string, val int64) error { 25 | m.putValue(t, key, val) 26 | return nil 27 | } 28 | 29 | func (m *memoryThreadMetadata) GetInt64(t thread.ID, key string) (*int64, error) { 30 | val, ok := m.getValue(t, key).(int64) 31 | if !ok { 32 | return nil, nil 33 | } 34 | return &val, nil 35 | } 36 | 37 | func (m *memoryThreadMetadata) PutString(t thread.ID, key string, val string) error { 38 | m.putValue(t, key, val) 39 | return nil 40 | } 41 | 42 | func (m *memoryThreadMetadata) GetString(t thread.ID, key string) (*string, error) { 43 | val, ok := m.getValue(t, key).(string) 44 | if !ok { 45 | return nil, nil 46 | } 47 | return &val, nil 48 | } 49 | 50 | func (m *memoryThreadMetadata) PutBool(t thread.ID, key string, val bool) error { 51 | m.putValue(t, key, val) 52 | return nil 53 | } 54 | 55 | func (m *memoryThreadMetadata) GetBool(t thread.ID, key string) (*bool, error) { 56 | val, ok := m.getValue(t, key).(bool) 57 | if !ok { 58 | return nil, nil 59 | } 60 | return &val, nil 61 | } 62 | 63 | func (m *memoryThreadMetadata) PutBytes(t thread.ID, key string, val []byte) error { 64 | b := make([]byte, len(val)) 65 | copy(b, val) 66 | m.putValue(t, key, b) 67 | return nil 68 | } 69 | 70 | func (m *memoryThreadMetadata) GetBytes(t thread.ID, key string) (*[]byte, error) { 71 | val, ok := m.getValue(t, key).([]byte) 72 | if !ok { 73 | return nil, nil 74 | } 75 | return &val, nil 76 | } 77 | 78 | func (m *memoryThreadMetadata) putValue(t thread.ID, key string, val interface{}) { 79 | m.dslock.Lock() 80 | defer m.dslock.Unlock() 81 | m.ds[core.MetadataKey{T: t, K: key}] = val 82 | } 83 | 84 | func (m *memoryThreadMetadata) getValue(t thread.ID, key string) interface{} { 85 | m.dslock.RLock() 86 | defer m.dslock.RUnlock() 87 | if v, ok := m.ds[core.MetadataKey{T: t, K: key}]; ok { 88 | return v 89 | } 90 | return nil 91 | } 92 | 93 | func (m *memoryThreadMetadata) ClearMetadata(t thread.ID) error { 94 | m.dslock.Lock() 95 | defer m.dslock.Unlock() 96 | for k := range m.ds { 97 | if k.T.Equals(t) { 98 | delete(m.ds, k) 99 | } 100 | } 101 | return nil 102 | } 103 | 104 | func (m *memoryThreadMetadata) DumpMeta() (core.DumpMetadata, error) { 105 | m.dslock.RLock() 106 | defer m.dslock.RUnlock() 107 | 108 | var ( 109 | dump core.DumpMetadata 110 | vInt64 = make(map[core.MetadataKey]int64) 111 | vBool = make(map[core.MetadataKey]bool) 112 | vString = make(map[core.MetadataKey]string) 113 | vBytes = make(map[core.MetadataKey][]byte) 114 | ) 115 | 116 | for mk, value := range m.ds { 117 | switch v := value.(type) { 118 | case bool: 119 | vBool[mk] = v 120 | case int64: 121 | vInt64[mk] = v 122 | case string: 123 | vString[mk] = v 124 | case []byte: 125 | vBytes[mk] = v 126 | default: 127 | return dump, fmt.Errorf("unsupported value type %T, key: %v, value: %v", value, mk, value) 128 | } 129 | } 130 | 131 | dump.Data.Bool = vBool 132 | dump.Data.Int64 = vInt64 133 | dump.Data.String = vString 134 | dump.Data.Bytes = vBytes 135 | return dump, nil 136 | } 137 | 138 | func (m *memoryThreadMetadata) RestoreMeta(dump core.DumpMetadata) error { 139 | var dataLen = len(dump.Data.Bool) + 140 | len(dump.Data.Int64) + 141 | len(dump.Data.String) + 142 | len(dump.Data.Bytes) 143 | if !AllowEmptyRestore && dataLen == 0 { 144 | return core.ErrEmptyDump 145 | } 146 | 147 | m.dslock.Lock() 148 | defer m.dslock.Unlock() 149 | 150 | // clear local data 151 | m.ds = make(map[core.MetadataKey]interface{}, dataLen) 152 | 153 | // replace with dump 154 | for mk, val := range dump.Data.Bool { 155 | m.ds[mk] = val 156 | } 157 | for mk, val := range dump.Data.Int64 { 158 | m.ds[mk] = val 159 | } 160 | for mk, val := range dump.Data.String { 161 | m.ds[mk] = val 162 | } 163 | for mk, val := range dump.Data.Bytes { 164 | m.ds[mk] = val 165 | } 166 | 167 | return nil 168 | } 169 | -------------------------------------------------------------------------------- /net/api/pb/Makefile: -------------------------------------------------------------------------------- 1 | PB = $(wildcard *.proto) 2 | GO = $(PB:.proto=.pb.go) 3 | PROTOC_GEN_TS_PATH = "./javascript/node_modules/.bin/protoc-gen-ts" 4 | 5 | all: $(GO) 6 | 7 | %.pb.go: %.proto 8 | protoc -I=. \ 9 | --plugin="protoc-gen-ts=${PROTOC_GEN_TS_PATH}" --js_out="import_style=commonjs,binary:javascript/." --ts_out="service=grpc-web:javascript/." \ 10 | --go_out=. --go_opt=paths=source_relative \ 11 | --go-grpc_out=. --go-grpc_opt=paths=source_relative \ 12 | $< 13 | 14 | clean: 15 | rm -f *.pb.go 16 | rm -f *pb_test.go 17 | rm -f ./javascript/*.js 18 | rm -f ./javascript/*.d.ts 19 | 20 | .PHONY: clean -------------------------------------------------------------------------------- /net/api/pb/javascript/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@textile/threads-net-grpc", 3 | "version": "0.0.0", 4 | "lockfileVersion": 2, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "@textile/threads-net-grpc", 9 | "version": "0.0.0", 10 | "license": "MIT", 11 | "dependencies": { 12 | "@improbable-eng/grpc-web": "^0.14.1", 13 | "@types/google-protobuf": "^3.15.5", 14 | "google-protobuf": "^3.19.4" 15 | }, 16 | "devDependencies": { 17 | "ts-protoc-gen": "^0.15.0" 18 | } 19 | }, 20 | "node_modules/@improbable-eng/grpc-web": { 21 | "version": "0.14.1", 22 | "resolved": "https://registry.npmjs.org/@improbable-eng/grpc-web/-/grpc-web-0.14.1.tgz", 23 | "integrity": "sha512-XaIYuunepPxoiGVLLHmlnVminUGzBTnXr8Wv7khzmLWbNw4TCwJKX09GSMJlKhu/TRk6gms0ySFxewaETSBqgw==", 24 | "dependencies": { 25 | "browser-headers": "^0.4.1" 26 | }, 27 | "peerDependencies": { 28 | "google-protobuf": "^3.14.0" 29 | } 30 | }, 31 | "node_modules/@types/google-protobuf": { 32 | "version": "3.15.5", 33 | "resolved": "https://registry.npmjs.org/@types/google-protobuf/-/google-protobuf-3.15.5.tgz", 34 | "integrity": "sha512-6bgv24B+A2bo9AfzReeg5StdiijKzwwnRflA8RLd1V4Yv995LeTmo0z69/MPbBDFSiZWdZHQygLo/ccXhMEDgw==" 35 | }, 36 | "node_modules/browser-headers": { 37 | "version": "0.4.1", 38 | "resolved": "https://registry.npmjs.org/browser-headers/-/browser-headers-0.4.1.tgz", 39 | "integrity": "sha512-CA9hsySZVo9371qEHjHZtYxV2cFtVj5Wj/ZHi8ooEsrtm4vOnl9Y9HmyYWk9q+05d7K3rdoAE0j3MVEFVvtQtg==" 40 | }, 41 | "node_modules/google-protobuf": { 42 | "version": "3.19.4", 43 | "resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.19.4.tgz", 44 | "integrity": "sha512-OIPNCxsG2lkIvf+P5FNfJ/Km95CsXOBecS9ZcAU6m2Rq3svc0Apl9nB3GMDNKfQ9asNv4KjyAqGwPQFrVle3Yg==" 45 | }, 46 | "node_modules/ts-protoc-gen": { 47 | "version": "0.15.0", 48 | "resolved": "https://registry.npmjs.org/ts-protoc-gen/-/ts-protoc-gen-0.15.0.tgz", 49 | "integrity": "sha512-TycnzEyrdVDlATJ3bWFTtra3SCiEP0W0vySXReAuEygXCUr1j2uaVyL0DhzjwuUdQoW5oXPwk6oZWeA0955V+g==", 50 | "dev": true, 51 | "dependencies": { 52 | "google-protobuf": "^3.15.5" 53 | }, 54 | "bin": { 55 | "protoc-gen-ts": "bin/protoc-gen-ts" 56 | } 57 | } 58 | }, 59 | "dependencies": { 60 | "@improbable-eng/grpc-web": { 61 | "version": "0.14.1", 62 | "resolved": "https://registry.npmjs.org/@improbable-eng/grpc-web/-/grpc-web-0.14.1.tgz", 63 | "integrity": "sha512-XaIYuunepPxoiGVLLHmlnVminUGzBTnXr8Wv7khzmLWbNw4TCwJKX09GSMJlKhu/TRk6gms0ySFxewaETSBqgw==", 64 | "requires": { 65 | "browser-headers": "^0.4.1" 66 | } 67 | }, 68 | "@types/google-protobuf": { 69 | "version": "3.15.5", 70 | "resolved": "https://registry.npmjs.org/@types/google-protobuf/-/google-protobuf-3.15.5.tgz", 71 | "integrity": "sha512-6bgv24B+A2bo9AfzReeg5StdiijKzwwnRflA8RLd1V4Yv995LeTmo0z69/MPbBDFSiZWdZHQygLo/ccXhMEDgw==" 72 | }, 73 | "browser-headers": { 74 | "version": "0.4.1", 75 | "resolved": "https://registry.npmjs.org/browser-headers/-/browser-headers-0.4.1.tgz", 76 | "integrity": "sha512-CA9hsySZVo9371qEHjHZtYxV2cFtVj5Wj/ZHi8ooEsrtm4vOnl9Y9HmyYWk9q+05d7K3rdoAE0j3MVEFVvtQtg==" 77 | }, 78 | "google-protobuf": { 79 | "version": "3.19.4", 80 | "resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.19.4.tgz", 81 | "integrity": "sha512-OIPNCxsG2lkIvf+P5FNfJ/Km95CsXOBecS9ZcAU6m2Rq3svc0Apl9nB3GMDNKfQ9asNv4KjyAqGwPQFrVle3Yg==" 82 | }, 83 | "ts-protoc-gen": { 84 | "version": "0.15.0", 85 | "resolved": "https://registry.npmjs.org/ts-protoc-gen/-/ts-protoc-gen-0.15.0.tgz", 86 | "integrity": "sha512-TycnzEyrdVDlATJ3bWFTtra3SCiEP0W0vySXReAuEygXCUr1j2uaVyL0DhzjwuUdQoW5oXPwk6oZWeA0955V+g==", 87 | "dev": true, 88 | "requires": { 89 | "google-protobuf": "^3.15.5" 90 | } 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /net/api/pb/javascript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@textile/threads-net-grpc", 3 | "version": "0.0.0", 4 | "description": "A gRPC client for interacting with a remote ThreadDB network service.", 5 | "repository": { 6 | "type": "git", 7 | "url": "git+https://github.com/textileio/go-threads.git" 8 | }, 9 | "author": "Textile", 10 | "license": "MIT", 11 | "files": [ 12 | "threadsnet_pb.js", 13 | "threadsnet_pb_service.js", 14 | "threadsnet_pb.d.ts", 15 | "threadsnet_pb_service.d.ts" 16 | ], 17 | "dependencies": { 18 | "@improbable-eng/grpc-web": "^0.14.1", 19 | "@types/google-protobuf": "^3.15.5", 20 | "google-protobuf": "^3.19.4" 21 | }, 22 | "devDependencies": { 23 | "ts-protoc-gen": "^0.15.0" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /net/api/pb/threadsnet.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package threads.net.pb; 3 | 4 | option go_package = "github.com/go-threads/net/api/pb/threads_net_pb"; 5 | option java_multiple_files = true; 6 | option java_package = "io.textile.threads_net_grpc"; 7 | option java_outer_classname = "ThreadsNet"; 8 | option objc_class_prefix = "THREADSNET"; 9 | 10 | message GetHostIDRequest {} 11 | 12 | message GetHostIDReply { 13 | bytes peerID = 1; 14 | } 15 | 16 | message GetTokenRequest { 17 | oneof payload { 18 | string key = 1; 19 | bytes signature = 2; 20 | } 21 | } 22 | 23 | message GetTokenReply { 24 | oneof payload { 25 | bytes challenge = 1; 26 | string token = 2; 27 | } 28 | } 29 | 30 | message CreateThreadRequest { 31 | bytes threadID = 1; 32 | Keys keys = 2; 33 | } 34 | 35 | message Keys { 36 | bytes threadKey = 1; 37 | bytes logKey = 2; 38 | } 39 | 40 | message ThreadInfoReply { 41 | bytes threadID = 1; 42 | bytes threadKey = 2; 43 | repeated LogInfo logs = 3; 44 | repeated bytes addrs = 4; 45 | } 46 | 47 | message LogInfo { 48 | bytes ID = 1; 49 | bytes pubKey = 2; 50 | bytes privKey = 3; 51 | repeated bytes addrs = 4; 52 | bytes head = 5; 53 | bytes counter = 6; 54 | } 55 | 56 | message AddThreadRequest { 57 | bytes addr = 1; 58 | Keys keys = 2; 59 | } 60 | 61 | message GetThreadRequest { 62 | bytes threadID = 1; 63 | } 64 | 65 | message PullThreadRequest { 66 | bytes threadID = 1; 67 | } 68 | 69 | message PullThreadReply {} 70 | 71 | message DeleteThreadRequest { 72 | bytes threadID = 1; 73 | } 74 | 75 | message DeleteThreadReply {} 76 | 77 | message AddReplicatorRequest { 78 | bytes threadID = 1; 79 | bytes addr = 2; 80 | } 81 | 82 | message AddReplicatorReply { 83 | bytes peerID = 1; 84 | } 85 | 86 | message CreateRecordRequest { 87 | bytes threadID = 1; 88 | bytes body = 2; 89 | } 90 | 91 | message NewRecordReply { 92 | bytes threadID = 1; 93 | bytes logID = 2; 94 | Record record = 3; 95 | } 96 | 97 | message AddRecordRequest { 98 | bytes threadID = 1; 99 | bytes logID = 2; 100 | Record record = 3; 101 | } 102 | 103 | message Record { 104 | bytes recordNode = 1; 105 | bytes eventNode = 2; 106 | bytes headerNode = 3; 107 | bytes bodyNode = 4; 108 | } 109 | 110 | message AddRecordReply {} 111 | 112 | message GetRecordRequest { 113 | bytes threadID = 1; 114 | bytes recordID = 2; 115 | } 116 | 117 | message GetRecordReply { 118 | Record record = 1; 119 | } 120 | 121 | message SubscribeRequest { 122 | repeated bytes threadIDs = 1; 123 | } 124 | 125 | service API { 126 | rpc GetHostID(GetHostIDRequest) returns (GetHostIDReply) {} 127 | rpc GetToken(stream GetTokenRequest) returns (stream GetTokenReply) {} 128 | rpc CreateThread(CreateThreadRequest) returns (ThreadInfoReply) {} 129 | rpc AddThread(AddThreadRequest) returns (ThreadInfoReply) {} 130 | rpc GetThread(GetThreadRequest) returns (ThreadInfoReply) {} 131 | rpc PullThread(PullThreadRequest) returns (PullThreadReply) {} 132 | rpc DeleteThread(DeleteThreadRequest) returns (DeleteThreadReply) {} 133 | rpc AddReplicator(AddReplicatorRequest) returns (AddReplicatorReply) {} 134 | rpc CreateRecord(CreateRecordRequest) returns (NewRecordReply) {} 135 | rpc AddRecord(AddRecordRequest) returns (AddRecordReply) {} 136 | rpc GetRecord(GetRecordRequest) returns (GetRecordReply) {} 137 | rpc Subscribe(SubscribeRequest) returns (stream NewRecordReply) {} 138 | } 139 | -------------------------------------------------------------------------------- /net/api/test_util.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "errors" 5 | "io/ioutil" 6 | "math/rand" 7 | "net" 8 | "os" 9 | "time" 10 | 11 | ma "github.com/multiformats/go-multiaddr" 12 | "github.com/textileio/go-threads/common" 13 | pb "github.com/textileio/go-threads/net/api/pb" 14 | "github.com/textileio/go-threads/util" 15 | "google.golang.org/grpc" 16 | ) 17 | 18 | // CreateTestService creates a test network API gRPC service for test purpose. 19 | // It uses either the addr passed in as host addr, or pick an available local addr if it is empty 20 | func CreateTestService(addr string, debug bool) (hostAddr ma.Multiaddr, gRPCAddr ma.Multiaddr, stop func(), err error) { 21 | time.Sleep(time.Second * time.Duration(rand.Intn(5))) 22 | dir, err := ioutil.TempDir("", "") 23 | if err != nil { 24 | return 25 | } 26 | if addr == "" { 27 | hostAddr = util.FreeLocalAddr() 28 | } else { 29 | hostAddr, _ = ma.NewMultiaddr(addr) 30 | } 31 | n, err := common.DefaultNetwork( 32 | common.WithNetBadgerPersistence(dir), 33 | common.WithNetHostAddr(hostAddr), 34 | common.WithNetPubSub(true), 35 | common.WithNetDebug(debug), 36 | ) 37 | if err != nil { 38 | return 39 | } 40 | service, err := NewService(n, Config{ 41 | Debug: debug, 42 | }) 43 | if err != nil { 44 | return 45 | } 46 | gRPCAddr = util.FreeLocalAddr() 47 | target, err := util.TCPAddrFromMultiAddr(gRPCAddr) 48 | if err != nil { 49 | return 50 | } 51 | server := grpc.NewServer() 52 | listener, err := net.Listen("tcp", target) 53 | if err != nil { 54 | return 55 | } 56 | go func() { 57 | pb.RegisterAPIServer(server, service) 58 | if err := server.Serve(listener); err != nil && !errors.Is(err, grpc.ErrServerStopped) { 59 | log.Fatalf("serve error: %v", err) 60 | } 61 | }() 62 | 63 | return hostAddr, gRPCAddr, func() { 64 | util.StopGRPCServer(server) 65 | if err := n.Close(); err != nil { 66 | return 67 | } 68 | _ = os.RemoveAll(dir) 69 | }, nil 70 | } 71 | -------------------------------------------------------------------------------- /net/pb/Makefile: -------------------------------------------------------------------------------- 1 | PB = $(wildcard *.proto) 2 | GO = $(PB:.proto=.pb.go) 3 | 4 | all: $(GO) 5 | 6 | %.pb.go: %.proto 7 | protoc -I=. -I=$(GOPATH)/src -I=$(GOPATH)/src/github.com/gogo/protobuf/protobuf --gogofaster_out=\ 8 | plugins=grpc:\ 9 | . $< 10 | 11 | clean: 12 | rm -f *.pb.go 13 | rm -f *pb_test.go 14 | 15 | .PHONY: clean -------------------------------------------------------------------------------- /net/pb/lstore.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package net.pb; 3 | 4 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 5 | 6 | option (gogoproto.benchgen_all) = true; 7 | option (gogoproto.populate_all) = true; 8 | 9 | // AddrBookRecord represents a record for a log in the address book. 10 | message AddrBookRecord { 11 | // Thread ID. 12 | bytes threadID = 1 [(gogoproto.customtype) = "ProtoThreadID"]; 13 | 14 | // The peer ID. 15 | bytes peerID = 2 [(gogoproto.customtype) = "ProtoPeerID"]; 16 | 17 | // The multiaddresses. This is a sorted list where element 0 expires the soonest. 18 | repeated AddrEntry addrs = 3; 19 | 20 | // AddrEntry represents a single multiaddress. 21 | message AddrEntry { 22 | bytes addr = 1 [(gogoproto.customtype) = "ProtoAddr"]; 23 | 24 | // The point in time when this address expires. 25 | int64 expiry = 2; 26 | 27 | // The original TTL of this address. 28 | int64 ttl = 3; 29 | } 30 | } 31 | 32 | // HeadBookRecord represents the list of heads currently in a log 33 | message HeadBookRecord { 34 | // List of current heads of a log. 35 | repeated HeadEntry heads = 1; 36 | 37 | // HeadEntry represents a single cid. 38 | message HeadEntry { 39 | bytes cid = 1 [(gogoproto.customtype) = "ProtoCid"]; 40 | 41 | int64 counter = 2; 42 | } 43 | } -------------------------------------------------------------------------------- /net/queue/common.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "context" 5 | 6 | logging "github.com/ipfs/go-log/v2" 7 | "github.com/libp2p/go-libp2p-core/peer" 8 | "github.com/textileio/go-threads/core/thread" 9 | ) 10 | 11 | var log = logging.Logger("netqueue") 12 | 13 | type ( 14 | PeerCall func(context.Context, peer.ID, thread.ID) error 15 | 16 | CallQueue interface { 17 | // Make call immediately and synchronously return its result. 18 | Call(p peer.ID, t thread.ID, c PeerCall) error 19 | 20 | // Schedule call to be invoked later. 21 | Schedule(p peer.ID, t thread.ID, priority int, c PeerCall) bool 22 | } 23 | ) 24 | 25 | type ( 26 | ThreadPack struct { 27 | Peer peer.ID 28 | Threads []thread.ID 29 | } 30 | 31 | ThreadPacker interface { 32 | // Add thread to peer's queue 33 | Add(pid peer.ID, tid thread.ID) 34 | 35 | // Start packing incoming thread requests 36 | Run() <-chan ThreadPack 37 | } 38 | ) 39 | -------------------------------------------------------------------------------- /net/queue/ff_test.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/libp2p/go-libp2p-core/peer" 8 | "github.com/textileio/go-threads/core/thread" 9 | ) 10 | 11 | func TestOperationQueue(t *testing.T) { 12 | var ( 13 | q = newPeerQueue() 14 | t1 = thread.NewIDV1(thread.Raw, 32) 15 | t2 = thread.NewIDV1(thread.Raw, 32) 16 | t3 = thread.NewIDV1(thread.Raw, 32) 17 | 18 | // auxiliary 19 | eval = func(op PeerCall) { _ = op(nil, "", thread.Undef) } 20 | val int 21 | ) 22 | 23 | if _, tid, _, ok := q.Pop(); ok || tid != thread.Undef { 24 | t.Error("unexpected element in the empty queue") 25 | } 26 | if size := q.Size(); size != 0 { 27 | t.Errorf("size of the empty queue is %d", size) 28 | } 29 | 30 | operations := []struct { 31 | tid thread.ID 32 | call PeerCall 33 | priority int 34 | }{ 35 | { 36 | tid: t1, 37 | call: func(context.Context, peer.ID, thread.ID) error { val = 1; return nil }, 38 | priority: 1, 39 | }, 40 | { 41 | tid: t2, 42 | call: func(context.Context, peer.ID, thread.ID) error { val = 2; return nil }, 43 | priority: 1, 44 | }, 45 | { 46 | tid: t3, 47 | call: func(context.Context, peer.ID, thread.ID) error { val = 3; return nil }, 48 | priority: 1, 49 | }, 50 | } 51 | 52 | for _, e := range operations { 53 | if !q.Add(e.tid, e.call, e.priority) { 54 | t.Error("no indication of adding new thread operation") 55 | } 56 | } 57 | 58 | // sequence: t1 -> t2 -> t3 59 | if size := q.Size(); size != len(operations) { 60 | t.Errorf("bad queue size, expected: %d, got: %d", len(operations), size) 61 | } 62 | 63 | // remove t2, sequence: t1 -> t3 64 | if !q.Remove(t2) || q.Size() != len(operations)-1 { 65 | t.Error("bad operation removal") 66 | } 67 | 68 | // evaluate call for t1 69 | if op, tid, _, ok := q.Pop(); !ok || tid != t1 { 70 | t.Error("cannot get expected operation") 71 | } else { 72 | eval(op) 73 | expected := 1 74 | if val != expected { 75 | t.Errorf("wrong call, expected value: %d, got: %d", expected, val) 76 | } 77 | } 78 | 79 | // add all the operations again, but last one should still be in the queue 80 | for i, e := range operations { 81 | if q.Add(e.tid, e.call, e.priority) == (i == 2) { 82 | t.Error("wrong indication of adding thread operation") 83 | } 84 | } 85 | 86 | // sequence: t3 -> t1 -> t2, evaluate t3 87 | if op, tid, _, ok := q.Pop(); !ok || tid != t3 { 88 | t.Error("cannot get expected operation") 89 | } else { 90 | eval(op) 91 | expected := 3 92 | if val != expected { 93 | t.Errorf("wrong call, expected value: %d, got: %d", expected, val) 94 | } 95 | } 96 | 97 | // replace t1 call 98 | if q.Add(t1, func(context.Context, peer.ID, thread.ID) error { val = 123; return nil }, 5) { 99 | t.Error("replacing lower-priority call should not return true") 100 | } 101 | 102 | // sequence: t1 (replaced) -> t2 103 | if op, tid, _, ok := q.Pop(); !ok || tid != t1 { 104 | t.Error("cannot get expected operation") 105 | } else { 106 | eval(op) 107 | expected := 123 108 | if val != expected { 109 | t.Errorf("wrong call, expected value: %d, got: %d", expected, val) 110 | } 111 | } 112 | 113 | // sequence: t2 114 | if op, tid, _, ok := q.Pop(); !ok || tid != t2 { 115 | t.Error("cannot get expected operation") 116 | } else { 117 | eval(op) 118 | expected := 2 119 | if val != expected { 120 | t.Errorf("wrong call, expected value: %d, got: %d", expected, val) 121 | } 122 | } 123 | 124 | // sequence: empty 125 | if _, tid, _, ok := q.Pop(); ok || q.Size() != 0 || tid != thread.Undef { 126 | t.Error("unexpected operations in the queue") 127 | } 128 | } 129 | 130 | func TestOperationQueue_Pop(t *testing.T) { 131 | var ( 132 | q = newPeerQueue() 133 | t1 = thread.NewIDV1(thread.Raw, 32) 134 | t2 = thread.NewIDV1(thread.Raw, 32) 135 | 136 | checkedPop = func(expect bool, tid thread.ID) { 137 | if _, stid, _, ok := q.Pop(); expect && !ok { 138 | t.Errorf("expected call for %s, but queue is empty", tid) 139 | } else if !expect && ok { 140 | t.Error("unexpected call popped from the queue") 141 | } else if expect && ok && stid != tid { 142 | t.Errorf("expected call for %s, but get call for %s", tid, stid) 143 | } 144 | } 145 | ) 146 | 147 | // empty queue 148 | checkedPop(false, thread.Undef) 149 | 150 | // add single thread 151 | q.Add(t1, nil, 1) 152 | checkedPop(true, t1) 153 | checkedPop(false, thread.Undef) 154 | 155 | // add and remove 156 | q.Add(t1, nil, 1) 157 | q.Remove(t1) 158 | checkedPop(false, thread.Undef) 159 | 160 | // add two threads with duplicates 161 | q.Add(t1, nil, 1) 162 | q.Add(t2, nil, 1) 163 | q.Add(t2, nil, 1) 164 | q.Add(t1, nil, 1) 165 | checkedPop(true, t1) 166 | checkedPop(true, t2) 167 | checkedPop(false, thread.Undef) 168 | checkedPop(false, thread.Undef) 169 | } 170 | -------------------------------------------------------------------------------- /net/queue/tp.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/libp2p/go-libp2p-core/peer" 8 | "github.com/textileio/go-threads/core/thread" 9 | ) 10 | 11 | var ( 12 | // Size of incoming requests buffer 13 | InBufSize = 1 14 | 15 | // Size of packed threads buffer 16 | OutBufSize = 1 17 | ) 18 | 19 | var _ ThreadPacker = (*threadPacker)(nil) 20 | 21 | type ( 22 | tEntry struct { 23 | tid thread.ID 24 | added int64 25 | } 26 | 27 | request struct { 28 | pid peer.ID 29 | tid thread.ID 30 | added int64 31 | } 32 | 33 | threadPacker struct { 34 | ctx context.Context 35 | peers map[peer.ID][]tEntry 36 | input chan request 37 | timeout time.Duration 38 | maxPackSize int 39 | } 40 | ) 41 | 42 | // Packer accumulates peer-related thread requests and packs it into the 43 | // limited-size containers during time-window constrained by provided timeout. 44 | func NewThreadPacker(ctx context.Context, maxPackSize int, timeout time.Duration) *threadPacker { 45 | return &threadPacker{ 46 | peers: make(map[peer.ID][]tEntry), 47 | input: make(chan request, InBufSize), 48 | timeout: timeout, 49 | maxPackSize: maxPackSize, 50 | ctx: ctx, 51 | } 52 | } 53 | 54 | func (q *threadPacker) Add(pid peer.ID, tid thread.ID) { 55 | q.input <- request{ 56 | pid: pid, 57 | tid: tid, 58 | added: time.Now().Unix(), 59 | } 60 | } 61 | 62 | func (q *threadPacker) Run() <-chan ThreadPack { 63 | var sink = make(chan ThreadPack, OutBufSize) 64 | 65 | go func() { 66 | tm := time.NewTicker(q.timeout) 67 | defer tm.Stop() 68 | 69 | for { 70 | select { 71 | case <-q.ctx.Done(): 72 | for pid := range q.peers { 73 | q.drainPeerQueue(pid, sink) 74 | } 75 | close(sink) 76 | return 77 | 78 | case <-tm.C: 79 | // periodic check for inactive peer queues with overdue entries 80 | var now = time.Now().Unix() 81 | for pid, pq := range q.peers { 82 | if len(pq) > 0 && now-pq[0].added >= int64(q.timeout/time.Second) { 83 | q.drainPeerQueue(pid, sink) 84 | } 85 | } 86 | 87 | case req := <-q.input: 88 | var pq = q.peers[req.pid] 89 | pq = append(pq, tEntry{tid: req.tid, added: req.added}) 90 | q.peers[req.pid] = pq 91 | if len(pq) >= q.maxPackSize || req.added-pq[0].added >= int64(q.timeout/time.Second) { 92 | // max size limit reached or pack is overdue 93 | q.drainPeerQueue(req.pid, sink) 94 | } 95 | } 96 | } 97 | }() 98 | 99 | return sink 100 | } 101 | 102 | func (q *threadPacker) drainPeerQueue(pid peer.ID, sink chan<- ThreadPack) { 103 | pq := q.peers[pid] 104 | if len(pq) == 0 { 105 | return 106 | } 107 | 108 | var ( 109 | pack = ThreadPack{ 110 | Peer: pid, 111 | Threads: make([]thread.ID, 0, len(pq)), 112 | } 113 | unique = make(map[thread.ID]struct{}, len(pq)) 114 | ) 115 | 116 | for i := 0; i < len(pq); i++ { 117 | var tid = pq[i].tid 118 | if _, exist := unique[tid]; !exist { 119 | pack.Threads = append(pack.Threads, tid) 120 | unique[tid] = struct{}{} 121 | } 122 | } 123 | sink <- pack 124 | 125 | // reset queue 126 | q.peers[pid] = pq[:0] 127 | } 128 | -------------------------------------------------------------------------------- /net/queue/tp_test.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/textileio/go-threads/core/thread" 9 | "github.com/textileio/go-threads/test" 10 | ) 11 | 12 | func TestThreadPacker(t *testing.T) { 13 | var ( 14 | maxPack = 3 15 | timeout = 1 * time.Second 16 | ctx, cancel = context.WithCancel(context.Background()) 17 | tp = NewThreadPacker(ctx, maxPack, timeout) 18 | 19 | pid = test.GeneratePeerIDs(1)[0] 20 | tids = make([]thread.ID, 2*maxPack+1) 21 | ) 22 | 23 | for i := 0; i < 2*maxPack+1; i++ { 24 | tids[i] = thread.NewIDV1(thread.Raw, 32) 25 | } 26 | 27 | go func() { 28 | // add: entire pack + another incomplete one 29 | for i := 0; i < 2*maxPack-1; i++ { 30 | tp.Add(pid, tids[i]) 31 | } 32 | 33 | // wait until incomplete pack will be flushed 34 | time.Sleep(timeout + 50*time.Millisecond) 35 | 36 | // add remaining threads 37 | tp.Add(pid, tids[2*maxPack-1]) 38 | tp.Add(pid, tids[2*maxPack]) 39 | 40 | // add thread duplicate 41 | tp.Add(pid, tids[2*maxPack-1]) 42 | 43 | // let last request propagate and stop the packer 44 | time.Sleep(10 * time.Millisecond) 45 | cancel() 46 | }() 47 | 48 | var packs []ThreadPack 49 | for p := range tp.Run() { 50 | packs = append(packs, p) 51 | } 52 | 53 | var equal = func(p1, p2 ThreadPack) bool { 54 | if p1.Peer != p2.Peer || len(p1.Threads) != len(p2.Threads) { 55 | return false 56 | } 57 | for i := 0; i < len(p1.Threads); i++ { 58 | if p1.Threads[i] != p2.Threads[i] { 59 | return false 60 | } 61 | } 62 | return true 63 | } 64 | 65 | if numPacks := len(packs); numPacks != 3 { 66 | t.Errorf("wrong number of packs: %d, expected: 3", numPacks) 67 | } 68 | 69 | if !equal(packs[0], ThreadPack{Peer: pid, Threads: tids[:3]}) { 70 | t.Error("unexpected first pack") 71 | } 72 | if !equal(packs[1], ThreadPack{Peer: pid, Threads: tids[3:5]}) { 73 | t.Error("unexpected second pack") 74 | } 75 | if !equal(packs[2], ThreadPack{Peer: pid, Threads: tids[5:]}) { 76 | t.Error("unexpected final pack") 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /net/record.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/ipfs/go-cid" 8 | "github.com/libp2p/go-libp2p-core/peer" 9 | core "github.com/textileio/go-threads/core/net" 10 | "github.com/textileio/go-threads/core/thread" 11 | ) 12 | 13 | type linkedRecord interface { 14 | Cid() cid.Cid 15 | PrevID() cid.Cid 16 | } 17 | 18 | // Collector maintains an ordered list of records from multiple sources (thread-safe) 19 | type recordCollector struct { 20 | rs map[peer.ID]*recordSequence 21 | counters map[peer.ID]int64 22 | lock sync.Mutex 23 | } 24 | 25 | func newRecordCollector() *recordCollector { 26 | return &recordCollector{ 27 | rs: make(map[peer.ID]*recordSequence), 28 | counters: make(map[peer.ID]int64), 29 | } 30 | } 31 | 32 | // Store the record of the log. 33 | func (r *recordCollector) Store(lid peer.ID, rec core.Record) { 34 | r.lock.Lock() 35 | defer r.lock.Unlock() 36 | 37 | seq, found := r.rs[lid] 38 | if !found { 39 | seq = newRecordSequence() 40 | r.rs[lid] = seq 41 | } 42 | 43 | seq.Store(rec) 44 | } 45 | 46 | func (r *recordCollector) UpdateHeadCounter(lid peer.ID, counter int64) { 47 | r.lock.Lock() 48 | defer r.lock.Unlock() 49 | 50 | // we update the counter only if we have some records 51 | val, found := r.counters[lid] 52 | if !found { 53 | r.counters[lid] = counter 54 | } else if val == thread.CounterUndef || counter == thread.CounterUndef { 55 | // if a peer does not support the new logic we cannot rely on counter comparison 56 | r.counters[lid] = thread.CounterUndef 57 | // setting the counter to have the maximum value of all the logs we got 58 | } else if val < counter { 59 | r.counters[lid] = counter 60 | } 61 | } 62 | 63 | // List all previously stored records in a proper order if the latter exists. 64 | func (r *recordCollector) List() (map[peer.ID]peerRecords, error) { 65 | r.lock.Lock() 66 | defer r.lock.Unlock() 67 | 68 | logSeqs := make(map[peer.ID]peerRecords, len(r.rs)) 69 | for id, seq := range r.rs { 70 | ordered, ok := seq.List() 71 | if !ok { 72 | return nil, fmt.Errorf("disjoint record sequence in log %s", id) 73 | } 74 | 75 | counter, found := r.counters[id] 76 | // this should never happen because we do this for every log 77 | if !found { 78 | return nil, fmt.Errorf("did not find log counter in log %s", id) 79 | } 80 | 81 | casted := make([]core.Record, len(ordered)) 82 | for i := 0; i < len(ordered); i++ { 83 | casted[i] = ordered[i].(core.Record) 84 | } 85 | logSeqs[id] = peerRecords{ 86 | records: casted, 87 | counter: counter, 88 | } 89 | } 90 | 91 | return logSeqs, nil 92 | } 93 | 94 | // Not a thread-safe structure 95 | type recordSequence struct { 96 | fragments [][]linkedRecord 97 | set map[cid.Cid]struct{} 98 | } 99 | 100 | func newRecordSequence() *recordSequence { 101 | return &recordSequence{set: make(map[cid.Cid]struct{})} 102 | } 103 | 104 | func (s *recordSequence) Store(rec linkedRecord) { 105 | // verify if record is already contained in some fragment 106 | if _, found := s.set[rec.Cid()]; found { 107 | return 108 | } 109 | s.set[rec.Cid()] = struct{}{} 110 | 111 | // now try to find a sequence to be attached to 112 | for i, fragment := range s.fragments { 113 | if fragment[len(fragment)-1].Cid() == rec.PrevID() { 114 | s.fragments[i] = append(fragment, rec) 115 | return 116 | } else if fragment[0].PrevID() == rec.Cid() { 117 | s.fragments[i] = append([]linkedRecord{rec}, fragment...) 118 | return 119 | } 120 | } 121 | 122 | // start a new fragment 123 | s.fragments = append(s.fragments, []linkedRecord{rec}) 124 | } 125 | 126 | // return reconstructed sequence and success flag 127 | func (s *recordSequence) List() ([]linkedRecord, bool) { 128 | LOOP: 129 | // avoid recursion as sequences could be pretty large 130 | for { 131 | if len(s.fragments) == 1 { 132 | return s.fragments[0], true 133 | } 134 | 135 | // take a fragment ... 136 | fragment := s.fragments[0] 137 | fHead, fTail := fragment[len(fragment)-1], fragment[0] 138 | 139 | // ... and try to compose it with another one 140 | for i, candidate := range s.fragments[1:] { 141 | cHead, cTail := candidate[len(candidate)-1], candidate[0] 142 | // index shifted by slicing 143 | i += 1 144 | 145 | if fHead.Cid() == cTail.PrevID() { 146 | // composition: (tail) <- fragment <- candidate <- (head) 147 | s.fragments[0] = append(fragment, candidate...) 148 | s.fragments = append(s.fragments[:i], s.fragments[i+1:]...) 149 | continue LOOP 150 | 151 | } else if fTail.PrevID() == cHead.Cid() { 152 | // composition: (tail) <- candidate <- fragment <- (head) 153 | s.fragments[i] = append(candidate, fragment...) 154 | s.fragments = s.fragments[1:] 155 | continue LOOP 156 | } 157 | } 158 | 159 | // no composition found, hence there are at least two disjoint fragments 160 | return nil, false 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /net/record_test.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/ipfs/go-cid" 9 | util "github.com/ipfs/go-ipfs-util" 10 | "golang.org/x/exp/rand" 11 | ) 12 | 13 | type mockRecord struct { 14 | id, prevID cid.Cid 15 | } 16 | 17 | func (r mockRecord) Cid() cid.Cid { 18 | return r.id 19 | } 20 | 21 | func (r mockRecord) PrevID() cid.Cid { 22 | return r.prevID 23 | } 24 | 25 | func TestNet_RecordSequenceConsistent(t *testing.T) { 26 | seqLen := 100 27 | seq := generateSequence(cid.Undef, seqLen) 28 | recs := newRecordSequence() 29 | 30 | // emulate random order with 2x redundancy 31 | seqCpy := make([]linkedRecord, 2*len(seq)) 32 | copy(seqCpy[:len(seq)], seq) 33 | copy(seqCpy[len(seq):], seq) 34 | rand.Shuffle(len(seqCpy), func(i, j int) { seqCpy[i], seqCpy[j] = seqCpy[j], seqCpy[i] }) 35 | 36 | for _, rec := range seqCpy { 37 | recs.Store(rec) 38 | } 39 | 40 | collected, ok := recs.List() 41 | if !ok { 42 | t.Error("cannot reconstruct consistent record sequence") 43 | } 44 | 45 | if !equalSequences(collected, seq) { 46 | t.Errorf("reconstructed sequence doesn't match original one\noriginal: %s\nrestored: %s", 47 | formatSequence(seq), formatSequence(collected)) 48 | } 49 | } 50 | 51 | func TestNet_RecordSequenceGaps(t *testing.T) { 52 | seqLen := 100 53 | seq := generateSequence(cid.Undef, seqLen) 54 | recs := newRecordSequence() 55 | 56 | // two disjoint subsequences with a single missing record between 57 | s1 := seq[:seqLen/2] 58 | s2 := seq[seqLen/2+1:] 59 | 60 | for _, rec := range s1 { 61 | recs.Store(rec) 62 | } 63 | 64 | for _, rec := range s2 { 65 | recs.Store(rec) 66 | } 67 | 68 | if _, ok := recs.List(); ok { 69 | t.Errorf("record sequence with gaps unexpectedly reconstructed") 70 | } 71 | } 72 | 73 | func generateSequence(from cid.Cid, size int) []linkedRecord { 74 | var ( 75 | prev = from 76 | seq = make([]linkedRecord, size) 77 | ) 78 | 79 | for i := 0; i < size; i++ { 80 | rec := generateRecord([]byte(fmt.Sprintf("record:%d", i)), prev) 81 | seq[i] = rec 82 | prev = rec.Cid() 83 | } 84 | 85 | return seq 86 | } 87 | 88 | func generateRecord(data []byte, prev cid.Cid) linkedRecord { 89 | mh := util.Hash(data) 90 | id := cid.NewCidV0(mh) 91 | return mockRecord{id: id, prevID: prev} 92 | } 93 | 94 | func equalSequences(s1, s2 []linkedRecord) bool { 95 | if len(s1) != len(s2) { 96 | return false 97 | } 98 | 99 | for i := 0; i < len(s1); i++ { 100 | if s1[i].Cid() != s2[i].Cid() || s1[i].PrevID() != s2[i].PrevID() { 101 | return false 102 | } 103 | } 104 | 105 | return true 106 | } 107 | 108 | func formatSequence(seq []linkedRecord) string { 109 | var ( 110 | recs = make([]string, len(seq)) 111 | formatCID = func(id cid.Cid) string { 112 | if id == cid.Undef { 113 | return "Undef" 114 | } 115 | ir := id.String() 116 | return fmt.Sprintf("...%s", ir[len(ir)-6:]) 117 | } 118 | ) 119 | 120 | for i, rec := range seq { 121 | recs[i] = fmt.Sprintf("(Prev: %s, ID: %s)", formatCID(rec.PrevID()), formatCID(rec.Cid())) 122 | } 123 | 124 | return strings.Join(recs, " -> ") 125 | } 126 | -------------------------------------------------------------------------------- /net/util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "sync" 5 | 6 | apipb "github.com/textileio/go-threads/net/api/pb" 7 | netpb "github.com/textileio/go-threads/net/pb" 8 | ) 9 | 10 | func RecFromServiceRec(r *netpb.Log_Record) *apipb.Record { 11 | return &apipb.Record{ 12 | RecordNode: r.RecordNode, 13 | EventNode: r.EventNode, 14 | HeaderNode: r.HeaderNode, 15 | BodyNode: r.BodyNode, 16 | } 17 | } 18 | 19 | func RecToServiceRec(r *apipb.Record) *netpb.Log_Record { 20 | return &netpb.Log_Record{ 21 | RecordNode: r.RecordNode, 22 | EventNode: r.EventNode, 23 | HeaderNode: r.HeaderNode, 24 | BodyNode: r.BodyNode, 25 | } 26 | } 27 | 28 | func NewSemaphore(capacity int) *Semaphore { 29 | return &Semaphore{inner: make(chan struct{}, capacity)} 30 | } 31 | 32 | type Semaphore struct { 33 | inner chan struct{} 34 | } 35 | 36 | // Blocking acquire 37 | func (s *Semaphore) Acquire() { 38 | s.inner <- struct{}{} 39 | } 40 | 41 | // Non-blocking acquire 42 | func (s *Semaphore) TryAcquire() bool { 43 | select { 44 | case s.inner <- struct{}{}: 45 | return true 46 | default: 47 | return false 48 | } 49 | } 50 | 51 | func (s *Semaphore) Release() { 52 | select { 53 | case <-s.inner: 54 | default: 55 | panic("thread semaphore inconsistency: release before acquire!") 56 | } 57 | } 58 | 59 | type SemaphoreKey interface { 60 | Key() string 61 | } 62 | 63 | func NewSemaphorePool(semaCap int) *SemaphorePool { 64 | return &SemaphorePool{ss: make(map[string]*Semaphore), semaCap: semaCap} 65 | } 66 | 67 | type SemaphorePool struct { 68 | ss map[string]*Semaphore 69 | semaCap int 70 | mu sync.Mutex 71 | } 72 | 73 | func (p *SemaphorePool) Get(k SemaphoreKey) *Semaphore { 74 | var ( 75 | s *Semaphore 76 | exist bool 77 | key = k.Key() 78 | ) 79 | 80 | p.mu.Lock() 81 | if s, exist = p.ss[key]; !exist { 82 | s = NewSemaphore(p.semaCap) 83 | p.ss[key] = s 84 | } 85 | p.mu.Unlock() 86 | 87 | return s 88 | } 89 | 90 | func (p *SemaphorePool) Stop() { 91 | p.mu.Lock() 92 | defer p.mu.Unlock() 93 | 94 | // grab all semaphores and hold 95 | for _, s := range p.ss { 96 | s.Acquire() 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /test/benchmarks_suite.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sort" 7 | "testing" 8 | 9 | pstore "github.com/libp2p/go-libp2p-core/peerstore" 10 | core "github.com/textileio/go-threads/core/logstore" 11 | "github.com/textileio/go-threads/core/thread" 12 | ) 13 | 14 | var threadstoreBenchmarks = map[string]func(core.Logstore, chan *logpair) func(*testing.B){ 15 | "AddAddrs": benchmarkAddAddrs, 16 | "SetAddrs": benchmarkSetAddrs, 17 | "GetAddrs": benchmarkGetAddrs, 18 | // The in-between get allows us to benchmark the read-through cache. 19 | "AddGetAndClearAddrs": benchmarkAddGetAndClearAddrs, 20 | // Calls LogsWithAddr on a threadstore with 1000 logs. 21 | "Get1000LogsWithAddrs": benchmarkGet1000LogsWithAddrs, 22 | } 23 | 24 | func BenchmarkLogstore(b *testing.B, factory LogstoreFactory, variant string) { 25 | // Parameterises benchmarks to tackle logs with 1, 10, 100 multiaddrs. 26 | params := []struct { 27 | n int 28 | ch chan *logpair 29 | }{ 30 | {1, make(chan *logpair, 100)}, 31 | {10, make(chan *logpair, 100)}, 32 | {100, make(chan *logpair, 100)}, 33 | } 34 | 35 | ctx, cancel := context.WithCancel(context.Background()) 36 | defer cancel() 37 | 38 | // Start all test log producing goroutines, where each produces logs with as many 39 | // multiaddrs as the n field in the param struct. 40 | for _, p := range params { 41 | go AddressProducer(ctx, b, p.ch, p.n) 42 | } 43 | 44 | // So tests are always run in the same order. 45 | ordernames := make([]string, 0, len(threadstoreBenchmarks)) 46 | for name := range threadstoreBenchmarks { 47 | ordernames = append(ordernames, name) 48 | } 49 | sort.Strings(ordernames) 50 | 51 | for _, name := range ordernames { 52 | bench := threadstoreBenchmarks[name] 53 | for _, p := range params { 54 | // Create a new threadstore. 55 | ts, closeFunc := factory() 56 | 57 | // Run the test. 58 | b.Run(fmt.Sprintf("%s-%dAddrs-%s", name, p.n, variant), bench(ts, p.ch)) 59 | 60 | // Cleanup. 61 | if closeFunc != nil { 62 | closeFunc() 63 | } 64 | } 65 | } 66 | } 67 | 68 | func benchmarkAddAddrs(ls core.Logstore, addrs chan *logpair) func(*testing.B) { 69 | return func(b *testing.B) { 70 | tid := thread.NewIDV1(thread.Raw, 24) 71 | b.ResetTimer() 72 | for i := 0; i < b.N; i++ { 73 | pp := <-addrs 74 | _ = ls.AddAddrs(tid, pp.ID, pp.Addr, pstore.PermanentAddrTTL) 75 | } 76 | } 77 | } 78 | 79 | func benchmarkSetAddrs(ls core.Logstore, addrs chan *logpair) func(*testing.B) { 80 | return func(b *testing.B) { 81 | tid := thread.NewIDV1(thread.Raw, 24) 82 | b.ResetTimer() 83 | for i := 0; i < b.N; i++ { 84 | pp := <-addrs 85 | _ = ls.SetAddrs(tid, pp.ID, pp.Addr, pstore.PermanentAddrTTL) 86 | } 87 | } 88 | } 89 | 90 | func benchmarkGetAddrs(ls core.Logstore, addrs chan *logpair) func(*testing.B) { 91 | return func(b *testing.B) { 92 | tid := thread.NewIDV1(thread.Raw, 24) 93 | pp := <-addrs 94 | _ = ls.SetAddrs(tid, pp.ID, pp.Addr, pstore.PermanentAddrTTL) 95 | 96 | b.ResetTimer() 97 | for i := 0; i < b.N; i++ { 98 | _, _ = ls.Addrs(tid, pp.ID) 99 | } 100 | } 101 | } 102 | 103 | func benchmarkAddGetAndClearAddrs(ls core.Logstore, addrs chan *logpair) func(*testing.B) { 104 | return func(b *testing.B) { 105 | tid := thread.NewIDV1(thread.Raw, 24) 106 | b.ResetTimer() 107 | for i := 0; i < b.N; i++ { 108 | pp := <-addrs 109 | _ = ls.AddAddrs(tid, pp.ID, pp.Addr, pstore.PermanentAddrTTL) 110 | _, _ = ls.Addrs(tid, pp.ID) 111 | _ = ls.ClearAddrs(tid, pp.ID) 112 | } 113 | } 114 | } 115 | 116 | func benchmarkGet1000LogsWithAddrs(ls core.Logstore, addrs chan *logpair) func(*testing.B) { 117 | return func(b *testing.B) { 118 | tid := thread.NewIDV1(thread.Raw, 24) 119 | var logs = make([]*logpair, 1000) 120 | for i := range logs { 121 | pp := <-addrs 122 | _ = ls.AddAddrs(tid, pp.ID, pp.Addr, pstore.PermanentAddrTTL) 123 | logs[i] = pp 124 | } 125 | 126 | b.ResetTimer() 127 | for i := 0; i < b.N; i++ { 128 | _, _ = ls.LogsWithAddrs(tid) 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /test/util.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/libp2p/go-libp2p-core/peer" 9 | pt "github.com/libp2p/go-libp2p-core/test" 10 | ma "github.com/multiformats/go-multiaddr" 11 | ) 12 | 13 | func Multiaddr(m string) ma.Multiaddr { 14 | maddr, err := ma.NewMultiaddr(m) 15 | if err != nil { 16 | panic(err) 17 | } 18 | return maddr 19 | } 20 | 21 | type logpair struct { 22 | ID peer.ID 23 | Addr []ma.Multiaddr 24 | } 25 | 26 | func RandomPeer(b *testing.B, addrCount int) *logpair { 27 | var ( 28 | pid peer.ID 29 | err error 30 | addrs = make([]ma.Multiaddr, addrCount) 31 | aFmt = "/ip4/127.0.0.1/tcp/%d/ipfs/%s" 32 | ) 33 | 34 | b.Helper() 35 | if pid, err = pt.RandPeerID(); err != nil { 36 | b.Fatal(err) 37 | } 38 | 39 | for i := 0; i < addrCount; i++ { 40 | if addrs[i], err = ma.NewMultiaddr(fmt.Sprintf(aFmt, i, pid.Pretty())); err != nil { 41 | b.Fatal(err) 42 | } 43 | } 44 | return &logpair{pid, addrs} 45 | } 46 | 47 | func AddressProducer(ctx context.Context, b *testing.B, addrs chan *logpair, addrsPerPeer int) { 48 | b.Helper() 49 | defer close(addrs) 50 | for { 51 | p := RandomPeer(b, addrsPerPeer) 52 | select { 53 | case addrs <- p: 54 | case <-ctx.Done(): 55 | return 56 | } 57 | } 58 | } 59 | 60 | func GenerateAddrs(count int) []ma.Multiaddr { 61 | var addrs = make([]ma.Multiaddr, count) 62 | for i := 0; i < count; i++ { 63 | addrs[i] = Multiaddr(fmt.Sprintf("/ip4/1.1.1.%d/tcp/1111", i)) 64 | } 65 | return addrs 66 | } 67 | 68 | func GeneratePeerIDs(count int) []peer.ID { 69 | var ids = make([]peer.ID, count) 70 | for i := 0; i < count; i++ { 71 | ids[i], _ = pt.RandPeerID() 72 | } 73 | return ids 74 | } 75 | 76 | func AssertAddressesEqual(t *testing.T, exp, act []ma.Multiaddr) { 77 | t.Helper() 78 | if len(exp) != len(act) { 79 | t.Fatalf("lengths not the same. expected %d, got %d\n", len(exp), len(act)) 80 | } 81 | 82 | for _, a := range exp { 83 | found := false 84 | 85 | for _, b := range act { 86 | if a.Equal(b) { 87 | found = true 88 | break 89 | } 90 | } 91 | 92 | if !found { 93 | t.Fatalf("expected address %s not found", a) 94 | } 95 | } 96 | } 97 | 98 | func check(t *testing.T, err error) { 99 | if err != nil { 100 | t.Fatalf("unexpected error: %v", err) 101 | } 102 | } 103 | --------------------------------------------------------------------------------