├── deploy ├── .gitignore ├── cloud │ ├── kubernetes │ │ ├── .gitignore │ │ └── config │ │ │ └── grafana │ │ │ ├── datasource.prometheus.json │ │ │ ├── README.md │ │ │ ├── post-start.sh │ │ │ └── gen.go │ └── terraform │ │ ├── minikube │ │ ├── terraform.tfvars │ │ └── variables.tf │ │ └── gcp │ │ ├── main.template.tf │ │ ├── terraform.tfvars │ │ ├── module │ │ ├── README.md │ │ ├── main.tf │ │ └── variables.tf │ │ └── variables.tf └── Dockerfile ├── libri ├── acceptance │ ├── .gitignore │ ├── get-test-data.sh │ └── public-testnet.md ├── cmd │ ├── banners │ │ ├── author.template.txt │ │ └── librarian.template.txt │ ├── version_test.go │ ├── librarian.go │ ├── banners_test.go │ ├── version.go │ ├── health.go │ ├── health_test.go │ ├── root.go │ ├── test.go │ ├── banners.go │ ├── download.go │ ├── io.go │ └── author_test.go ├── main.go ├── author │ ├── keychain │ │ ├── keychain.proto │ │ ├── storage_test.go │ │ ├── keychain.pb.go │ │ ├── keychain_test.go │ │ └── storage.go │ ├── io │ │ ├── common │ │ │ ├── testing_test.go │ │ │ └── testing.go │ │ ├── pack │ │ │ ├── envelope.go │ │ │ └── envelope_test.go │ │ ├── enc │ │ │ ├── enc_test.go │ │ │ ├── enc.go │ │ │ ├── benchmarks_test.go │ │ │ ├── mac.go │ │ │ └── metadata.go │ │ ├── page │ │ │ └── storage.go │ │ ├── ship │ │ │ └── shipper.go │ │ └── print │ │ │ └── scanner.go │ ├── lifecycle.go │ ├── helpers.go │ ├── config_test.go │ ├── helpers_test.go │ └── logging.go ├── librarian │ ├── server │ │ ├── comm │ │ │ ├── know_test.go │ │ │ ├── know.go │ │ │ ├── outcomes_test.go │ │ │ ├── prefer.go │ │ │ ├── doctor.go │ │ │ ├── prefer_test.go │ │ │ └── doctor_test.go │ │ ├── peer │ │ │ ├── storage.go │ │ │ ├── storage_test.go │ │ │ └── testing.go │ │ ├── storage │ │ │ └── storage.proto │ │ ├── storage_test.go │ │ ├── storage.go │ │ ├── replicate │ │ │ ├── metrics_test.go │ │ │ └── metrics.go │ │ ├── requests.go │ │ ├── routing │ │ │ ├── bucket_test.go │ │ │ ├── storage.go │ │ │ ├── balancer.go │ │ │ └── benchmarks_test.go │ │ ├── requests_test.go │ │ └── metrics_test.go │ ├── client │ │ ├── testing.go │ │ ├── creators_test.go │ │ ├── creators.go │ │ ├── requests.go │ │ ├── context.go │ │ ├── context_test.go │ │ ├── pool.go │ │ └── pool_test.go │ └── api │ │ ├── librarian_test.go │ │ ├── metadata.go │ │ ├── subscription.go │ │ ├── metadata_test.go │ │ ├── testing.go │ │ └── librarian.go └── common │ ├── ecid │ ├── storage.proto │ ├── storage_test.go │ ├── storage.go │ └── storage.pb.go │ ├── parse │ ├── parse.go │ └── parse_test.go │ ├── logging │ ├── logging_test.go │ └── logging.go │ ├── errors │ ├── errors_test.go │ └── errors.go │ ├── subscribe │ ├── filter.go │ ├── subscription_test.go │ ├── filter_test.go │ └── subscription.go │ └── storage │ ├── testing_test.go │ ├── storer_test.go │ ├── checker_test.go │ ├── checker.go │ └── testing.go ├── .gitignore ├── scripts ├── run-build-container.sh ├── install-git-hooks.sh ├── git-hooks │ └── pre-push ├── test-cover.sh ├── build-static.sh ├── minikube-test.sh ├── run-author-benchmarks.sh └── stress-test.sh ├── .gometalinter.json ├── Gopkg.toml ├── version └── version.go ├── CONTRIBUTING.md ├── Makefile └── .circleci └── config.yml /deploy/.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | clusters/ 3 | -------------------------------------------------------------------------------- /deploy/cloud/kubernetes/.gitignore: -------------------------------------------------------------------------------- 1 | libri.yml 2 | -------------------------------------------------------------------------------- /libri/acceptance/.gitignore: -------------------------------------------------------------------------------- 1 | data 2 | logs 3 | -------------------------------------------------------------------------------- /libri/cmd/banners/author.template.txt: -------------------------------------------------------------------------------- 1 | Libri Author Client v{{ .LibriVersion }} 2 | 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | 3 | # created during CI 4 | artifacts 5 | 6 | # Intellij files 7 | .idea/ 8 | *.iml 9 | -------------------------------------------------------------------------------- /libri/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/drausin/libri/libri/cmd" 5 | ) 6 | 7 | func main() { 8 | cmd.Execute() 9 | } 10 | -------------------------------------------------------------------------------- /libri/cmd/version_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import "testing" 4 | 5 | func TestVersionCmd(t *testing.T) { 6 | versionCmd.Run(versionCmd, []string{}) 7 | } 8 | -------------------------------------------------------------------------------- /libri/author/keychain/keychain.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package keychain; 4 | 5 | message StoredKeychain { 6 | repeated bytes privateKeys = 1; 7 | } 8 | 9 | -------------------------------------------------------------------------------- /deploy/cloud/kubernetes/config/grafana/datasource.prometheus.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "prometheus", 3 | "type": "prometheus", 4 | "url": "http://prometheus.default.svc.cluster.local:9090", 5 | "access": "proxy", 6 | "basicAuth": false 7 | } 8 | -------------------------------------------------------------------------------- /deploy/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.7 2 | 3 | RUN apk update && \ 4 | apk add bash bash-completion util-linux coreutils findutils grep e2fsprogs-extra 5 | 6 | RUN mkdir /data 7 | ADD bin/* /usr/local/bin/ 8 | 9 | ENTRYPOINT ["libri"] 10 | -------------------------------------------------------------------------------- /scripts/run-build-container.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash -eou pipefail 2 | 3 | docker run --rm -it \ 4 | -h libri-build \ 5 | -v ~/.go/src:/go/src \ 6 | -v ~/.bashrc:/root/.bashrc \ 7 | -v ~/.gitconfig:/root/.gitconfig \ 8 | daedalus2718/libri-build:latest 9 | -------------------------------------------------------------------------------- /libri/acceptance/get-test-data.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eou pipefail 4 | 5 | DATA_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/data" 6 | mkdir -p ${DATA_DIR} 7 | pushd ${DATA_DIR} 8 | 9 | wget --quiet https://www.whatsapp.com/security/WhatsApp-Security-Whitepaper.pdf 10 | 11 | popd -------------------------------------------------------------------------------- /scripts/install-git-hooks.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eou pipefail 4 | 5 | GIT_HOOKS_DIR="scripts/git-hooks" 6 | 7 | for filepath in ${GIT_HOOKS_DIR}/* ; do 8 | filename=$(basename ${filepath}) 9 | cp ${filepath} .git/hooks/${filename} 10 | echo "installed ${filename}" 11 | done 12 | -------------------------------------------------------------------------------- /deploy/cloud/kubernetes/config/grafana/README.md: -------------------------------------------------------------------------------- 1 | 2 | The service dashboard config file is templated to facilitate easy changes. See 3 | `dashboard.service.json.template`. Regenerate `dashboard.service.json` via 4 | 5 | go run gen.go 6 | 7 | Since we don't expect `dashboard.service.json` to change too much, it's committed to the repo. -------------------------------------------------------------------------------- /libri/cmd/librarian.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | ) 6 | 7 | // librarianCmd represents the librarian command 8 | var librarianCmd = &cobra.Command{ 9 | Use: "librarian", 10 | Short: "operate a librarian server, a peer in the Libri network", 11 | } 12 | 13 | func init() { 14 | RootCmd.AddCommand(librarianCmd) 15 | } 16 | -------------------------------------------------------------------------------- /libri/author/io/common/testing_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "github.com/magiconair/properties/assert" 8 | ) 9 | 10 | func TestNewCompressableBytes(t *testing.T) { 11 | rng := rand.New(rand.NewSource(0)) 12 | nBytes := 128 13 | buf := NewCompressableBytes(rng, nBytes) 14 | assert.Equal(t, nBytes, buf.Len()) 15 | } 16 | -------------------------------------------------------------------------------- /scripts/git-hooks/pre-push: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eou pipefail 4 | 5 | # check no outstanding formatting issues 6 | make fix 7 | if [[ $(git status --porcelain | wc -l) -ne 0 ]]; then 8 | echo 'Found and resolved formatting issues. Please examine and commit, perhaps via' 9 | echo 10 | echo ' git commit -a --amend' 11 | echo 12 | exit 1 13 | fi 14 | -------------------------------------------------------------------------------- /libri/cmd/banners_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | // these test basically just ensure there are no panics, usually b/c of mismatched tempalate and 9 | // config variables 10 | 11 | func TestWriteLibrarianBanner(t *testing.T) { 12 | WriteLibrarianBanner(os.Stdout) 13 | } 14 | 15 | func TestWriteAuthorBanner(t *testing.T) { 16 | WriteAuthorBanner(os.Stdout) 17 | } 18 | -------------------------------------------------------------------------------- /libri/librarian/server/comm/know_test.go: -------------------------------------------------------------------------------- 1 | package comm 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "github.com/drausin/libri/libri/common/id" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestAlwaysKnower_Know(t *testing.T) { 12 | k := NewAlwaysKnower() 13 | rng := rand.New(rand.NewSource(0)) 14 | peerID := id.NewPseudoRandom(rng) 15 | 16 | assert.True(t, k.Know(peerID)) 17 | } 18 | -------------------------------------------------------------------------------- /libri/common/ecid/storage.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package ecid; 4 | 5 | // ECDSAPrivateKey represents an ECDSA key-pair, whose public key x-value is used as the peer ID 6 | // to outside world. 7 | message ECDSAPrivateKey { 8 | // name of the curve used 9 | string curve = 1; 10 | 11 | // private key 12 | bytes D = 2; 13 | 14 | // x-value of public key 15 | bytes X = 3; 16 | 17 | // y-value of public key 18 | bytes Y = 4; 19 | } 20 | -------------------------------------------------------------------------------- /libri/cmd/banners/librarian.template.txt: -------------------------------------------------------------------------------- 1 | 2 | ██╗ ██╗ ██████╗ ██████╗ ██╗ 3 | ██║ ██║ ██╔══██╗ ██╔══██╗ ██║ 4 | ██║ ██║ ██████╔╝ ██████╔╝ ██║ 5 | ██║ ██║ ██╔══██╗ ██╔══██╗ ██║ 6 | ███████╗ ██║ ██████╔╝ ██║ ██║ ██║ 7 | ╚══════╝ ╚═╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ 8 | 9 | Libri Librarian Server 10 | 11 | Libri version {{ .LibriVersion }} 12 | Go version: {{ .GoVersion }} 13 | GOOS: {{ .GoOS }} 14 | GOARCH: {{ .GoArch }} 15 | NumCPU: {{ .NumCPU }} 16 | 17 | -------------------------------------------------------------------------------- /libri/librarian/server/comm/know.go: -------------------------------------------------------------------------------- 1 | package comm 2 | 3 | import "github.com/drausin/libri/libri/common/id" 4 | 5 | // Knower defines which peers are known, and thus usually more trustworthy, versus unknown. 6 | type Knower interface { 7 | 8 | // Know returns whether a peer is known. 9 | Know(peerID id.ID) bool 10 | } 11 | 12 | // NewAlwaysKnower returns a Knower than treats all peers as known. 13 | func NewAlwaysKnower() Knower { 14 | return &alwaysKnower{} 15 | } 16 | 17 | type alwaysKnower struct{} 18 | 19 | func (k *alwaysKnower) Know(peerID id.ID) bool { 20 | return true 21 | } 22 | -------------------------------------------------------------------------------- /deploy/cloud/terraform/minikube/terraform.tfvars: -------------------------------------------------------------------------------- 1 | cluster_host = "minikube" 2 | cluster_admin_user = "admin" 3 | 4 | # librarians 5 | num_librarians = 4 6 | librarian_libri_version = "snapshot" 7 | librarian_cpu_limit = "50m" 8 | librarian_ram_limit = "300M" 9 | librarian_disk_size_gb = 1 10 | 11 | librarian_public_port_start = 30100 12 | librarian_local_port = 20100 13 | librarian_local_metrics_port = 20200 14 | 15 | # monitoring 16 | grafana_port = 30300 17 | prometheus_port = 30090 18 | grafana_ram_limit = "100M" 19 | prometheus_ram_limit = "250M" 20 | grafana_cpu_limit = "250m" 21 | prometheus_cpu_limit = "250m" 22 | -------------------------------------------------------------------------------- /libri/cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/drausin/libri/libri/common/errors" 7 | "github.com/drausin/libri/version" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | // versionCmd represents the librarian command 12 | var versionCmd = &cobra.Command{ 13 | Use: "version", 14 | Short: "print the libri version", 15 | Long: "print the libri version", 16 | Run: func(cmd *cobra.Command, args []string) { 17 | _, err := os.Stdout.WriteString(version.Current.Version.String() + "\n") 18 | errors.MaybePanic(err) 19 | }, 20 | } 21 | 22 | func init() { 23 | RootCmd.AddCommand(versionCmd) 24 | } 25 | -------------------------------------------------------------------------------- /libri/author/lifecycle.go: -------------------------------------------------------------------------------- 1 | package author 2 | 3 | import "os" 4 | 5 | // Close disconnects the author from its librarians and closes the DB. 6 | func (a *Author) Close() error { 7 | // send stop signal to listener 8 | a.stop <- struct{}{} 9 | 10 | // disconnect from librarians 11 | if err := a.librarians.CloseAll(); err != nil { 12 | return err 13 | } 14 | 15 | // close the DB 16 | a.db.Close() 17 | 18 | return nil 19 | } 20 | 21 | // CloseAndRemove closes the author and removes all local state. 22 | func (a *Author) CloseAndRemove() error { 23 | if err := a.Close(); err != nil { 24 | return err 25 | } 26 | return os.RemoveAll(a.config.DataDir) 27 | } 28 | -------------------------------------------------------------------------------- /scripts/test-cover.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eou pipefail 4 | 5 | # this will only work on CircleCI/in build container 6 | OUT_DIR="artifacts/cover" 7 | cd /go/src/github.com/drausin/libri 8 | mkdir -p ${OUT_DIR} 9 | 10 | PKGS=$(go list ./... | grep -v /vendor/) 11 | echo ${PKGS} | sed 's| |\n|g' | xargs -I {} bash -c ' 12 | COVER_FILE=artifacts/cover/$(echo {} | sed -r "s|github.com/drausin/libri/||g" | sed "s|/|-|g").cov && 13 | go test -race -coverprofile=${COVER_FILE} {} 14 | ' 15 | 16 | # merge profiles together, removing results from auto-generated code 17 | gocovmerge ${OUT_DIR}/*.cov | grep -v '.pb.go:' > ${OUT_DIR}/test-coverage-merged.cov 18 | -------------------------------------------------------------------------------- /.gometalinter.json: -------------------------------------------------------------------------------- 1 | { 2 | "Enable": [ 3 | "deadcode", 4 | "maligned", 5 | "errcheck", 6 | "gosec", 7 | "goconst", 8 | "gocyclo", 9 | "golint", 10 | "lll", 11 | "megacheck", 12 | "ineffassign", 13 | "interfacer", 14 | "misspell", 15 | "structcheck", 16 | "unconvert", 17 | "varcheck", 18 | "vet", 19 | "vetshadow" 20 | ], 21 | "VendoredLinters": true, 22 | "Vendor": true, 23 | "Concurrency": 4, 24 | "LineLength": 100, 25 | "Format": "({{.Linter}}): {{.Severity}}: {{.Path}}:{{.Line}}:{{if .Col}}{{.Col}}{{end}}: {{.Message}}", 26 | "Exclude": [ 27 | ".+vendor/.+", 28 | ".+pb\\.go" 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /libri/librarian/client/testing.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/golang/protobuf/proto" 7 | ) 8 | 9 | // TestNoOpSigner implements the signature.Signer interface but just returns a dummy token. 10 | type TestNoOpSigner struct{} 11 | 12 | // Sign returns a dummy token. 13 | func (s *TestNoOpSigner) Sign(m proto.Message) (string, error) { 14 | return "noop.token.sig", nil 15 | } 16 | 17 | // TestErrSigner implements the signature.Signer interface but always returns an error. 18 | type TestErrSigner struct{} 19 | 20 | // Sign returns an error. 21 | func (s *TestErrSigner) Sign(m proto.Message) (string, error) { 22 | return "", errors.New("some sign error") 23 | } 24 | -------------------------------------------------------------------------------- /deploy/cloud/terraform/gcp/main.template.tf: -------------------------------------------------------------------------------- 1 | 2 | terraform { 3 | backend "gcs" { 4 | bucket = "{{ .Bucket }}" 5 | prefix = "{{ .ClusterName }}/terraform/current.tfstate" 6 | project = "{{ .GCPProject }}" 7 | } 8 | } 9 | 10 | module "{{ .ClusterName }}" { 11 | source = "{{ .LocalModulePath }}" 12 | credentials_file = "${var.credentials_file}" 13 | gcs_clusters_bucket = "{{ .Bucket }}" 14 | cluster_name = "{{ .ClusterName }}" 15 | gcp_project = "{{ .GCPProject }}" 16 | num_librarians = "${var.num_librarians}" 17 | num_cluster_nodes = "${var.num_cluster_nodes}" 18 | gce_node_machine_type = "${var.cluster_node_machine_type}" 19 | librarian_disk_type = "${var.librarian_disk_type}" 20 | } 21 | -------------------------------------------------------------------------------- /libri/cmd/health.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/spf13/cobra" 6 | ) 7 | 8 | var errFailedHealthcheck = errors.New("some or all librarians unhealthy") 9 | 10 | // healthCmd represents the health command 11 | var healthCmd = &cobra.Command{ 12 | Use: "health", 13 | Short: "check health of librarian peers", 14 | RunE: func(cmd *cobra.Command, args []string) error { 15 | author, _, err := newTestAuthorGetter().get() 16 | if err != nil { 17 | return err 18 | } 19 | if allHealthy, _ := author.Healthcheck(); !allHealthy { 20 | return errFailedHealthcheck 21 | } 22 | return nil 23 | }, 24 | } 25 | 26 | func init() { 27 | testCmd.AddCommand(healthCmd) 28 | } 29 | -------------------------------------------------------------------------------- /deploy/cloud/terraform/gcp/terraform.tfvars: -------------------------------------------------------------------------------- 1 | cluster_host = "gcp" 2 | 3 | # librarians 4 | num_librarians = 8 5 | librarian_libri_version = "snapshot" 6 | librarian_disk_size_gb = 10 7 | librarian_disk_type = "pd-standard" 8 | librarian_cpu_limit = "200m" 9 | librarian_ram_limit = "2G" 10 | 11 | librarian_public_port_start = 30100 12 | librarian_local_port = 20100 13 | librarian_local_metrics_port = 20200 14 | 15 | # monitoring 16 | grafana_port = 30300 17 | prometheus_port = 30090 18 | grafana_ram_limit = "250M" 19 | prometheus_ram_limit = "1G" 20 | grafana_cpu_limit = "100m" 21 | prometheus_cpu_limit = "250m" 22 | 23 | # Kubernetes cluster 24 | num_cluster_nodes = 2 25 | cluster_node_machine_type = "n1-highmem-2" # 2 CPUs, 6.5 GB RAM 26 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | 2 | # Gopkg.toml example 3 | # 4 | # Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md 5 | # for detailed Gopkg.toml documentation. 6 | # 7 | # required = ["github.com/user/thing/cmd/thing"] 8 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 9 | # 10 | # [[constraint]] 11 | # name = "github.com/user/project" 12 | # version = "1.0.0" 13 | # 14 | # [[constraint]] 15 | # name = "github.com/user/project2" 16 | # branch = "dev" 17 | # source = "github.com/myfork/project2" 18 | # 19 | # [[override]] 20 | # name = "github.com/x/y" 21 | # version = "2.4.0" 22 | 23 | 24 | [[constraint]] 25 | name = "github.com/tecbot/gorocksdb" 26 | revision = "214b6b7bc0f06812ab5602fdc502a3e619916f38" 27 | -------------------------------------------------------------------------------- /libri/librarian/server/comm/outcomes_test.go: -------------------------------------------------------------------------------- 1 | package comm 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestQueryType_String(t *testing.T) { 10 | assert.Equal(t, "REQUEST", Request.String()) 11 | assert.Equal(t, "RESPONSE", Response.String()) 12 | } 13 | 14 | func TestOutcome_String(t *testing.T) { 15 | assert.Equal(t, "SUCCESS", Success.String()) 16 | assert.Equal(t, "ERROR", Error.String()) 17 | } 18 | 19 | func TestMetrics_Record(t *testing.T) { 20 | m := newScalarMetrics() 21 | 22 | m.Record() 23 | assert.NotEmpty(t, m.Earliest) 24 | assert.Equal(t, m.Earliest, m.Latest) 25 | assert.Equal(t, uint64(1), m.Count) 26 | 27 | m.Record() 28 | assert.True(t, m.Earliest.Before(m.Latest)) 29 | assert.Equal(t, uint64(2), m.Count) 30 | } 31 | -------------------------------------------------------------------------------- /libri/author/io/pack/envelope.go: -------------------------------------------------------------------------------- 1 | package pack 2 | 3 | import ( 4 | "github.com/drausin/libri/libri/common/id" 5 | "github.com/drausin/libri/libri/librarian/api" 6 | ) 7 | 8 | // NewEnvelopeDoc returns a new envelope document for the given entry key and author and reader 9 | // public keys. 10 | func NewEnvelopeDoc( 11 | entryKey id.ID, 12 | authorPub []byte, 13 | readerPub []byte, 14 | eekCiphertext []byte, 15 | eekCiphertextMAC []byte, 16 | ) *api.Document { 17 | envelope := &api.Envelope{ 18 | AuthorPublicKey: authorPub, 19 | ReaderPublicKey: readerPub, 20 | EntryKey: entryKey.Bytes(), 21 | EekCiphertext: eekCiphertext, 22 | EekCiphertextMac: eekCiphertextMAC, 23 | } 24 | return &api.Document{ 25 | Contents: &api.Document_Envelope{ 26 | Envelope: envelope, 27 | }, 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /libri/librarian/api/librarian_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/magiconair/properties/assert" 7 | ) 8 | 9 | func TestEndpoint_String(t *testing.T) { 10 | cases := map[string]struct { 11 | value Endpoint 12 | expected string 13 | }{ 14 | "all": {value: All, expected: "All"}, 15 | "intro": {value: Introduce, expected: "Introduce"}, 16 | "find": {value: Find, expected: "Find"}, 17 | "store": {value: Store, expected: "Store"}, 18 | "verify": {value: Verify, expected: "Verify"}, 19 | "get": {value: Get, expected: "Get"}, 20 | "put": {value: Put, expected: "Put"}, 21 | "subscribe": {value: Subscribe, expected: "Subscribe"}, 22 | } 23 | for desc, c := range cases { 24 | assert.Equal(t, c.expected, c.value.String(), desc) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /libri/author/io/common/testing.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "bytes" 5 | "math/rand" 6 | 7 | "github.com/drausin/libri/libri/common/errors" 8 | ) 9 | 10 | // NewCompressableBytes generates a buffer of repeated strings with a given length. 11 | func NewCompressableBytes(rng *rand.Rand, size int) *bytes.Buffer { 12 | dict := []string{ 13 | "these", "are", "some", "test", "words", "that", "will", "be", "compressed", 14 | } 15 | words := new(bytes.Buffer) 16 | for { 17 | word := dict[int(rng.Int31n(int32(len(dict))))] + " " 18 | if words.Len()+len(word) > size { 19 | // pad words to exact length 20 | _, err := words.Write(make([]byte, size-words.Len())) 21 | errors.MaybePanic(err) 22 | break 23 | } 24 | _, err := words.WriteString(word) 25 | errors.MaybePanic(err) 26 | } 27 | 28 | return words 29 | } 30 | -------------------------------------------------------------------------------- /libri/common/parse/parse.go: -------------------------------------------------------------------------------- 1 | package parse 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | ) 7 | 8 | // Addr parses a net.TCPAddr from a host address and port. 9 | func Addr(host string, port int) (*net.TCPAddr, error) { 10 | return net.ResolveTCPAddr("tcp4", fmt.Sprintf("%s:%d", host, port)) 11 | } 12 | 13 | // Addrs parses an array of net.TCPAddrs from an array of IPv4:Port address strings. 14 | func Addrs(addrs []string) ([]*net.TCPAddr, error) { 15 | netAddrs := make([]*net.TCPAddr, 0, len(addrs)) 16 | nErrs := 0 17 | for _, a := range addrs { 18 | netAddr, err := net.ResolveTCPAddr("tcp4", a) 19 | if err != nil { 20 | nErrs++ 21 | if nErrs == len(addrs) { 22 | // bail if none of the addrs could be parsed 23 | return nil, err 24 | } 25 | continue 26 | } 27 | netAddrs = append(netAddrs, netAddr) 28 | } 29 | return netAddrs, nil 30 | } 31 | -------------------------------------------------------------------------------- /deploy/cloud/terraform/gcp/module/README.md: -------------------------------------------------------------------------------- 1 | ## Libri cluster in Google Cloud Platform 2 | 3 | This Terraform module defines all the infrastructure required to run a libri 4 | cluster on GCP, including 5 | 6 | - Google Container Engine (GKE) cluster for running Kubernetes 7 | - Persistent disks for each librarian in the cluster 8 | - firewall rule to expoose librarians to public internet 9 | 10 | 11 | ### Usage 12 | 13 | ``` 14 | module "libri-staging" { 15 | source = "/path/to/local/module" 16 | 17 | # these are required 18 | credentials_file = "my-gcp-credentials.json" 19 | gcs_clusters_bucket = "my-libri-clusters" 20 | cluster_name = "libri-staging" 21 | gcp_project = "libri-12345" 22 | 23 | # these, among others, are optional and are commonly set in separate 24 | # properties.tfvars file 25 | num_librarians = 4 26 | num_cluster_nodes = 2 27 | gce_node_machine_type = "n1-standard-1" 28 | } 29 | ``` 30 | -------------------------------------------------------------------------------- /libri/librarian/server/peer/storage.go: -------------------------------------------------------------------------------- 1 | package peer 2 | 3 | import ( 4 | "net" 5 | 6 | "github.com/drausin/libri/libri/common/id" 7 | "github.com/drausin/libri/libri/librarian/server/storage" 8 | ) 9 | 10 | // FromStored creates a new peer.Peer instance from a storage.Peer instance. 11 | func FromStored(stored *storage.Peer) Peer { 12 | return New( 13 | id.FromBytes(stored.Id), 14 | stored.Name, 15 | fromStoredAddress(stored.PublicAddress), 16 | ) 17 | } 18 | 19 | // fromStoredAddress creates a net.TCPAddr from a storage.Address. 20 | func fromStoredAddress(stored *storage.Address) *net.TCPAddr { 21 | return &net.TCPAddr{ 22 | IP: net.ParseIP(stored.Ip), 23 | Port: int(stored.Port), 24 | } 25 | } 26 | 27 | // toStoredAddress creates a storage.Address from a net.TCPAddr. 28 | func toStoredAddress(address *net.TCPAddr) *storage.Address { 29 | return &storage.Address{ 30 | Ip: address.IP.String(), 31 | Port: uint32(address.Port), 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /libri/cmd/health_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | "github.com/spf13/viper" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | // TestHealthCmd_ok : this path is annoying to test b/c it involves lots of setup; but this is 13 | // tested in acceptance/local-demo.sh, so it's ok to skip here 14 | 15 | func TestHealthCmd_err(t *testing.T) { 16 | dataDir, err := ioutil.TempDir("", "test-author-data-dir") 17 | defer func() { err = os.RemoveAll(dataDir) }() 18 | viper.Set(dataDirFlag, dataDir) 19 | 20 | // check newTestAuthorGetter() error bubbles up 21 | viper.Set(testLibrariansFlag, "bad librarians address") 22 | err = healthCmd.RunE(healthCmd, []string{}) 23 | assert.NotNil(t, err) 24 | 25 | // check Healthcheck() error from ok but missing librarians bubbles up 26 | viper.Set(testLibrariansFlag, "localhost:20200 localhost:20201") 27 | err = healthCmd.RunE(healthCmd, []string{}) 28 | assert.NotNil(t, err) 29 | } 30 | -------------------------------------------------------------------------------- /scripts/build-static.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eou pipefail 4 | 5 | # Build a static binary for use in Docker container. This script is mainly intended for CircleCI 6 | # builds and does't at the moment work locally on OSX. 7 | # 8 | # Usage: 9 | # 10 | # ./build-static path/to/output/binary 11 | # 12 | # where "path/to/output/binary" is the path to write the output binary to. 13 | # 14 | 15 | OUTPUT_FILE=${1} 16 | 17 | VERSION_PKG="github.com/drausin/libri/version" 18 | GIT_BRANCH_VAR="${VERSION_PKG}.GitBranch=$(git symbolic-ref -q --short HEAD)" 19 | GIT_REVISION_VAR="${VERSION_PKG}.GitRevision=$(git rev-parse --short HEAD)" 20 | BUILD_DATE_VAR="${VERSION_PKG}.BuildDate=$(date -u +"%Y-%m-%d")" 21 | VERSION_VARS="-X ${GIT_BRANCH_VAR} -X ${GIT_REVISION_VAR} -X ${BUILD_DATE_VAR}" 22 | 23 | GOOS=linux go build \ 24 | -ldflags "-extldflags '-lpthread -static' ${VERSION_VARS}" \ 25 | -a \ 26 | -installsuffix cgo \ 27 | -o ${OUTPUT_FILE} \ 28 | libri/main.go 29 | -------------------------------------------------------------------------------- /libri/author/io/pack/envelope_test.go: -------------------------------------------------------------------------------- 1 | package pack 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "github.com/drausin/libri/libri/author/io/enc" 8 | "github.com/drausin/libri/libri/common/id" 9 | "github.com/drausin/libri/libri/librarian/api" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestNewEnvelopeDoc(t *testing.T) { 14 | rng := rand.New(rand.NewSource(0)) 15 | _, authorPub, readerPub := enc.NewPseudoRandomKEK(rng) 16 | ciphertext := api.RandBytes(rng, api.EEKLength) 17 | ciphertextMAC := api.RandBytes(rng, api.HMAC256Length) 18 | 19 | entryKey := id.NewPseudoRandom(rng) 20 | docEnvelope := NewEnvelopeDoc(entryKey, authorPub, readerPub, ciphertext, ciphertextMAC) 21 | envelope := docEnvelope.Contents.(*api.Document_Envelope).Envelope 22 | assert.Equal(t, authorPub, envelope.AuthorPublicKey) 23 | assert.Equal(t, readerPub, envelope.ReaderPublicKey) 24 | assert.Equal(t, entryKey.Bytes(), envelope.EntryKey) 25 | assert.Equal(t, ciphertext, envelope.EekCiphertext) 26 | assert.Equal(t, ciphertextMAC, envelope.EekCiphertextMac) 27 | } 28 | -------------------------------------------------------------------------------- /libri/librarian/server/peer/storage_test.go: -------------------------------------------------------------------------------- 1 | package peer 2 | 3 | import ( 4 | "math/rand" 5 | "net" 6 | "testing" 7 | 8 | "github.com/drausin/libri/libri/librarian/server/storage" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestFromStored(t *testing.T) { 13 | sp := NewTestStoredPeer(rand.New(rand.NewSource(0)), 0) 14 | p := FromStored(sp) 15 | AssertPeersEqual(t, sp, p) 16 | } 17 | 18 | func TestToStored(t *testing.T) { 19 | p := NewTestPeer(rand.New(rand.NewSource(0)), 0) 20 | sp := p.ToStored() 21 | AssertPeersEqual(t, sp, p) 22 | } 23 | 24 | func TestFromStoredAddress(t *testing.T) { 25 | ip, port := "192.168.1.1", uint32(1000) 26 | sa := &storage.Address{Ip: ip, Port: port} 27 | a := fromStoredAddress(sa) 28 | assert.Equal(t, ip, a.IP.String()) 29 | assert.Equal(t, int(port), a.Port) 30 | } 31 | 32 | func TestToStoredAddress(t *testing.T) { 33 | ip, port := "192.168.1.1", 1000 34 | a := &net.TCPAddr{IP: net.ParseIP(ip), Port: port} 35 | sa := toStoredAddress(a) 36 | assert.Equal(t, ip, sa.Ip) 37 | assert.Equal(t, uint32(port), sa.Port) 38 | } 39 | -------------------------------------------------------------------------------- /scripts/minikube-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eou pipefail 4 | 5 | TEST_IMAGE="daedalus2718/libri:snapshot" 6 | docker pull ${TEST_IMAGE} 7 | 8 | pushd deploy/cloud 9 | CLUSTERS_DIR="clusters" 10 | mkdir -p ${CLUSTERS_DIR} 11 | CLUSTER_DIR="${CLUSTERS_DIR}/rc" 12 | 13 | go run cluster.go init minikube --clusterDir "${CLUSTER_DIR}" --clusterName "rc" 14 | 15 | echo -e "\ncreated test cluster in ${CLUSTER_DIR}; ssh into minikube and pull Docker image if needed [press any key to continue]" 16 | read 17 | 18 | go run cluster.go apply --clusterDir ${CLUSTER_DIR} 19 | 20 | echo -e '\nin separate shell, check that all pods came up successfully [press any key to continue]' 21 | read 22 | 23 | minikube_ip=$(minikube ip) 24 | librarian_addrs="${minikube_ip}:30100,${minikube_ip}:30101,${minikube_ip}:30102,${minikube_ip}:30103" 25 | 26 | docker run --rm ${TEST_IMAGE} test health -a "${librarian_addrs}" 27 | docker run --rm ${TEST_IMAGE} test io -a "${librarian_addrs}" 28 | 29 | echo -e '\neverything looks good! press any key to tear down the cluster and clean up' 30 | read 31 | kubectl delete -f ${CLUSTER_DIR}/libri.yml 32 | rm -r ${CLUSTER_DIR} 33 | popd 34 | -------------------------------------------------------------------------------- /libri/librarian/server/comm/prefer.go: -------------------------------------------------------------------------------- 1 | package comm 2 | 3 | import ( 4 | "github.com/drausin/libri/libri/common/id" 5 | "github.com/drausin/libri/libri/librarian/api" 6 | ) 7 | 8 | // Preferer judges whether one peer is preferable over another. 9 | type Preferer interface { 10 | 11 | // Prefer indicates whether peer 1 should be preferred over peer 2 when prioritization 12 | // is necessary. 13 | Prefer(peerID1, peerID2 id.ID) bool 14 | } 15 | 16 | // NewRpPreferer returns a Preferer that prefers peers with a larger number of successful 17 | // Verify or Find responses. 18 | func NewRpPreferer(getter QueryGetter) Preferer { 19 | return &rpPreferer{getter} 20 | } 21 | 22 | type rpPreferer struct { 23 | getter QueryGetter 24 | } 25 | 26 | func (p *rpPreferer) Prefer(peerID1, peerID2 id.ID) bool { 27 | nRps1 := p.getter.Get(peerID1, api.Verify)[Response][Success].Count 28 | nRps2 := p.getter.Get(peerID2, api.Verify)[Response][Success].Count 29 | if nRps1 == 0 || nRps2 == 0 { 30 | nRps1 = p.getter.Get(peerID1, api.Find)[Response][Success].Count 31 | nRps2 = p.getter.Get(peerID2, api.Find)[Response][Success].Count 32 | } 33 | return nRps1 > nRps2 34 | } 35 | -------------------------------------------------------------------------------- /libri/common/logging/logging_test.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "testing" 5 | 6 | "fmt" 7 | 8 | "github.com/pkg/errors" 9 | "github.com/stretchr/testify/assert" 10 | "go.uber.org/zap" 11 | "go.uber.org/zap/zapcore" 12 | ) 13 | 14 | func TestNewDevLogger(t *testing.T) { 15 | l := NewDevLogger(zap.DebugLevel) 16 | assert.NotNil(t, l) 17 | } 18 | 19 | func TestNewDevInfoLogger(t *testing.T) { 20 | l := NewDevInfoLogger() 21 | assert.NotNil(t, l) 22 | } 23 | 24 | func TestNewProdLogger(t *testing.T) { 25 | l := NewProdLogger(zap.InfoLevel) 26 | assert.NotNil(t, l) 27 | } 28 | 29 | func TestToErrArray(t *testing.T) { 30 | nErrs := 3 31 | errMap := make(map[string]error) 32 | for i := 0; i < nErrs; i++ { 33 | err := fmt.Errorf("error %d", i) 34 | errMap[string(i)] = err 35 | } 36 | assert.Equal(t, nErrs, len(ToErrArray(errMap))) 37 | } 38 | 39 | func TestErrArray_MarshalLogArray(t *testing.T) { 40 | errs := ErrArray{errors.New("error 1"), errors.New("error 2"), errors.New("error 3")} 41 | oe := zapcore.NewJSONEncoder(zap.NewDevelopmentEncoderConfig()).(zapcore.ArrayEncoder) 42 | err := errs.MarshalLogArray(oe) 43 | assert.Nil(t, err) 44 | } 45 | -------------------------------------------------------------------------------- /deploy/cloud/kubernetes/config/grafana/post-start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | DIR="$( cd "$( dirname "${0}" )" && pwd )" 4 | GRAFANA_URL=${1} 5 | DATASOURCES_URL="${GRAFANA_URL}/api/datasources" 6 | DASHBOARDS_URL="${GRAFANA_URL}/api/dashboards/import" 7 | 8 | echo -n "waiting for grafana to start ... " 9 | until $(curl --silent --fail --show-error --output /dev/null ${DATASOURCES_URL}); do 10 | sleep 1 11 | done 12 | echo "done" 13 | 14 | for file in ${DIR}/datasource.*.json ; do 15 | echo "importing datasource from ${file}" 16 | curl --silent --fail --show-error \ 17 | --request POST "${DATASOURCES_URL}" \ 18 | --header "Content-Type: application/json" \ 19 | --data-binary "@${file}" 20 | echo 21 | done 22 | 23 | for file in ${DIR}/dashboard.*.json ; do 24 | echo "importing dashboard from ${file}" 25 | (echo '{"dashboard":';cat "${file}";echo ',"inputs":[{"name":"DS_PROMETHEUS","pluginId":"prometheus","type":"datasource","value":"prometheus"}]}') | \ 26 | curl --silent --fail --show-error \ 27 | --request POST ${DASHBOARDS_URL} \ 28 | --header "Content-Type: application/json" \ 29 | --data-binary @-; 30 | echo 31 | done 32 | -------------------------------------------------------------------------------- /libri/common/ecid/storage_test.go: -------------------------------------------------------------------------------- 1 | package ecid 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestToFromStored_ok(t *testing.T) { 11 | rng := rand.New(rand.NewSource(0)) 12 | for c := 0; c < 10; c++ { 13 | original := NewPseudoRandom(rng) 14 | stored := ToStored(original) 15 | retrieved, err := FromStored(stored) 16 | 17 | assert.Nil(t, err) 18 | assert.Equal(t, original.Bytes(), retrieved.Bytes()) 19 | assert.Equal(t, original.Key().D.Bytes(), retrieved.Key().D.Bytes()) 20 | } 21 | } 22 | 23 | func TestToFromStored_curveErr(t *testing.T) { 24 | rng := rand.New(rand.NewSource(0)) 25 | original := NewPseudoRandom(rng) 26 | stored := ToStored(original) 27 | stored.Curve = "other curve" 28 | retrieved, err := FromStored(stored) 29 | 30 | assert.NotNil(t, err) 31 | assert.Nil(t, retrieved) 32 | } 33 | 34 | func TestToFromStored_pubKeyErr(t *testing.T) { 35 | rng := rand.New(rand.NewSource(0)) 36 | original := NewPseudoRandom(rng) 37 | stored := ToStored(original) 38 | stored.X = []byte("the wrong X coord") 39 | retrieved, err := FromStored(stored) 40 | 41 | assert.NotNil(t, err) 42 | assert.Nil(t, retrieved) 43 | } 44 | -------------------------------------------------------------------------------- /deploy/cloud/kubernetes/config/grafana/gen.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "text/template" 8 | 9 | "github.com/drausin/libri/libri/common/errors" 10 | ) 11 | 12 | const ( 13 | templateFilepath = "dashboard.service.json.template" 14 | outputFilepath = "dashboard.service.json" 15 | ) 16 | 17 | // Config defines the values used in the service template. 18 | type Config struct { 19 | Endpoints []string 20 | } 21 | 22 | var config = Config{ 23 | Endpoints: []string{"Get", "Put", "Find", "Store", "Verify"}, 24 | } 25 | 26 | func main() { 27 | wd, err := os.Getwd() 28 | errors.MaybePanic(err) 29 | absTemplateFilepath := filepath.Join(wd, templateFilepath) 30 | 31 | funcs := template.FuncMap{ 32 | "timesplus": func(x, y, z int) int { return x*y + z }, 33 | } 34 | tmpl, err := template.New(templateFilepath). 35 | Funcs(funcs). 36 | Delims("[[", "]]"). 37 | ParseFiles(absTemplateFilepath) 38 | errors.MaybePanic(err) 39 | 40 | absOutFilepath := filepath.Join(wd, outputFilepath) 41 | out, err := os.Create(absOutFilepath) 42 | errors.MaybePanic(err) 43 | err = tmpl.Execute(out, config) 44 | errors.MaybePanic(err) 45 | fmt.Printf("wrote %s\n", outputFilepath) 46 | } 47 | -------------------------------------------------------------------------------- /libri/author/keychain/storage_test.go: -------------------------------------------------------------------------------- 1 | package keychain 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | const ( 10 | veryLightScryptN = 2 11 | veryLightScryptP = 1 12 | ) 13 | 14 | func TestToFromStored_ok(t *testing.T) { 15 | nKeys := 3 16 | kc1 := New(nKeys) 17 | 18 | auth2 := "test passphrase" 19 | stored1, err := encryptToStored(kc1, auth2, veryLightScryptN, veryLightScryptP) 20 | assert.Nil(t, err) 21 | assert.Equal(t, nKeys, len(stored1.PrivateKeys)) 22 | 23 | kc2, err := decryptFromStored(stored1, auth2) 24 | assert.Nil(t, err) 25 | assert.Equal(t, kc1, kc2) 26 | 27 | auth3 := "a different test passphrase" 28 | stored2, err := encryptToStored(kc1, auth3, veryLightScryptN, veryLightScryptP) 29 | assert.Nil(t, err) 30 | assert.Equal(t, nKeys, len(stored2.PrivateKeys)) 31 | assert.NotEqual(t, stored1, stored2) 32 | 33 | kc3, err := decryptFromStored(stored2, auth3) 34 | assert.Nil(t, err) 35 | assert.Equal(t, kc1, kc3) 36 | } 37 | 38 | func TestToFromStored_err(t *testing.T) { 39 | nKeys := 3 40 | kc1 := New(nKeys) 41 | 42 | stored1, err := encryptToStored(kc1, "test passphrase", veryLightScryptN, veryLightScryptP) 43 | assert.Nil(t, err) 44 | 45 | kc2, err := decryptFromStored(stored1, "wrong passphrase") 46 | assert.NotNil(t, err) 47 | assert.Nil(t, kc2) 48 | } 49 | -------------------------------------------------------------------------------- /libri/common/errors/errors_test.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/pkg/errors" 7 | "github.com/stretchr/testify/assert" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | func TestMaybePanic(t *testing.T) { 12 | MaybePanic(nil) 13 | assert.Panics(t, func() { 14 | MaybePanic(errors.New("some error")) 15 | }) 16 | } 17 | 18 | func TestMonitorRunningErrorCount(t *testing.T) { 19 | errs := make(chan error, 8) 20 | fatal := make(chan error) 21 | maxRunningErrRate := float32(0.1) 22 | queueSize := 100 23 | maxRunningErrCount := int(float32(maxRunningErrRate) * float32(queueSize)) 24 | lg := zap.NewNop() 25 | 26 | go MonitorRunningErrors(errs, fatal, queueSize, maxRunningErrRate, lg) 27 | 28 | // check get fatal error when go over threshold 29 | for c := 0; c < maxRunningErrCount; c++ { 30 | errs <- errors.New("some To error") 31 | } 32 | fataErr := <-fatal 33 | assert.Equal(t, ErrTooManyErrs, fataErr) 34 | 35 | go MonitorRunningErrors(errs, fatal, queueSize, maxRunningErrRate, lg) 36 | 37 | // check don't get fatal error when below threshold 38 | for c := 0; c < 200; c++ { 39 | var err error 40 | if c%25 == 0 { 41 | err = errors.New("some To error") 42 | } 43 | errs <- err 44 | } 45 | 46 | var fatalErr error 47 | select { 48 | case fatalErr = <-fatal: 49 | default: 50 | } 51 | assert.Nil(t, fatalErr) 52 | } 53 | -------------------------------------------------------------------------------- /libri/cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/drausin/libri/libri/common/errors" 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | "go.uber.org/zap" 11 | "go.uber.org/zap/zapcore" 12 | ) 13 | 14 | const ( 15 | dataDirFlag = "dataDir" 16 | logLevelFlag = "logLevel" 17 | envVarPrefix = "LIBRI" 18 | ) 19 | 20 | // RootCmd represents the base command when called without any subcommands 21 | var RootCmd = &cobra.Command{ 22 | Use: "libri", 23 | Short: "execute Libri client and server commands", 24 | } 25 | 26 | // Execute is the main entrypoint for the libri CLI. 27 | func Execute() { 28 | if err := RootCmd.Execute(); err != nil { 29 | fmt.Println(err) 30 | os.Exit(1) 31 | } 32 | } 33 | 34 | func init() { 35 | RootCmd.PersistentFlags().StringP(dataDirFlag, "d", "", 36 | "local data directory") 37 | RootCmd.PersistentFlags().StringP(logLevelFlag, "l", zap.InfoLevel.String(), 38 | "log level") 39 | 40 | // bind viper flags 41 | viper.SetEnvPrefix("LIBRI") // look for env vars with "LIBRI_" prefix 42 | viper.AutomaticEnv() // read in environment variables that match 43 | errors.MaybePanic(viper.BindPFlags(RootCmd.PersistentFlags())) 44 | } 45 | 46 | func getLogLevel() zapcore.Level { 47 | var ll zapcore.Level 48 | errors.MaybePanic(ll.Set(viper.GetString(logLevelFlag))) 49 | return ll 50 | } 51 | -------------------------------------------------------------------------------- /libri/common/ecid/storage.go: -------------------------------------------------------------------------------- 1 | package ecid 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | "fmt" 6 | "math/big" 7 | 8 | "github.com/drausin/libri/libri/common/id" 9 | "github.com/ethereum/go-ethereum/crypto/secp256k1" 10 | ) 11 | 12 | // FromStored creates a new ID instance from a ECID instance. 13 | func FromStored(stored *ECDSAPrivateKey) (ID, error) { 14 | key := new(ecdsa.PrivateKey) 15 | 16 | switch stored.Curve { 17 | case "secp256k1": 18 | key.PublicKey.Curve = secp256k1.S256() 19 | default: 20 | return nil, fmt.Errorf("unrecognized curve %v", stored.Curve) 21 | } 22 | 23 | key.PublicKey.X = new(big.Int).SetBytes(stored.X) 24 | key.PublicKey.Y = new(big.Int).SetBytes(stored.Y) 25 | key.D = new(big.Int).SetBytes(stored.D) 26 | 27 | if !key.Curve.IsOnCurve(key.PublicKey.X, key.PublicKey.Y) { 28 | // redundancy check: should never hit this, but here just in case 29 | return nil, fmt.Errorf("public key (x = %v, y = %v) is not on curve %v", 30 | key.PublicKey.X, key.PublicKey.Y, key.PublicKey.Curve.Params().Name) 31 | } 32 | 33 | return &ecid{ 34 | key: key, 35 | id: id.FromInt(key.X), 36 | }, nil 37 | } 38 | 39 | // ToStored creates a new ECID instance from an ID instance. 40 | func ToStored(ecid ID) *ECDSAPrivateKey { 41 | key := ecid.Key() 42 | return &ECDSAPrivateKey{ 43 | Curve: CurveName, 44 | X: key.X.Bytes(), 45 | Y: key.Y.Bytes(), 46 | D: key.D.Bytes(), 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /libri/common/subscribe/filter.go: -------------------------------------------------------------------------------- 1 | package subscribe 2 | 3 | import ( 4 | "encoding/gob" 5 | "math/rand" 6 | 7 | "github.com/drausin/libri/libri/librarian/api" 8 | "github.com/willf/bloom" 9 | ) 10 | 11 | var minFilterElements = 16 12 | 13 | // ToAPI converts a *bloom.BloomFilter (via narrower gob.GobEncoder) to an *api.BloomFilter. 14 | func ToAPI(f gob.GobEncoder) (*api.BloomFilter, error) { 15 | encoded, err := f.GobEncode() 16 | if err != nil { 17 | // should never happen 18 | return nil, err 19 | } 20 | return &api.BloomFilter{ 21 | Encoded: encoded, 22 | }, nil 23 | } 24 | 25 | // FromAPI converts an *api.BloomFilter to a *bloom.BloomFilter. 26 | func FromAPI(f *api.BloomFilter) (*bloom.BloomFilter, error) { 27 | decoded := bloom.New(1, 1) 28 | err := decoded.GobDecode(f.Encoded) 29 | if err != nil { 30 | return nil, err 31 | } 32 | return decoded, nil 33 | } 34 | 35 | func newFilter(elements [][]byte, fp float64, rng *rand.Rand) *bloom.BloomFilter { 36 | if fp == 1.0 { 37 | return alwaysInFilter() 38 | } 39 | for len(elements) < minFilterElements { 40 | elements = append(elements, api.RandBytes(rng, api.ECPubKeyLength)) 41 | } 42 | filter := bloom.NewWithEstimates(uint(len(elements)), fp) 43 | for _, e := range elements { 44 | filter.Add(e) 45 | } 46 | return filter 47 | } 48 | 49 | func alwaysInFilter() *bloom.BloomFilter { 50 | filter := bloom.New(1, 1) 51 | filter.Add([]byte{1}) // could be anything 52 | return filter 53 | } 54 | -------------------------------------------------------------------------------- /libri/librarian/server/storage/storage.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package storage; 4 | 5 | // Address is a IPv4 address. 6 | message Address { 7 | // IP address 8 | string ip = 2; 9 | 10 | // TCP port 11 | uint32 port = 3; 12 | } 13 | 14 | // Peer is the basic information associated with each peer in the network. 15 | message Peer { 16 | // big-endian byte representation of 32-byte ID 17 | bytes id = 1; 18 | 19 | // self-reported name of the peer 20 | string name = 2; 21 | 22 | // public IP address 23 | Address public_address = 3; 24 | } 25 | 26 | // StoredRoutingTable contains the essential information associated with a routing table. 27 | message RoutingTable { 28 | // big-endian byte representation of 32-byte self ID 29 | bytes self_id = 1; 30 | 31 | // array of peers in table 32 | repeated Peer peers = 2; 33 | } 34 | 35 | message DocumentMetrics { 36 | uint64 n_documents = 1; 37 | 38 | uint64 total_size = 2; 39 | } 40 | 41 | message ReplicationMetrics { 42 | 43 | // n_verified is the total number of verified documents (fully or partial) 44 | uint64 n_verified = 1; 45 | 46 | // n_underreplicated is the number of under-repliced documents found 47 | uint64 n_underreplicated = 2; 48 | 49 | // n_replicated is the number of documents successfully replicated 50 | uint64 n_replicated = 3; 51 | 52 | // latest_pass is the epoch time (in seconds) since the last full replication 53 | int64 latest_pass = 4; 54 | } -------------------------------------------------------------------------------- /libri/common/errors/errors.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "errors" 5 | 6 | "go.uber.org/zap" 7 | ) 8 | 9 | // ErrTooManyErrs indicates when too many errors have occurred. 10 | var ErrTooManyErrs = errors.New("too many errors") 11 | 12 | // MaybePanic panics if the argument is not nil. It is useful for wrapping error-only return 13 | // functions known to only return nil values. 14 | func MaybePanic(err error) { 15 | if err != nil { 16 | panic(err) 17 | } 18 | } 19 | 20 | // MonitorRunningErrors reads (possibly nil) errors from errs and sends an error to fatal if 21 | // the running non-nil error rate gets above maxRunningErrRate. 22 | func MonitorRunningErrors( 23 | errs chan error, fatal chan error, queueSize int, maxRunningErrRate float32, logger *zap.Logger, 24 | ) { 25 | maxRunningErrCount := int(maxRunningErrRate * float32(queueSize)) 26 | 27 | // fill error queue with non-errors 28 | runningErrs := make(chan error, queueSize) 29 | for c := 0; c < queueSize; c++ { 30 | runningErrs <- nil 31 | } 32 | 33 | // consume from errs and keep running error count; send fatal error if ever above threshold 34 | runningNErrs := 0 35 | for latestErr := range errs { 36 | if latestErr != nil { 37 | runningNErrs++ 38 | logger.Info("received non-fatal error", zap.Error(latestErr)) 39 | if runningNErrs >= maxRunningErrCount { 40 | fatal <- ErrTooManyErrs 41 | return 42 | } 43 | } 44 | if earliestErr := <-runningErrs; earliestErr != nil { 45 | runningNErrs-- 46 | } 47 | runningErrs <- latestErr 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /libri/librarian/server/storage_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "errors" 5 | "math/rand" 6 | "testing" 7 | 8 | "github.com/drausin/libri/libri/common/ecid" 9 | clogging "github.com/drausin/libri/libri/common/logging" 10 | cstorage "github.com/drausin/libri/libri/common/storage" 11 | "github.com/golang/protobuf/proto" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func TestLoadOrCreatePeerID_ok(t *testing.T) { 16 | 17 | // create new peer ID 18 | id1, err := loadOrCreatePeerID(clogging.NewDevInfoLogger(), &cstorage.TestSLD{}) 19 | assert.NotNil(t, id1) 20 | assert.Nil(t, err) 21 | 22 | // load existing 23 | rng := rand.New(rand.NewSource(0)) 24 | peerID2 := ecid.NewPseudoRandom(rng) 25 | bytes, err := proto.Marshal(ecid.ToStored(peerID2)) 26 | assert.Nil(t, err) 27 | 28 | id2, err := loadOrCreatePeerID(clogging.NewDevInfoLogger(), &cstorage.TestSLD{Bytes: bytes}) 29 | 30 | assert.Equal(t, peerID2, id2) 31 | assert.Nil(t, err) 32 | } 33 | 34 | func TestLoadOrCreatePeerID_err(t *testing.T) { 35 | id1, err := loadOrCreatePeerID(clogging.NewDevInfoLogger(), &cstorage.TestSLD{ 36 | LoadErr: errors.New("some load error"), 37 | }) 38 | assert.Nil(t, id1) 39 | assert.NotNil(t, err) 40 | 41 | id2, err := loadOrCreatePeerID(clogging.NewDevInfoLogger(), &cstorage.TestSLD{ 42 | Bytes: []byte("the wrong bytes"), 43 | }) 44 | assert.Nil(t, id2) 45 | assert.NotNil(t, err) 46 | } 47 | 48 | func TestSavePeerID(t *testing.T) { 49 | rng := rand.New(rand.NewSource(0)) 50 | assert.Nil(t, savePeerID(&cstorage.TestSLD{}, ecid.NewPseudoRandom(rng))) 51 | } 52 | -------------------------------------------------------------------------------- /scripts/run-author-benchmarks.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eou pipefail 4 | 5 | AUTHOR_BENCH_PKGS='github.com/drausin/libri/libri/author/io/enc 6 | github.com/drausin/libri/libri/author/io/comp 7 | github.com/drausin/libri/libri/author/io/page' 8 | N_TRIALS=16 9 | BENCH_DIR='artifacts/bench' 10 | RAW_RESULT_FILE="${BENCH_DIR}/author.raw.bench" 11 | RESULT_FILE="${BENCH_DIR}/author.bench" 12 | TEST_BINARY="pkg.test" 13 | 14 | FAST_BENCH_DURATION='0.1s' 15 | SLOW_BENCH_DURATION='3s' 16 | 17 | mkdir -p ${BENCH_DIR} 18 | rm -f ${RAW_RESULT_FILE} 19 | 20 | for pkg in ${AUTHOR_BENCH_PKGS}; do 21 | echo "benchmarking ${pkg}" 22 | echo -n "- compiling test binary ... " 23 | go test -c ${pkg} -o ${TEST_BINARY} 24 | echo "done" 25 | echo "- running benchmarks ${N_TRIALS} times ... " 26 | echo -n " - running fast benchmarks ... " 27 | ./${TEST_BINARY} -test.bench='.+/^(small|medium|large).*' -test.benchmem -test.cpu 4 \ 28 | -test.count ${N_TRIALS} -test.benchtime ${FAST_BENCH_DURATION} \ 29 | -test.run 'Benchmark*' 2>&1 >> ${RAW_RESULT_FILE} 30 | echo "done" 31 | echo -n " - running slow benchmarks ... " 32 | ./${TEST_BINARY} -test.bench='.+/^xlarge.*' -test.benchmem -test.cpu 4 \ 33 | -test.count ${N_TRIALS} -test.benchtime ${SLOW_BENCH_DURATION} \ 34 | -test.run 'Benchmark*' 2>&1 >> ${RAW_RESULT_FILE} 35 | rm ${TEST_BINARY} 36 | echo "done" 37 | echo -e "done\n" 38 | done 39 | 40 | grep -v PASS ${RAW_RESULT_FILE} > ${RESULT_FILE} 41 | rm ${RAW_RESULT_FILE} 42 | 43 | echo "benchmarking complete!" 44 | -------------------------------------------------------------------------------- /deploy/cloud/terraform/gcp/module/main.tf: -------------------------------------------------------------------------------- 1 | 2 | provider "google" { 3 | project = "${var.gcp_project}" 4 | region = "${var.gce_node_region}" 5 | credentials = "${file(var.credentials_file)}" 6 | version = "~> 1.10" 7 | } 8 | 9 | resource "google_container_cluster" "libri" { 10 | description = "libri cluster" 11 | name = "${var.cluster_name}" 12 | zone = "${var.gce_node_zone}" 13 | initial_node_count = "${var.num_cluster_nodes}" 14 | project = "${var.gcp_project}" 15 | 16 | master_auth { 17 | username = "admin" 18 | password = "demetriusijustdontknowwhywoahwoah" 19 | } 20 | 21 | node_config { 22 | machine_type = "${var.gce_node_machine_type}" 23 | disk_size_gb = "${var.node_disk_size_gb}" 24 | } 25 | 26 | provisioner "local-exec" { 27 | command = "gcloud container clusters get-credentials ${var.cluster_name} --zone ${var.gce_node_zone}" 28 | } 29 | } 30 | 31 | resource "google_compute_disk" "data-librarians" { 32 | count = "${var.num_librarians}" 33 | name = "data-librarians-${count.index}" 34 | type = "${var.librarian_disk_type}" 35 | zone = "${var.gce_node_zone}" 36 | size = "${var.librarian_disk_size_gb}" 37 | } 38 | 39 | resource "google_compute_firewall" "default" { 40 | description = "opens up ports for libri cluster communication to the outside world" 41 | name = "${var.cluster_name}" 42 | network = "default" 43 | 44 | allow { 45 | protocol = "tcp" 46 | ports = [ 47 | "${var.min_libri_port}-${var.min_libri_port + var.num_librarians - 1}", 48 | "30300", 49 | "30090", 50 | ] 51 | } 52 | 53 | source_ranges = ["0.0.0.0/0"] 54 | } 55 | -------------------------------------------------------------------------------- /libri/librarian/server/storage.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "github.com/drausin/libri/libri/common/ecid" 5 | "github.com/drausin/libri/libri/common/storage" 6 | "github.com/golang/protobuf/proto" 7 | "go.uber.org/zap" 8 | ) 9 | 10 | // logger keys 11 | const ( 12 | // LoggerPeerID is a peer ID. 13 | LoggerPeerID = "peerId" 14 | 15 | // NumPeers is a number of peers. 16 | NumPeers = "numPeers" 17 | 18 | // NumBuckets is a number of routing table buckets. 19 | NumBuckets = "numBuckets" 20 | ) 21 | 22 | var ( 23 | peerIDKey = []byte("PeerID") 24 | ) 25 | 26 | func loadOrCreatePeerID(logger *zap.Logger, nsl storage.StorerLoader) (ecid.ID, error) { 27 | bytes, err := nsl.Load(peerIDKey) 28 | if err != nil { 29 | logger.Error("error loading peer ID", zap.Error(err)) 30 | return nil, err 31 | } 32 | 33 | if bytes != nil { 34 | // return saved PeerID 35 | stored := &ecid.ECDSAPrivateKey{} 36 | if err := proto.Unmarshal(bytes, stored); err != nil { 37 | return nil, err 38 | } 39 | peerID, err := ecid.FromStored(stored) 40 | if err != nil { 41 | logger.Error("error deserializing peer ID keys", zap.Error(err)) 42 | return nil, err 43 | } 44 | logger.Info("loaded exsting peer ID", zap.String(LoggerPeerID, peerID.String())) 45 | return peerID, nil 46 | } 47 | 48 | // return new PeerID 49 | peerID := ecid.NewRandom() 50 | logger.Info("created new peer ID", zap.String(LoggerPeerID, peerID.String())) 51 | return peerID, savePeerID(nsl, peerID) 52 | } 53 | 54 | func savePeerID(ns storage.Storer, peerID ecid.ID) error { 55 | bytes, err := proto.Marshal(ecid.ToStored(peerID)) 56 | if err != nil { 57 | return err 58 | } 59 | return ns.Store(peerIDKey, bytes) 60 | } 61 | -------------------------------------------------------------------------------- /libri/librarian/server/replicate/metrics_test.go: -------------------------------------------------------------------------------- 1 | package replicate 2 | 3 | import ( 4 | "testing" 5 | 6 | prom "github.com/prometheus/client_golang/prometheus" 7 | dto "github.com/prometheus/client_model/go" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestMetrics_incVerification(t *testing.T) { 12 | m := newMetrics() 13 | m.register() 14 | defer m.unregister() 15 | 16 | // do some incs 17 | for _, r := range []result{succeeded, exhausted, errored} { 18 | for _, s := range []replicationStatus{full, under} { 19 | m.incVerification(r, s) 20 | } 21 | } 22 | 23 | // check that we have a single count for each verification result and status 24 | verMetrics := make(chan prom.Metric, 6) 25 | m.verification.Collect(verMetrics) 26 | close(verMetrics) 27 | c := 0 28 | for verMetric := range verMetrics { 29 | written := dto.Metric{} 30 | verMetric.Write(&written) 31 | assert.Equal(t, float64(1.0), *written.Counter.Value) 32 | assert.Equal(t, 2, len(written.Label)) 33 | c++ 34 | } 35 | assert.Equal(t, 6, c) 36 | } 37 | 38 | func TestMetrics_incReplication(t *testing.T) { 39 | m := newMetrics() 40 | m.register() 41 | defer m.unregister() 42 | 43 | // do some incs 44 | for _, r := range []result{succeeded, exhausted, errored} { 45 | m.incReplication(r) 46 | } 47 | 48 | // check we have a single count for each replication result 49 | repMetrics := make(chan prom.Metric, 3) 50 | m.replication.Collect(repMetrics) 51 | close(repMetrics) 52 | c := 0 53 | for repMetric := range repMetrics { 54 | written := dto.Metric{} 55 | repMetric.Write(&written) 56 | assert.Equal(t, float64(1.0), *written.Counter.Value) 57 | assert.Equal(t, 1, len(written.Label)) 58 | c++ 59 | } 60 | assert.Equal(t, 3, c) 61 | } 62 | -------------------------------------------------------------------------------- /libri/librarian/server/requests.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | "github.com/drausin/libri/libri/common/ecid" 8 | "github.com/drausin/libri/libri/common/id" 9 | "github.com/drausin/libri/libri/librarian/api" 10 | "github.com/drausin/libri/libri/librarian/client" 11 | "github.com/golang/protobuf/proto" 12 | "golang.org/x/net/context" 13 | ) 14 | 15 | // RequestVerifier verifies requests by checking the signature in the context. 16 | type RequestVerifier interface { 17 | Verify(ctx context.Context, msg proto.Message, meta *api.RequestMetadata) error 18 | } 19 | 20 | type verifier struct { 21 | sigVerifier client.Verifier 22 | } 23 | 24 | // NewRequestVerifier creates a new RequestVerifier instance. 25 | func NewRequestVerifier() RequestVerifier { 26 | return &verifier{ 27 | sigVerifier: client.NewVerifier(), 28 | } 29 | } 30 | 31 | func (rv *verifier) Verify( 32 | ctx context.Context, msg proto.Message, meta *api.RequestMetadata, 33 | ) error { 34 | encToken, encOrgToken, err := client.FromSignatureContext(ctx) 35 | if err != nil { 36 | return err 37 | } 38 | pubKey, err := ecid.FromPublicKeyBytes(meta.PubKey) 39 | if err != nil { 40 | return err 41 | } 42 | if meta.RequestId == nil { 43 | return errors.New("RequestId must not be nil") 44 | } 45 | if len(meta.RequestId) != id.Length { 46 | return fmt.Errorf("invalid RequestId length: %v; expected length %v", 47 | len(meta.RequestId), id.Length) 48 | } 49 | if err = rv.sigVerifier.Verify(encToken, pubKey, msg); err != nil || encOrgToken == "" { 50 | return err 51 | } 52 | orgPubKey, err := ecid.FromPublicKeyBytes(meta.OrgPubKey) 53 | if err != nil { 54 | return err 55 | } 56 | return rv.sigVerifier.Verify(encOrgToken, orgPubKey, msg) 57 | } 58 | -------------------------------------------------------------------------------- /libri/librarian/server/routing/bucket_test.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "container/heap" 5 | "math/rand" 6 | "testing" 7 | 8 | "github.com/drausin/libri/libri/librarian/api" 9 | "github.com/drausin/libri/libri/librarian/server/comm" 10 | "github.com/drausin/libri/libri/librarian/server/peer" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestBucket_PushPop(t *testing.T) { 15 | for n := 1; n <= 128; n *= 2 { 16 | rec := comm.NewQueryRecorderGetter(comm.NewAlwaysKnower()) 17 | preferer, doctor := comm.NewRpPreferer(rec), comm.NewNaiveDoctor() 18 | b := newFirstBucket(DefaultMaxActivePeers, preferer, doctor) 19 | rng := rand.New(rand.NewSource(int64(n))) 20 | for i, p := range peer.NewTestPeers(rng, n) { 21 | 22 | // simulate i successful responses from peer p so that heap ordering is well-defined 23 | for j := 0; j <= i; j++ { 24 | rec.Record(p.ID(), api.Verify, comm.Response, comm.Success) 25 | } 26 | heap.Push(b, p) 27 | } 28 | prev := heap.Pop(b).(peer.Peer) 29 | for b.Len() > 0 { 30 | cur := heap.Pop(b).(peer.Peer) 31 | assert.True(t, preferer.Prefer(cur.ID(), prev.ID())) 32 | prev = cur 33 | } 34 | } 35 | } 36 | 37 | func TestBucket_Peak(t *testing.T) { 38 | rec := comm.NewQueryRecorderGetter(comm.NewAlwaysKnower()) 39 | preferer, doctor := comm.NewRpPreferer(rec), comm.NewNaiveDoctor() 40 | b := newFirstBucket(DefaultMaxActivePeers, preferer, doctor) 41 | 42 | // nothing to peak b/c bucket is empty 43 | assert.Equal(t, 0, len(b.Peak(2))) 44 | 45 | // add some peers 46 | rng := rand.New(rand.NewSource(0)) 47 | for _, p := range peer.NewTestPeers(rng, 4) { 48 | heap.Push(b, p) 49 | } 50 | 51 | assert.Equal(t, 2, len(b.Peak(2))) 52 | assert.Equal(t, 4, len(b.Peak(4))) 53 | assert.Equal(t, 4, len(b.Peak(8))) 54 | } 55 | -------------------------------------------------------------------------------- /deploy/cloud/terraform/minikube/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "cluster_host" { 3 | description = "host of the cluster [gcp|minikube]" 4 | } 5 | 6 | variable "cluster_admin_user" { 7 | description = "k8s admin user creating the cluster" 8 | } 9 | 10 | variable "num_librarians" { 11 | description = "current number of librarian peers in cluster" 12 | } 13 | 14 | variable "librarian_disk_size_gb" { 15 | description = "size (GB) of persistant disk used by each librarian" 16 | } 17 | 18 | variable "librarian_libri_version" { 19 | description = "libri version (e.g., 0.1.0, latest, snapshot) to use for librarian container" 20 | } 21 | 22 | variable "librarian_cpu_limit" { 23 | description = "librarian container CPU limit (e.g., 500m, 0.5, 1)" 24 | } 25 | 26 | variable "librarian_ram_limit" { 27 | description = "librarian container RAM limit (e.g., 500M, 1G)" 28 | } 29 | 30 | variable "librarian_public_port_start" { 31 | description = "public port for librarian-0 service" 32 | } 33 | 34 | variable "librarian_local_port" { 35 | description = "local port for each librarian instance" 36 | } 37 | 38 | variable "librarian_local_metrics_port" { 39 | description = "local metrics port for each librarian instance" 40 | } 41 | 42 | variable "grafana_port" { 43 | description = "port for Grafana service" 44 | } 45 | 46 | variable "prometheus_port" { 47 | description = "port for Prometheus service" 48 | } 49 | 50 | variable "grafana_ram_limit" { 51 | description = "Grafana pod RAM limit (e.g., 500M, 1G)" 52 | } 53 | 54 | variable "grafana_cpu_limit" { 55 | description = "Grafana pod CPU limit (e.g., 500m, 0.5, 1)" 56 | } 57 | 58 | variable "prometheus_ram_limit" { 59 | description = "Prometheus pod RAM limit (e.g., 500M, 1G)" 60 | } 61 | 62 | variable "prometheus_cpu_limit" { 63 | description = "Prometheus pod CPU limit (e.g., 500m, 0.5, 1)" 64 | } 65 | -------------------------------------------------------------------------------- /libri/librarian/server/comm/doctor.go: -------------------------------------------------------------------------------- 1 | package comm 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/drausin/libri/libri/common/id" 7 | "github.com/drausin/libri/libri/librarian/api" 8 | ) 9 | 10 | // Doctor determines whether a peer is currently healthy. 11 | type Doctor interface { 12 | 13 | // Healthy indicates whether the peer is currently deemed healthy. 14 | Healthy(peerID id.ID) bool 15 | } 16 | 17 | type naiveDoctor struct{} 18 | 19 | // NewNaiveDoctor returns a Doctor that naively assumes all peers are healthy. 20 | func NewNaiveDoctor() Doctor { 21 | return &naiveDoctor{} 22 | } 23 | 24 | func (d *naiveDoctor) Healthy(peerID id.ID) bool { 25 | return true 26 | } 27 | 28 | // NewResponseTimeDoctor returns a Doctor that assumes peers are health if their latest successful 29 | // response is within a fixed window of the latest unsuccessful/errored response. 30 | func NewResponseTimeDoctor(recorder QueryGetter) Doctor { 31 | return &responseTimeDoctor{ 32 | recorder: recorder, 33 | } 34 | } 35 | 36 | type responseTimeDoctor struct { 37 | recorder QueryGetter 38 | } 39 | 40 | func (d *responseTimeDoctor) Healthy(peerID id.ID) bool { 41 | // Verify endpoint should most regularly be used, so just check that one for now 42 | verifyOutcomes := d.recorder.Get(peerID, api.Verify) 43 | latestErrTime := verifyOutcomes[Response][Error].Latest 44 | latestSuccessTime := verifyOutcomes[Response][Success].Latest 45 | 46 | if latestErrTime.IsZero() && latestSuccessTime.IsZero() { 47 | // fall back to Find if no Verifications 48 | verifyOutcomes = d.recorder.Get(peerID, api.Find) 49 | latestErrTime = verifyOutcomes[Response][Error].Latest 50 | latestSuccessTime = verifyOutcomes[Response][Success].Latest 51 | } 52 | 53 | // assume healthy if latest error time less than 5 mins after success time 54 | return latestErrTime.Before(latestSuccessTime.Add(5 * time.Minute)) 55 | } 56 | -------------------------------------------------------------------------------- /scripts/stress-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ou pipefail 4 | 5 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 6 | TEST_BINARY="pkg.test" 7 | OUTPUT_DIR="${DIR}/../artifacts/stress-test" 8 | ORIG_GOMAXPROCS=${GOMAXPROCS:-} 9 | N_TRIALS=16 10 | PKGS=$(go list ./... | grep -v /vendor/) 11 | 12 | rm -rf ${OUTPUT_DIR} ${TEST_BINARY} 13 | mkdir -p ${OUTPUT_DIR} 14 | 15 | # stress test indiv packages 16 | errored=false 17 | for pkg in ${PKGS}; do 18 | pkg_dir=$(echo ${pkg} | sed -r "s|github.com/drausin/libri/||g") 19 | pushd ${pkg_dir} > /dev/null 20 | echo "stress testing ${pkg}" 21 | echo -n "- compiling test binary ... " 22 | if [[ ${pkg} == github.com/drausin/libri/libri/acceptance ]]; then 23 | go test -c -tags acceptance -o ${TEST_BINARY} 24 | else 25 | go test -c -race -o ${TEST_BINARY} 26 | fi 27 | echo "done" 28 | pkg_base_name=$(echo ${pkg_dir} | sed "s|/|-|g") 29 | 30 | # run tests if test binary exists (i.e., package has tests in it) 31 | if [[ -f ${TEST_BINARY} ]]; then 32 | for c in $(seq 1 ${N_TRIALS}) ; do 33 | export GOMAXPROCS=$[ 1 + $[ RANDOM % 64 ]] 34 | LOG_FILE="${OUTPUT_DIR}/${pkg_base_name}.t${c}.p${GOMAXPROCS}.log" 35 | echo -ne "\r- trial ${c} of ${N_TRIALS} with GOMAXPROCS=${GOMAXPROCS} ... " 36 | ./${TEST_BINARY} -test.v &> ${LOG_FILE} 37 | if [[ $? -ne 0 ]]; then 38 | echo "ERROR found, please consult ${LOG_FILE}" 39 | errored=true 40 | fi 41 | done 42 | rm ${TEST_BINARY} 43 | echo -e "done\n" 44 | fi 45 | popd > /dev/null 46 | done 47 | 48 | export GOMAXPROCS=${ORIG_GOMAXPROCS} 49 | 50 | if [[ ${errored} = "true" ]]; then 51 | echo 'One or more stress tests had errors.' 52 | exit 1 53 | fi 54 | 55 | echo 'All stress tests passed.' 56 | 57 | -------------------------------------------------------------------------------- /libri/common/logging/logging.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "github.com/drausin/libri/libri/common/errors" 5 | "go.uber.org/zap" 6 | "go.uber.org/zap/zapcore" 7 | ) 8 | 9 | // GetLogLevel returns a new log level instance. 10 | func GetLogLevel(logLevelStr string) zapcore.Level { 11 | var logLevel zapcore.Level 12 | err := logLevel.Set(logLevelStr) 13 | errors.MaybePanic(err) 14 | return logLevel 15 | } 16 | 17 | // NewDevLogger creates a new logger with a given log level for use in development (i.e., not 18 | // production). 19 | func NewDevLogger(logLevel zapcore.Level) *zap.Logger { 20 | config := zap.NewDevelopmentConfig() 21 | config.DisableCaller = true 22 | config.Level.SetLevel(logLevel) 23 | 24 | logger, err := config.Build() 25 | errors.MaybePanic(err) 26 | return logger 27 | } 28 | 29 | // NewDevInfoLogger creates a new development logger at the INFO level. 30 | func NewDevInfoLogger() *zap.Logger { 31 | return NewDevLogger(zap.InfoLevel) 32 | } 33 | 34 | // NewProdLogger creates a new logger with a given log level for use in production. 35 | func NewProdLogger(logLevel zapcore.Level) *zap.Logger { 36 | config := zap.NewProductionConfig() 37 | config.Level.SetLevel(logLevel) 38 | 39 | logger, err := config.Build() 40 | errors.MaybePanic(err) 41 | return logger 42 | } 43 | 44 | // ErrArray is an array of errors 45 | type ErrArray []error 46 | 47 | // ToErrArray concerts a map of errors to an array of errors. 48 | func ToErrArray(errMap map[string]error) ErrArray { 49 | errArray := make([]error, len(errMap)) 50 | i := 0 51 | for _, err := range errMap { 52 | errArray[i] = err 53 | i++ 54 | } 55 | return errArray 56 | } 57 | 58 | // MarshalLogArray marshals the array of errors. 59 | func (errs ErrArray) MarshalLogArray(arr zapcore.ArrayEncoder) error { 60 | for _, err := range errs { 61 | arr.AppendString(err.Error()) 62 | } 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /libri/author/helpers.go: -------------------------------------------------------------------------------- 1 | package author 2 | 3 | import ( 4 | "net" 5 | 6 | "github.com/drausin/libri/libri/author/io/enc" 7 | "github.com/drausin/libri/libri/author/keychain" 8 | "google.golang.org/grpc" 9 | healthpb "google.golang.org/grpc/health/grpc_health_v1" 10 | ) 11 | 12 | type envelopeKeySampler interface { 13 | sample() ([]byte, []byte, *enc.KEK, *enc.EEK, error) 14 | } 15 | 16 | type envelopeKeySamplerImpl struct { 17 | authorKeys keychain.GetterSampler 18 | selfReaderKeys keychain.GetterSampler 19 | } 20 | 21 | // sample samples a random pair of keys (author and reader) for the author to use 22 | // in creating the document *Keys instance. The method returns the author and reader public keys 23 | // along with the *Keys object. 24 | func (s *envelopeKeySamplerImpl) sample() ([]byte, []byte, *enc.KEK, *enc.EEK, error) { 25 | authorID, err := s.authorKeys.Sample() 26 | if err != nil { 27 | return nil, nil, nil, nil, err 28 | } 29 | selfReaderID, err := s.selfReaderKeys.Sample() 30 | if err != nil { 31 | return nil, nil, nil, nil, err 32 | } 33 | kek, err := enc.NewKEK(authorID.Key(), &selfReaderID.Key().PublicKey) 34 | if err != nil { 35 | return nil, nil, nil, nil, err 36 | } 37 | eek, err := enc.NewEEK() 38 | if err != nil { 39 | return nil, nil, nil, nil, err 40 | } 41 | return authorID.PublicKeyBytes(), selfReaderID.PublicKeyBytes(), kek, eek, nil 42 | } 43 | 44 | // use var so it's easy to replace for tests w/o a single-method interface 45 | var getLibrarianHealthClients = func( 46 | librarianAddrs []*net.TCPAddr, 47 | ) (map[string]healthpb.HealthClient, error) { 48 | 49 | healthClients := make(map[string]healthpb.HealthClient) 50 | for _, librarianAddr := range librarianAddrs { 51 | addrStr := librarianAddr.String() 52 | conn, err := grpc.Dial(addrStr, grpc.WithInsecure()) 53 | if err != nil { 54 | return nil, err 55 | } 56 | healthClients[addrStr] = healthpb.NewHealthClient(conn) 57 | } 58 | return healthClients, nil 59 | } 60 | -------------------------------------------------------------------------------- /libri/cmd/test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/drausin/libri/libri/author" 5 | "github.com/drausin/libri/libri/author/keychain" 6 | "github.com/drausin/libri/libri/common/errors" 7 | "github.com/spf13/cobra" 8 | "github.com/spf13/viper" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | const ( 13 | testLibrariansFlag = "testLibrarians" 14 | ) 15 | 16 | // testCmd represents the test command 17 | var testCmd = &cobra.Command{ 18 | Use: "test", 19 | Short: "test a set of librarian servers", 20 | } 21 | 22 | func init() { 23 | RootCmd.AddCommand(testCmd) 24 | 25 | testCmd.PersistentFlags().StringSliceP(testLibrariansFlag, "a", nil, 26 | "comma-separated addresses (IPv4:Port) of librarian(s)") 27 | testCmd.PersistentFlags().Int(timeoutFlag, 10, 28 | "timeout (seconds) for requests to librarians") 29 | 30 | // bind viper flags 31 | viper.SetEnvPrefix(envVarPrefix) // look for env vars with "LIBRI_" prefix 32 | viper.AutomaticEnv() // read in environment variables that match 33 | errors.MaybePanic(viper.BindPFlags(testCmd.PersistentFlags())) 34 | } 35 | 36 | type testAuthorGetter interface { 37 | get() (*author.Author, *zap.Logger, error) 38 | } 39 | 40 | type testAuthorGetterImpl struct { 41 | acg authorConfigGetter 42 | librariansFlag string 43 | } 44 | 45 | func newTestAuthorGetter() testAuthorGetter { 46 | return &testAuthorGetterImpl{ 47 | acg: &authorConfigGetterImpl{}, 48 | librariansFlag: testLibrariansFlag, 49 | } 50 | } 51 | 52 | func (g *testAuthorGetterImpl) get() (*author.Author, *zap.Logger, error) { 53 | config, logger, err := g.acg.get(g.librariansFlag) 54 | if err != nil { 55 | return nil, logger, err 56 | } 57 | // since we're just doing tests, no need to worry about saving encrypted keychains and 58 | // generating more than one key on each 59 | authorKeys, selfReaderKeys := keychain.New(1), keychain.New(1) 60 | a, err := author.NewAuthor(config, authorKeys, selfReaderKeys, logger) 61 | return a, logger, err 62 | } 63 | -------------------------------------------------------------------------------- /libri/common/parse/parse_test.go: -------------------------------------------------------------------------------- 1 | package parse 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestAddr(t *testing.T) { 11 | cases := []struct { 12 | ip string 13 | port int 14 | expected *net.TCPAddr 15 | }{ 16 | {"192.168.1.1", 20100, &net.TCPAddr{IP: net.ParseIP("192.168.1.1"), Port: 20100}}, 17 | {"192.168.1.1", 11001, &net.TCPAddr{IP: net.ParseIP("192.168.1.1"), Port: 11001}}, 18 | {"localhost", 20100, &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 20100}}, 19 | } 20 | for _, c := range cases { 21 | actual, err := Addr(c.ip, c.port) 22 | assert.Nil(t, err) 23 | assert.Equal(t, c.expected, actual) 24 | } 25 | } 26 | 27 | func TestAddrs_ok(t *testing.T) { 28 | addrs := []string{ 29 | "192.168.1.1:20100", 30 | "192.168.1.1:11001", 31 | "localhost:20100", 32 | } 33 | expectedNetAddrs := []*net.TCPAddr{ 34 | {IP: net.ParseIP("192.168.1.1"), Port: 20100}, 35 | {IP: net.ParseIP("192.168.1.1"), Port: 11001}, 36 | {IP: net.ParseIP("127.0.0.1"), Port: 20100}, 37 | } 38 | actualNetAddrs, err := Addrs(addrs) 39 | 40 | assert.Nil(t, err) 41 | for i, a := range actualNetAddrs { 42 | assert.Equal(t, expectedNetAddrs[i], a) 43 | } 44 | } 45 | 46 | func TestParseAddrs_err(t *testing.T) { 47 | addrs := []string{ 48 | "192.168.1.1", // no port 49 | "192.168.1.1:A", // bad port 50 | "192::168::1:1:11001", // IPv6 instead of IPv4 51 | "192.168.1.1.11001", // bad port delimiter 52 | } 53 | 54 | // test individually 55 | for _, a := range addrs { 56 | na, err := Addrs([]string{a}) 57 | assert.Nil(t, na, a) 58 | assert.NotNil(t, err, a) 59 | } 60 | } 61 | 62 | func TestParseAddrs_multi_err(t *testing.T) { 63 | oneBad := []string{"192.168.1.1:20100", "192.168.1.1:A"} 64 | as1, err := Addrs(oneBad) 65 | assert.Len(t, as1, 1) 66 | assert.Nil(t, err) 67 | 68 | bothBad := []string{"192.168.1.1", "192.168.1.1:A"} 69 | as2, err := Addrs(bothBad) 70 | assert.Nil(t, as2) 71 | assert.NotNil(t, err) 72 | } 73 | -------------------------------------------------------------------------------- /libri/librarian/server/comm/prefer_test.go: -------------------------------------------------------------------------------- 1 | package comm 2 | 3 | import ( 4 | "testing" 5 | 6 | "math/rand" 7 | 8 | "github.com/drausin/libri/libri/common/id" 9 | "github.com/drausin/libri/libri/librarian/api" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestRpPreferer_Prefer(t *testing.T) { 14 | rng := rand.New(rand.NewSource(0)) 15 | 16 | peerID1 := id.NewPseudoRandom(rng) 17 | peerID2 := id.NewPseudoRandom(rng) 18 | peerID3 := id.NewPseudoRandom(rng) 19 | peerID4 := id.NewPseudoRandom(rng) 20 | qo1 := endpointQueryOutcomes{ 21 | api.Verify: QueryOutcomes{ 22 | Response: map[Outcome]*ScalarMetrics{ 23 | Success: {Count: 1}, 24 | }, 25 | }, 26 | api.Find: QueryOutcomes{ 27 | Response: map[Outcome]*ScalarMetrics{ 28 | Success: {Count: 1}, 29 | }, 30 | }, 31 | } 32 | qo2 := endpointQueryOutcomes{ 33 | api.Verify: { 34 | Response: map[Outcome]*ScalarMetrics{ 35 | Success: {Count: 2}, 36 | }, 37 | }, 38 | api.Find: QueryOutcomes{ 39 | Response: map[Outcome]*ScalarMetrics{ 40 | Success: {Count: 2}, 41 | }, 42 | }, 43 | } 44 | qo3 := endpointQueryOutcomes{ 45 | api.Verify: { 46 | Response: map[Outcome]*ScalarMetrics{ 47 | Success: {Count: 3}, 48 | }, 49 | }, 50 | api.Find: QueryOutcomes{ 51 | Response: map[Outcome]*ScalarMetrics{ 52 | Success: {Count: 3}, 53 | }, 54 | }, 55 | } 56 | qo4 := endpointQueryOutcomes{ 57 | api.Verify: newQueryOutcomes(), 58 | api.Find: QueryOutcomes{ 59 | Response: map[Outcome]*ScalarMetrics{ 60 | Success: {Count: 5}, 61 | }, 62 | }, 63 | } 64 | 65 | rec := &scalarRG{ 66 | peers: map[string]endpointQueryOutcomes{ 67 | peerID1.String(): qo1, 68 | peerID2.String(): qo2, 69 | peerID3.String(): qo3, 70 | peerID4.String(): qo4, 71 | }, 72 | } 73 | p := NewRpPreferer(rec) 74 | 75 | // prefer peer w/ more successful Verify or Find responses 76 | assert.True(t, p.Prefer(peerID2, peerID1)) 77 | assert.True(t, p.Prefer(peerID3, peerID2)) 78 | assert.True(t, p.Prefer(peerID4, peerID3)) 79 | } 80 | -------------------------------------------------------------------------------- /libri/librarian/server/routing/storage.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "github.com/drausin/libri/libri/common/id" 5 | cstorage "github.com/drausin/libri/libri/common/storage" 6 | "github.com/drausin/libri/libri/librarian/server/comm" 7 | "github.com/drausin/libri/libri/librarian/server/peer" 8 | sstorage "github.com/drausin/libri/libri/librarian/server/storage" 9 | "github.com/golang/protobuf/proto" 10 | ) 11 | 12 | var tableKey = []byte("RoutingTable") 13 | 14 | // Load retrieves the routing table form the KV DB. 15 | func Load( 16 | nl cstorage.Loader, preferer comm.Preferer, doctor comm.Doctor, params *Parameters, 17 | ) (Table, error) { 18 | bytes, err := nl.Load(tableKey) 19 | if bytes == nil || err != nil { 20 | return nil, err 21 | } 22 | stored := &sstorage.RoutingTable{} 23 | err = proto.Unmarshal(bytes, stored) 24 | if err != nil { 25 | return nil, err 26 | } 27 | return fromStored(stored, params, preferer, doctor), nil 28 | } 29 | 30 | // Save stores a representation of the routing table to the KV DB. 31 | func (rt *table) Save(ns cstorage.Storer) error { 32 | bytes, err := proto.Marshal(toStored(rt)) 33 | if err != nil { 34 | return err 35 | } 36 | return ns.Store(tableKey, bytes) 37 | } 38 | 39 | // fromStored returns a new Table instance from a StoredRoutingTable instance. 40 | func fromStored( 41 | stored *sstorage.RoutingTable, 42 | params *Parameters, 43 | preferer comm.Preferer, 44 | doctor comm.Doctor, 45 | ) Table { 46 | peers := make([]peer.Peer, len(stored.Peers)) 47 | for i, sp := range stored.Peers { 48 | peers[i] = peer.FromStored(sp) 49 | } 50 | rt, _ := NewWithPeers(id.FromBytes(stored.SelfId), preferer, doctor, params, peers) 51 | return rt 52 | } 53 | 54 | // toStored creates a new StoredRoutingTable instance from the Table instance. 55 | func toStored(rt Table) *sstorage.RoutingTable { 56 | storedPeers := make([]*sstorage.Peer, len(rt.(*table).peers)) 57 | i := 0 58 | for _, p := range rt.(*table).peers { 59 | storedPeers[i] = p.ToStored() 60 | i++ 61 | } 62 | return &sstorage.RoutingTable{ 63 | SelfId: rt.SelfID().Bytes(), 64 | Peers: storedPeers, 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /libri/librarian/server/requests_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | "math/rand" 6 | "testing" 7 | 8 | "github.com/drausin/libri/libri/common/ecid" 9 | "github.com/drausin/libri/libri/librarian/api" 10 | "github.com/drausin/libri/libri/librarian/client" 11 | "github.com/golang/protobuf/proto" 12 | "github.com/stretchr/testify/assert" 13 | "golang.org/x/net/context" 14 | ) 15 | 16 | // alwaysSigVerifier implements the signature.Verifier interface but just blindly verifies 17 | // every signature. 18 | type alwaysSigVerifier struct{} 19 | 20 | func (asv *alwaysSigVerifier) Verify(encToken string, fromPubKey *ecdsa.PublicKey, 21 | m proto.Message) error { 22 | return nil 23 | } 24 | 25 | func TestRequestVerifier_Verify_ok(t *testing.T) { 26 | rv := &verifier{ 27 | sigVerifier: &alwaysSigVerifier{}, 28 | } 29 | 30 | rng := rand.New(rand.NewSource(0)) 31 | signedJWT, orgSignedJWT := "dummy.signed.token", "dummy.signed.org-token" 32 | ctx := client.NewIncomingSignatureContext(context.Background(), signedJWT, orgSignedJWT) 33 | meta := client.NewRequestMetadata(ecid.NewPseudoRandom(rng), ecid.NewPseudoRandom(rng)) 34 | 35 | assert.Nil(t, rv.Verify(ctx, nil, meta)) 36 | } 37 | 38 | func TestRequestVerifier_Verify_err(t *testing.T) { 39 | rv := &verifier{ 40 | sigVerifier: &alwaysSigVerifier{}, 41 | } 42 | 43 | assert.NotNil(t, rv.Verify(context.Background(), nil, nil)) // no signature in context 44 | 45 | rng := rand.New(rand.NewSource(0)) 46 | signedJWT, orgSignedJWT := "dummy.signed.token", "dummy.signed.org-token" 47 | ctx := client.NewIncomingSignatureContext(context.Background(), signedJWT, orgSignedJWT) 48 | 49 | assert.NotNil(t, rv.Verify(ctx, nil, &api.RequestMetadata{ 50 | PubKey: []byte{255, 254, 253}, // bad pub key 51 | })) 52 | 53 | assert.NotNil(t, rv.Verify(ctx, nil, &api.RequestMetadata{ 54 | PubKey: ecid.NewPseudoRandom(rng).PublicKeyBytes(), 55 | RequestId: nil, // can't be nil 56 | })) 57 | 58 | assert.NotNil(t, rv.Verify(ctx, nil, &api.RequestMetadata{ 59 | PubKey: ecid.NewPseudoRandom(rng).PublicKeyBytes(), 60 | RequestId: []byte{1, 2, 3}, // not 32 bytes 61 | })) 62 | } 63 | -------------------------------------------------------------------------------- /deploy/cloud/terraform/gcp/module/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | ############ 3 | # Required # 4 | ############ 5 | 6 | variable "credentials_file" { 7 | description = "GCP JSON credentials filepath" 8 | } 9 | 10 | variable "gcp_project" { 11 | description = "GCP project owning the cluster" 12 | } 13 | 14 | variable "gcs_clusters_bucket" { 15 | description = "GCS bucket for cluster state and backups" 16 | } 17 | 18 | variable "cluster_name" { 19 | description = "name of the libri cluster" 20 | } 21 | 22 | 23 | ############ 24 | # Optional # 25 | ############ 26 | 27 | variable "gce_node_region" { 28 | default = "us-east1" 29 | } 30 | 31 | variable "gce_node_zone" { 32 | default = "us-east1-b" 33 | } 34 | 35 | variable "gce_node_network" { 36 | default = "default" 37 | } 38 | 39 | variable "num_cluster_nodes" { 40 | description = "current number of cluster nodes" 41 | default = 3 42 | } 43 | 44 | variable "gce_node_image_type" { 45 | default = "ubuntu-1604-xenial-v20170125" 46 | } 47 | 48 | variable "gce_node_machine_type" { 49 | default = "n1-standard-1" 50 | } 51 | 52 | variable "node_disk_size_gb" { 53 | description = "size (GB) of disk used by each cluster node" 54 | default = 25 55 | } 56 | 57 | variable "num_librarians" { 58 | description = "current number of librarian peers in cluster" 59 | default = 3 60 | } 61 | 62 | variable "librarian_disk_size_gb" { 63 | description = "size (GB) of persistant disk used by each librarian" 64 | default = 10 65 | } 66 | 67 | variable "librarian_disk_type" { 68 | description = "type of persistent disk used by each librarian" 69 | default = "pd-standard" 70 | } 71 | 72 | variable "min_libri_port" { 73 | default = 30100 74 | } 75 | 76 | # Name of the ssh key pair to use for GCE instances. 77 | # The public key will be passed at instance creation, and the private 78 | # key will be used by the local ssh client. 79 | # 80 | # The path is expanded to: ~/.ssh/.pub 81 | # 82 | # If you use `gcloud compute ssh` or `gcloud compute copy-files`, you may want 83 | # to leave this as "google_compute_engine" for convenience. 84 | variable "key_name" { 85 | default = "google_compute_engine" 86 | } 87 | -------------------------------------------------------------------------------- /libri/librarian/api/metadata.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/dustin/go-humanize" 7 | "go.uber.org/zap/zapcore" 8 | ) 9 | 10 | const ( 11 | // logging keys 12 | logMediaType = "media_type" 13 | logCompressionCodec = "compression_codec" 14 | logCiphertextSize = "ciphertext_size" 15 | logCiphertextSizeHuman = "ciphertext_size_human" 16 | logUncompressedSize = "uncompressed_size" 17 | logUncompressedSizeHuman = "uncompressed_size_human" 18 | ) 19 | 20 | var ( 21 | // ErrMissingMediaType indicates when metadata has zero-valued MediaType. 22 | ErrMissingMediaType = errors.New("missing MediaType") 23 | 24 | // ErrMissingCiphertextSize indicates when metadata has zero-valued CiphertextSize. 25 | ErrMissingCiphertextSize = errors.New("missing CiphertextSize") 26 | 27 | // ErrMissingUncompressedSize indicates when metadata has zero-valued UncompressedSize. 28 | ErrMissingUncompressedSize = errors.New("missing UncompressedSize") 29 | ) 30 | 31 | // ValidateEntryMetadata checks that the metadata has all the required non-zero values. 32 | func ValidateEntryMetadata(m *EntryMetadata) error { 33 | if m.MediaType == "" { 34 | return ErrMissingMediaType 35 | } 36 | if m.CiphertextSize == 0 { 37 | return ErrMissingCiphertextSize 38 | } 39 | if err := ValidateHMAC256(m.CiphertextMac); err != nil { 40 | return err 41 | } 42 | if m.UncompressedSize == 0 { 43 | return ErrMissingUncompressedSize 44 | } 45 | if err := ValidateHMAC256(m.UncompressedMac); err != nil { 46 | return err 47 | } 48 | return nil 49 | } 50 | 51 | // MarshalLogObject converts the metadata into an object (which will become json) for logging. 52 | func (m *EntryMetadata) MarshalLogObject(oe zapcore.ObjectEncoder) error { 53 | oe.AddString(logMediaType, m.MediaType) 54 | oe.AddString(logCompressionCodec, m.CompressionCodec.String()) 55 | oe.AddUint64(logCiphertextSize, m.CiphertextSize) 56 | oe.AddString(logCiphertextSizeHuman, humanize.Bytes(m.CiphertextSize)) 57 | oe.AddUint64(logUncompressedSize, m.UncompressedSize) 58 | oe.AddString(logUncompressedSizeHuman, humanize.Bytes(m.UncompressedSize)) 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /libri/author/io/enc/enc_test.go: -------------------------------------------------------------------------------- 1 | package enc 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestNewEncrypter_ok(t *testing.T) { 11 | rng := rand.New(rand.NewSource(0)) 12 | keys := NewPseudoRandomEEK(rng) 13 | enc, err := NewEncrypter(keys) 14 | assert.Nil(t, err) 15 | assert.NotNil(t, enc.(*encrypter).gcmCipher) 16 | assert.NotNil(t, enc.(*encrypter).pageIVMAC) 17 | } 18 | 19 | func TestNewEncrypter_err(t *testing.T) { 20 | enc, err := NewEncrypter(&EEK{}) 21 | assert.NotNil(t, err) 22 | assert.Nil(t, enc) 23 | } 24 | 25 | func TestNewDecrypter_ok(t *testing.T) { 26 | rng := rand.New(rand.NewSource(0)) 27 | keys := NewPseudoRandomEEK(rng) 28 | enc, err := NewDecrypter(keys) 29 | assert.Nil(t, err) 30 | assert.NotNil(t, enc.(*decrypter).gcmCipher) 31 | assert.NotNil(t, enc.(*decrypter).pageIVMAC) 32 | } 33 | 34 | func TestNewDecrypter_err(t *testing.T) { 35 | enc, err := NewDecrypter(&EEK{}) 36 | assert.NotNil(t, err) 37 | assert.Nil(t, enc) 38 | } 39 | 40 | func TestEncryptDecrypt(t *testing.T) { 41 | rng := rand.New(rand.NewSource(0)) 42 | keys := NewPseudoRandomEEK(rng) 43 | nPlaintextBytesPerPage, nPages := 32, uint32(3) 44 | 45 | encrypter, err := NewEncrypter(keys) 46 | assert.Nil(t, err) 47 | 48 | decrypter, err := NewDecrypter(keys) 49 | assert.Nil(t, err) 50 | 51 | for p := uint32(0); p < nPages; p++ { 52 | 53 | plaintext1 := make([]byte, nPlaintextBytesPerPage) 54 | n0, err := rng.Read(plaintext1) 55 | assert.Equal(t, n0, nPlaintextBytesPerPage) 56 | assert.Nil(t, err) 57 | 58 | ciphertext, err := encrypter.Encrypt(plaintext1, p) 59 | assert.Nil(t, err) 60 | 61 | // ciphertext can be longer b/c of GCM auth overhead 62 | assert.True(t, len(ciphertext) >= len(plaintext1)) 63 | 64 | // check that page number matters 65 | diffCiphertext, err := encrypter.Encrypt(plaintext1, p+1) 66 | assert.Nil(t, err) 67 | assert.NotEqual(t, ciphertext, diffCiphertext) 68 | 69 | // check that decrypted plaintext matches original 70 | plaintext2, err := decrypter.Decrypt(ciphertext, p) 71 | assert.Nil(t, err) 72 | assert.Equal(t, plaintext1, plaintext2) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /libri/common/subscribe/subscription_test.go: -------------------------------------------------------------------------------- 1 | package subscribe 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestNewSubscription_ok(t *testing.T) { 12 | rng := rand.New(rand.NewSource(0)) 13 | s, err := NewSubscription([][]byte{}, 0.5, [][]byte{}, 0.5, rng) 14 | assert.Nil(t, err) 15 | assert.NotNil(t, s) 16 | assert.NotNil(t, s.AuthorPublicKeys) 17 | assert.NotNil(t, s.ReaderPublicKeys) 18 | } 19 | 20 | func TestNewSubscription_err(t *testing.T) { 21 | rng := rand.New(rand.NewSource(0)) 22 | 23 | cases := []struct { 24 | authorPubs [][]byte 25 | authorFp float64 26 | readerPubs [][]byte 27 | readerFp float64 28 | }{ 29 | {nil, 0.5, [][]byte{}, 0.5}, // 0 30 | {[][]byte{}, -0.05, [][]byte{}, 0.5}, // 1 31 | {[][]byte{}, 0.0, [][]byte{}, 0.5}, // 2 32 | {[][]byte{}, 1.5, [][]byte{}, 0.5}, // 3 33 | {[][]byte{}, 0.5, nil, 0.5}, // 4 34 | {[][]byte{}, 0.5, [][]byte{}, -0.05}, // 5 35 | {[][]byte{}, 0.5, [][]byte{}, 0.0}, // 6 36 | {[][]byte{}, 0.5, [][]byte{}, 1.5}, // 7 37 | } 38 | for i, c := range cases { 39 | info := fmt.Sprintf("i: %d", i) 40 | s, err := NewSubscription(c.authorPubs, c.authorFp, c.readerPubs, c.readerFp, rng) 41 | assert.NotNil(t, err, info) 42 | assert.Nil(t, s, info) 43 | } 44 | } 45 | 46 | func TestNewAuthorSubscription(t *testing.T) { 47 | rng := rand.New(rand.NewSource(0)) 48 | s, err := NewAuthorSubscription([][]byte{}, 0.5, rng) 49 | assert.Nil(t, err) 50 | assert.NotNil(t, s) 51 | assert.NotNil(t, s.AuthorPublicKeys) 52 | assert.NotNil(t, s.ReaderPublicKeys) 53 | } 54 | 55 | func TestNewReaderSubscription(t *testing.T) { 56 | rng := rand.New(rand.NewSource(0)) 57 | s, err := NewReaderSubscription([][]byte{}, 0.5, rng) 58 | assert.Nil(t, err) 59 | assert.NotNil(t, s) 60 | assert.NotNil(t, s.AuthorPublicKeys) 61 | assert.NotNil(t, s.ReaderPublicKeys) 62 | } 63 | 64 | func TestNewFPSubscription(t *testing.T) { 65 | rng := rand.New(rand.NewSource(0)) 66 | s, err := NewFPSubscription(0.5, rng) 67 | assert.Nil(t, err) 68 | assert.NotNil(t, s) 69 | assert.NotNil(t, s.AuthorPublicKeys) 70 | assert.NotNil(t, s.ReaderPublicKeys) 71 | } 72 | -------------------------------------------------------------------------------- /libri/librarian/server/routing/balancer.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "math/rand" 5 | "sync" 6 | "time" 7 | 8 | "github.com/drausin/libri/libri/librarian/api" 9 | "github.com/drausin/libri/libri/librarian/client" 10 | "github.com/drausin/libri/libri/librarian/server/peer" 11 | ) 12 | 13 | var ( 14 | tableSampleRetryWait = 15 * time.Second 15 | numRetries = 32 16 | sampleBatchSize = uint(8) 17 | ) 18 | 19 | // NewClientBalancer returns a new client.Balancer that uses the routing tables's Sample() 20 | // method and returns a unique client on every Next() call. 21 | func NewClientBalancer(rt Table, clients client.Pool) client.SetBalancer { 22 | return &tableSetBalancer{ 23 | rt: rt, 24 | rng: rand.New(rand.NewSource(0)), 25 | set: make(map[string]struct{}), 26 | cache: make([]peer.Peer, 0), 27 | clients: clients, 28 | } 29 | } 30 | 31 | type tableSetBalancer struct { 32 | rt Table 33 | rng *rand.Rand 34 | set map[string]struct{} 35 | cache []peer.Peer 36 | clients client.Pool 37 | mu sync.Mutex 38 | } 39 | 40 | func (b *tableSetBalancer) AddNext() (api.LibrarianClient, string, error) { 41 | for c := 0; c < numRetries; c++ { 42 | b.mu.Lock() 43 | if len(b.cache) == 0 { 44 | b.cache = b.rt.Sample(sampleBatchSize, b.rng) 45 | if len(b.cache) == 0 { 46 | b.mu.Unlock() 47 | // wait for routing table to possibly fill up a bit 48 | time.Sleep(tableSampleRetryWait) 49 | continue 50 | } 51 | } 52 | nextPeer := b.cache[0] 53 | b.cache = b.cache[1:] 54 | nextAddress := nextPeer.Address().String() 55 | if _, in := b.set[nextAddress]; !in { 56 | // update current state & return connection to new peer 57 | b.set[nextAddress] = struct{}{} 58 | b.mu.Unlock() 59 | lc, err := b.clients.Get(nextAddress) 60 | return lc, nextAddress, err 61 | } 62 | b.mu.Unlock() 63 | 64 | // wait for routing table to possibly fill up a bit 65 | time.Sleep(tableSampleRetryWait) 66 | } 67 | return nil, "", client.ErrNoNewClients 68 | } 69 | 70 | func (b *tableSetBalancer) Remove(address string) error { 71 | b.mu.Lock() 72 | defer b.mu.Unlock() 73 | if _, in := b.set[address]; !in { 74 | return client.ErrClientMissingFromSet 75 | } 76 | delete(b.set, address) 77 | return nil 78 | } 79 | -------------------------------------------------------------------------------- /libri/librarian/api/subscription.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import "errors" 4 | 5 | var ( 6 | // ErrEmptySubscriptionFilters indicates when a *Subscription has an empty author or reader 7 | // public key filter. 8 | ErrEmptySubscriptionFilters = errors.New("subscription has empty filters") 9 | 10 | // ErrMissingSubscription indicates when a Subscription is unexpectedly nil. 11 | ErrMissingSubscription = errors.New("missing Subscription") 12 | 13 | // ErrMissingPublication indicates when a Publication is unexpectedly nil. 14 | ErrMissingPublication = errors.New("missing Publication") 15 | ) 16 | 17 | // ValidateSubscription validates that a subscription is not missing any required fields. It returns 18 | // nil if the subscription is valid. 19 | func ValidateSubscription(s *Subscription) error { 20 | if s == nil { 21 | return ErrMissingSubscription 22 | } 23 | if s.AuthorPublicKeys == nil || s.AuthorPublicKeys.Encoded == nil { 24 | return ErrEmptySubscriptionFilters 25 | } 26 | if s.ReaderPublicKeys == nil || s.ReaderPublicKeys.Encoded == nil { 27 | return ErrEmptySubscriptionFilters 28 | } 29 | return nil 30 | } 31 | 32 | // ValidatePublication validates that a publication has all fields of the correct length. 33 | func ValidatePublication(p *Publication) error { 34 | if p == nil { 35 | return ErrMissingPublication 36 | } 37 | if err := ValidateBytes(p.EntryKey, DocumentKeyLength, "EntryKey"); err != nil { 38 | return err 39 | } 40 | if err := ValidateBytes(p.EnvelopeKey, DocumentKeyLength, "EnvelopeKey"); err != nil { 41 | return err 42 | } 43 | if err := ValidatePublicKey(p.AuthorPublicKey); err != nil { 44 | return err 45 | } 46 | if err := ValidatePublicKey(p.ReaderPublicKey); err != nil { 47 | return err 48 | } 49 | return nil 50 | } 51 | 52 | // GetPublication returns a *Publication object if one can be made from the given document key and 53 | // value. If not, it returns nil. 54 | func GetPublication(key []byte, value *Document) *Publication { 55 | switch x := value.Contents.(type) { 56 | case *Document_Envelope: 57 | return &Publication{ 58 | EntryKey: x.Envelope.EntryKey, 59 | EnvelopeKey: key, 60 | AuthorPublicKey: x.Envelope.AuthorPublicKey, 61 | ReaderPublicKey: x.Envelope.ReaderPublicKey, 62 | } 63 | } 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /deploy/cloud/terraform/gcp/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "cluster_host" { 3 | description = "host of the cluster [gcp|minikube]" 4 | } 5 | 6 | variable "cluster_admin_user" { 7 | description = "k8s admin user creating the cluster" 8 | } 9 | 10 | variable "credentials_file" { 11 | description = "GCP JSON credentials filepath" 12 | } 13 | 14 | variable "num_librarians" { 15 | description = "current number of librarian peers in cluster" 16 | } 17 | 18 | variable "librarian_disk_size_gb" { 19 | description = "size (GB) of persistant disk used by each librarian" 20 | } 21 | 22 | variable "librarian_disk_type" { 23 | description = "type of persistent disk used by each librarian" 24 | default = "pd-standard" 25 | } 26 | 27 | variable "librarian_libri_version" { 28 | description = "libri version (e.g., 0.1.0, latest, snapshot) to use for librarian container" 29 | } 30 | 31 | variable "librarian_cpu_limit" { 32 | description = "librarian container CPU limit (e.g., 500m, 0.5, 1)" 33 | } 34 | 35 | variable "librarian_ram_limit" { 36 | description = "librarian container RAM limit (e.g., 500M, 1G)" 37 | } 38 | 39 | variable "num_cluster_nodes" { 40 | description = "current number of cluster nodes" 41 | } 42 | 43 | variable "cluster_node_machine_type" { 44 | description = "GCE cluster node machine type" 45 | } 46 | 47 | variable "librarian_public_port_start" { 48 | description = "public port for librarian-0 service" 49 | } 50 | 51 | variable "librarian_local_port" { 52 | description = "local port for each librarian instance" 53 | } 54 | 55 | variable "librarian_local_metrics_port" { 56 | description = "local metrics port for each librarian instance" 57 | } 58 | 59 | variable "grafana_port" { 60 | description = "port for Grafana service" 61 | } 62 | 63 | variable "prometheus_port" { 64 | description = "port for Prometheus service" 65 | } 66 | 67 | variable "grafana_ram_limit" { 68 | description = "Grafana pod RAM limit (e.g., 500M, 1G)" 69 | } 70 | 71 | variable "grafana_cpu_limit" { 72 | description = "Grafana pod CPU limit (e.g., 500m, 0.5, 1)" 73 | } 74 | 75 | variable "prometheus_ram_limit" { 76 | description = "Prometheus pod RAM limit (e.g., 500M, 1G)" 77 | } 78 | 79 | variable "prometheus_cpu_limit" { 80 | description = "Prometheus pod CPU limit (e.g., 500m, 0.5, 1)" 81 | } 82 | -------------------------------------------------------------------------------- /libri/author/io/enc/enc.go: -------------------------------------------------------------------------------- 1 | package enc 2 | 3 | import ( 4 | "crypto/cipher" 5 | "crypto/hmac" 6 | "crypto/sha256" 7 | "encoding/binary" 8 | "hash" 9 | ) 10 | 11 | // Encrypter encrypts (compressed) plaintext of a page. 12 | type Encrypter interface { 13 | // Encrypt encrypts the given plaintext for a given pageIndex, returning the ciphertext. 14 | Encrypt(plaintext []byte, pageIndex uint32) ([]byte, error) 15 | } 16 | 17 | type encrypter struct { 18 | gcmCipher cipher.AEAD 19 | pageIVMAC hash.Hash 20 | } 21 | 22 | // NewEncrypter creates a new Encrypter using the encryption keys. 23 | func NewEncrypter(keys *EEK) (Encrypter, error) { 24 | gcmCipher, err := newGCMCipher(keys.AESKey) 25 | if err != nil { 26 | return nil, err 27 | } 28 | return &encrypter{ 29 | gcmCipher: gcmCipher, 30 | pageIVMAC: hmac.New(sha256.New, keys.PageIVSeed), 31 | }, nil 32 | } 33 | 34 | func (e *encrypter) Encrypt(plaintext []byte, pageIndex uint32) ([]byte, error) { 35 | pageIV := generatePageIV(pageIndex, e.pageIVMAC, e.gcmCipher.NonceSize()) 36 | ciphertext := e.gcmCipher.Seal(nil, pageIV, plaintext, nil) 37 | return ciphertext, nil 38 | } 39 | 40 | // Decrypter decrypts a page's ciphertext. 41 | type Decrypter interface { 42 | // Decrypt decrypts the ciphertext of a particular page. 43 | Decrypt(ciphertext []byte, pageIndex uint32) ([]byte, error) 44 | } 45 | 46 | type decrypter struct { 47 | gcmCipher cipher.AEAD 48 | pageIVMAC hash.Hash 49 | } 50 | 51 | // NewDecrypter creates a new Decrypter instance using the encryption keys. 52 | func NewDecrypter(keys *EEK) (Decrypter, error) { 53 | gcmCipher, err := newGCMCipher(keys.AESKey) 54 | if err != nil { 55 | return nil, err 56 | } 57 | return &decrypter{ 58 | gcmCipher: gcmCipher, 59 | pageIVMAC: hmac.New(sha256.New, keys.PageIVSeed), 60 | }, nil 61 | } 62 | 63 | func (d *decrypter) Decrypt(ciphertext []byte, pageIndex uint32) ([]byte, error) { 64 | pageIV := generatePageIV(pageIndex, d.pageIVMAC, d.gcmCipher.NonceSize()) 65 | return d.gcmCipher.Open(nil, pageIV, ciphertext, nil) 66 | } 67 | 68 | func generatePageIV(pageIndex uint32, pageIVMac hash.Hash, size int) []byte { 69 | pageIndexBytes := make([]byte, 4) 70 | binary.BigEndian.PutUint32(pageIndexBytes, pageIndex) 71 | pageIVMac.Reset() 72 | iv := pageIVMac.Sum(pageIndexBytes) 73 | return iv[:size] 74 | } 75 | -------------------------------------------------------------------------------- /libri/librarian/api/metadata_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "fmt" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "go.uber.org/zap" 11 | "go.uber.org/zap/zapcore" 12 | ) 13 | 14 | func TestValidateMetadata_ok(t *testing.T) { 15 | rng := rand.New(rand.NewSource(0)) 16 | m := &EntryMetadata{ 17 | MediaType: "application/x-pdf", 18 | CiphertextSize: 1, 19 | CiphertextMac: RandBytes(rng, 32), 20 | UncompressedSize: 2, 21 | UncompressedMac: RandBytes(rng, 32), 22 | } 23 | err := ValidateEntryMetadata(m) 24 | assert.Nil(t, err) 25 | } 26 | 27 | func TestValidateMetadata_err(t *testing.T) { 28 | rng := rand.New(rand.NewSource(0)) 29 | ms := []*EntryMetadata{ 30 | { // 1 31 | MediaType: "", 32 | CiphertextSize: 1, 33 | CiphertextMac: RandBytes(rng, 32), 34 | UncompressedSize: 2, 35 | UncompressedMac: RandBytes(rng, 32), 36 | }, 37 | { // 2 38 | MediaType: "application/x-pdf", 39 | CiphertextSize: 0, 40 | CiphertextMac: RandBytes(rng, 32), 41 | UncompressedSize: 2, 42 | UncompressedMac: RandBytes(rng, 32), 43 | }, 44 | { // 3 45 | MediaType: "application/x-pdf", 46 | CiphertextSize: 1, 47 | CiphertextMac: nil, 48 | UncompressedSize: 2, 49 | UncompressedMac: RandBytes(rng, 32), 50 | }, 51 | { // 4 52 | MediaType: "application/x-pdf", 53 | CiphertextSize: 1, 54 | CiphertextMac: RandBytes(rng, 32), 55 | UncompressedSize: 0, 56 | UncompressedMac: RandBytes(rng, 32), 57 | }, 58 | { // 5 59 | MediaType: "application/x-pdf", 60 | CiphertextSize: 1, 61 | CiphertextMac: RandBytes(rng, 32), 62 | UncompressedSize: 2, 63 | UncompressedMac: nil, 64 | }, 65 | } 66 | for i, m := range ms { 67 | err := ValidateEntryMetadata(m) 68 | assert.NotNil(t, err, fmt.Sprintf("case %d", i)) 69 | } 70 | } 71 | 72 | func TestMetadata_MarshalLogObject(t *testing.T) { 73 | oe := zapcore.NewJSONEncoder(zap.NewDevelopmentEncoderConfig()) 74 | rng := rand.New(rand.NewSource(0)) 75 | m := &EntryMetadata{ 76 | MediaType: "application/x-pdf", 77 | CiphertextSize: 1, 78 | CiphertextMac: RandBytes(rng, 32), 79 | UncompressedSize: 2, 80 | UncompressedMac: RandBytes(rng, 32), 81 | } 82 | err := m.MarshalLogObject(oe) 83 | assert.Nil(t, err) 84 | } 85 | -------------------------------------------------------------------------------- /libri/librarian/server/comm/doctor_test.go: -------------------------------------------------------------------------------- 1 | package comm 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | "time" 7 | 8 | "github.com/drausin/libri/libri/common/id" 9 | "github.com/drausin/libri/libri/librarian/api" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestNaiveDoctor_Healthy(t *testing.T) { 14 | rng := rand.New(rand.NewSource(0)) 15 | d := NewNaiveDoctor() 16 | assert.True(t, d.Healthy(id.NewPseudoRandom(rng))) 17 | } 18 | 19 | func TestResponseTimeDoctor_Healthy(t *testing.T) { 20 | now := time.Now() 21 | rng := rand.New(rand.NewSource(0)) 22 | peerID := id.NewPseudoRandom(rng) 23 | 24 | // check not healthy since latest success was 15 mins before latest error 25 | qo1 := newQueryOutcomes() 26 | qo1[Response][Error].Latest = now 27 | qo1[Response][Success].Latest = now.Add(-15 * time.Minute) 28 | d1 := NewResponseTimeDoctor(&fixedRecorder{getValue: qo1}) 29 | assert.False(t, d1.Healthy(peerID)) 30 | 31 | // check not healthy since no verifications and latest Find success 15 mins before latest 32 | // error 33 | g4 := &fixedGetter{ 34 | outcomes: endpointQueryOutcomes{ 35 | api.Verify: newQueryOutcomes(), 36 | api.Find: newQueryOutcomes(), 37 | }, 38 | } 39 | g4.outcomes[api.Find][Response][Error].Latest = now 40 | g4.outcomes[api.Find][Response][Success].Latest = now.Add(-15 * time.Minute) 41 | d4 := NewResponseTimeDoctor(g4) 42 | assert.False(t, d4.Healthy(peerID)) 43 | 44 | // check healthy since latest success only 3 mins before latest error 45 | qo2 := newQueryOutcomes() 46 | qo2[Response][Error].Latest = now 47 | qo2[Response][Success].Latest = now.Add(-3 * time.Minute) 48 | d2 := NewResponseTimeDoctor(&fixedRecorder{getValue: qo2}) 49 | assert.True(t, d2.Healthy(peerID)) 50 | 51 | // check healthy since latest success was 5 mins after latest error 52 | qo3 := newQueryOutcomes() 53 | qo3[Response][Error].Latest = now 54 | qo3[Response][Success].Latest = now.Add(5 * time.Minute) 55 | d3 := NewResponseTimeDoctor(&fixedRecorder{getValue: qo3}) 56 | assert.True(t, d3.Healthy(peerID)) 57 | } 58 | 59 | type fixedGetter struct { 60 | outcomes endpointQueryOutcomes 61 | } 62 | 63 | func (f *fixedGetter) Get(peerID id.ID, endpoint api.Endpoint) QueryOutcomes { 64 | return f.outcomes[endpoint] 65 | } 66 | 67 | func (f *fixedGetter) CountPeers(endpoint api.Endpoint, qt QueryType, known bool) int { 68 | panic("implement me") 69 | } 70 | -------------------------------------------------------------------------------- /libri/common/storage/testing_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "github.com/drausin/libri/libri/common/id" 8 | "github.com/drausin/libri/libri/librarian/api" 9 | "github.com/pkg/errors" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestTestSLD(t *testing.T) { 14 | key, value1, value2 := []byte{1, 2, 3}, []byte{4, 5, 6}, []byte{7, 8, 9} 15 | sld := &TestSLD{ 16 | Bytes: value1, 17 | LoadErr: errors.New("some Load error"), 18 | IterateErr: errors.New("some Iterate error"), 19 | StoreErr: errors.New("some Store error"), 20 | DeleteErr: errors.New("some Delete error"), 21 | } 22 | 23 | bytes, err := sld.Load(key) 24 | assert.NotNil(t, err) 25 | assert.Equal(t, value1, bytes) 26 | 27 | err = sld.Iterate(nil, nil, nil, nil) 28 | assert.NotNil(t, err) 29 | 30 | err = sld.Store(key, value2) 31 | assert.NotNil(t, err) 32 | assert.Equal(t, value2, sld.Bytes) 33 | 34 | err = sld.Delete(key) 35 | assert.NotNil(t, err) 36 | assert.Nil(t, sld.Bytes) 37 | } 38 | 39 | func TestTestDocSLD(t *testing.T) { 40 | rng := rand.New(rand.NewSource(0)) 41 | 42 | dsld := NewTestDocSLD() 43 | dsld.StoreErr = errors.New("some Store error") 44 | dsld.IterateErr = errors.New("some Iterate error") 45 | dsld.LoadErr = errors.New("some Load error") 46 | dsld.MacErr = errors.New("some Mac error") 47 | dsld.DeleteErr = errors.New("some Delete error") 48 | 49 | nDocs := 3 50 | keys := make([]id.ID, nDocs) 51 | for c := 0; c < nDocs; c++ { 52 | // kinda testing both happy and sad paths here at once, but...it's fine 53 | value1, key := api.NewTestDocument(rng) 54 | keys[c] = key 55 | err := dsld.Store(key, value1) 56 | assert.NotNil(t, err) 57 | 58 | value2, err := dsld.Load(key) 59 | assert.NotNil(t, err) 60 | assert.Equal(t, value1, value2) 61 | 62 | _, err = dsld.Mac(key, nil) 63 | assert.NotNil(t, err) 64 | } 65 | 66 | nIters := 0 67 | done := make(chan struct{}) 68 | err := dsld.Iterate(done, func(key id.ID, value []byte) { 69 | nIters++ 70 | }) 71 | assert.NotNil(t, err) 72 | assert.Equal(t, nDocs, nIters) 73 | 74 | nIters = 0 75 | close(done) 76 | err = dsld.Iterate(done, func(key id.ID, value []byte) { 77 | nIters++ 78 | }) 79 | assert.NotNil(t, err) 80 | assert.Zero(t, nIters) 81 | 82 | for c := 0; c < nDocs; c++ { 83 | err = dsld.Delete(keys[c]) 84 | assert.NotNil(t, err) 85 | } 86 | assert.Zero(t, len(dsld.Stored)) 87 | } 88 | -------------------------------------------------------------------------------- /libri/librarian/server/peer/testing.go: -------------------------------------------------------------------------------- 1 | package peer 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "net" 7 | "testing" 8 | "time" 9 | 10 | cerrors "github.com/drausin/libri/libri/common/errors" 11 | "github.com/drausin/libri/libri/common/id" 12 | "github.com/drausin/libri/libri/librarian/server/storage" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | // NewTestPeer generates a new peer suitable for testing using a random number generator for the 17 | // ID and an index. 18 | func NewTestPeer(rng *rand.Rand, idx int) Peer { 19 | return New( 20 | id.NewPseudoRandom(rng), 21 | fmt.Sprintf("test-peer-%d", idx+1), 22 | NewTestPublicAddr(idx), 23 | ) 24 | } 25 | 26 | // NewTestPublicAddr creates a new net.TCPAddr given a particular peer index. 27 | func NewTestPublicAddr(idx int) *net.TCPAddr { 28 | address, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("127.0.0.1:%v", 20100+idx)) 29 | cerrors.MaybePanic(err) 30 | return address 31 | } 32 | 33 | // NewTestPeers generates n new peers suitable for testing use with random IDs and incrementing 34 | // values of other fields. 35 | func NewTestPeers(rng *rand.Rand, n int) []Peer { 36 | ps := make([]Peer, n) 37 | for i := 0; i < n; i++ { 38 | ps[i] = NewTestPeer(rng, i) 39 | } 40 | return ps 41 | } 42 | 43 | // NewTestStoredPeer generates a new storage.Peer suitable for testing using a random number 44 | // generator for the ID and an index. 45 | func NewTestStoredPeer(rng *rand.Rand, idx int) *storage.Peer { 46 | now := time.Unix(int64(idx), 0).UTC() 47 | return &storage.Peer{ 48 | Id: id.NewPseudoRandom(rng).Bytes(), 49 | Name: fmt.Sprintf("peer-%d", idx+1), 50 | PublicAddress: &storage.Address{ 51 | Ip: "192.168.1.1", 52 | Port: uint32(20100 + idx), 53 | }, 54 | QueryOutcomes: &storage.QueryOutcomes{ 55 | Responses: &storage.QueryTypeOutcomes{ 56 | Earliest: now.Unix(), 57 | Latest: now.Unix(), 58 | NQueries: 1, 59 | NErrors: 0, 60 | }, 61 | Requests: &storage.QueryTypeOutcomes{}, // everything will be zero 62 | }, 63 | } 64 | } 65 | 66 | // AssertPeersEqual checks that the stored and non-stored representations of a peer are equal. 67 | func AssertPeersEqual(t *testing.T, sp *storage.Peer, p Peer) { 68 | assert.Equal(t, sp.Id, p.ID().Bytes()) 69 | publicAddres := p.(*peer).Address() 70 | assert.Equal(t, sp.PublicAddress.Ip, publicAddres.IP.String()) 71 | assert.Equal(t, sp.PublicAddress.Port, uint32(publicAddres.Port)) 72 | } 73 | -------------------------------------------------------------------------------- /libri/cmd/banners.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "io" 5 | "runtime" 6 | "text/template" 7 | "time" 8 | 9 | "github.com/drausin/libri/libri/common/errors" 10 | "github.com/drausin/libri/version" 11 | ) 12 | 13 | const librarianTemplate = ` 14 | 15 | ██╗ ██╗ ██████╗ ██████╗ ██╗ 16 | ██║ ██║ ██╔══██╗ ██╔══██╗ ██║ 17 | ██║ ██║ ██████╔╝ ██████╔╝ ██║ 18 | ██║ ██║ ██╔══██╗ ██╔══██╗ ██║ 19 | ███████╗ ██║ ██████╔╝ ██║ ██║ ██║ 20 | ╚══════╝ ╚═╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ 21 | 22 | Libri Librarian Server 23 | 24 | Libri version {{ .LibriVersion }} 25 | Build Date: {{ .BuildDate }} 26 | Git Branch: {{ .GitBranch }} 27 | Git Revision: {{ .GitRevision }} 28 | Go version: {{ .GoVersion }} 29 | GOOS: {{ .GoOS }} 30 | GOARCH: {{ .GoArch }} 31 | NumCPU: {{ .NumCPU }} 32 | 33 | ` 34 | 35 | const authorTemplate = `Libri Author Client v{{ .LibriVersion }} 36 | ` 37 | 38 | type librarianConfig struct { 39 | LibriVersion string 40 | GitBranch string 41 | GitRevision string 42 | BuildDate string 43 | Now string 44 | GoVersion string 45 | GoOS string 46 | GoArch string 47 | NumCPU int 48 | } 49 | 50 | type authorConfig struct { 51 | LibriVersion string 52 | } 53 | 54 | // WriteLibrarianBanner writes the librarian banner to the io.Writer. 55 | func WriteLibrarianBanner(w io.Writer) { 56 | config := &librarianConfig{ 57 | LibriVersion: version.Current.Version.String(), 58 | GitBranch: version.Current.GitBranch, 59 | GitRevision: version.Current.GitRevision, 60 | BuildDate: version.Current.BuildDate, 61 | Now: time.Now().UTC().Format(time.RFC3339), 62 | GoVersion: runtime.Version(), 63 | GoOS: runtime.GOOS, 64 | GoArch: runtime.GOARCH, 65 | NumCPU: runtime.NumCPU(), 66 | } 67 | tmpl, err := template.New("librarian-banner").Parse(librarianTemplate) 68 | errors.MaybePanic(err) 69 | err = tmpl.Execute(w, config) 70 | errors.MaybePanic(err) 71 | time.Sleep(10 * time.Millisecond) 72 | } 73 | 74 | // WriteAuthorBanner writes the author banner to the io.Writer. 75 | func WriteAuthorBanner(w io.Writer) { 76 | config := &authorConfig{ 77 | LibriVersion: version.Current.Version.String(), 78 | } 79 | tmpl, err := template.New("author-banner").Parse(authorTemplate) 80 | errors.MaybePanic(err) 81 | err = tmpl.Execute(w, config) 82 | errors.MaybePanic(err) 83 | } 84 | -------------------------------------------------------------------------------- /libri/librarian/client/creators_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/drausin/libri/libri/librarian/api" 7 | "github.com/pkg/errors" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestIntroducerCreator_Create_ok(t *testing.T) { 12 | p := &fixedPool{lc: api.NewLibrarianClient(nil), getAddresses: make(map[string]struct{})} 13 | ic := NewIntroducerCreator(p) 14 | i, err := ic.Create("some address") 15 | assert.Nil(t, err) 16 | assert.NotNil(t, i) 17 | } 18 | func TestIntroducerCreator_Create_err(t *testing.T) { 19 | p := &fixedPool{getErr: errors.New("some error"), getAddresses: make(map[string]struct{})} 20 | ic := NewIntroducerCreator(p) 21 | i, err := ic.Create("some address") 22 | assert.NotNil(t, err) 23 | assert.Nil(t, i) 24 | } 25 | 26 | func TestFinderCreator_Create_ok(t *testing.T) { 27 | p := &fixedPool{lc: api.NewLibrarianClient(nil), getAddresses: make(map[string]struct{})} 28 | fc := NewFinderCreator(p) 29 | f, err := fc.Create("some address") 30 | assert.Nil(t, err) 31 | assert.NotNil(t, f) 32 | } 33 | 34 | func TestFinderCreator_Create_err(t *testing.T) { 35 | p := &fixedPool{getErr: errors.New("some error"), getAddresses: make(map[string]struct{})} 36 | fc := NewFinderCreator(p) 37 | f, err := fc.Create("some address") 38 | assert.NotNil(t, err) 39 | assert.Nil(t, f) 40 | } 41 | 42 | func TestVerifyCreator_Create_ok(t *testing.T) { 43 | p := &fixedPool{lc: api.NewLibrarianClient(nil), getAddresses: make(map[string]struct{})} 44 | fc := NewVerifierCreator(p) 45 | v, err := fc.Create("some address") 46 | assert.Nil(t, err) 47 | assert.NotNil(t, v) 48 | } 49 | 50 | func TestVerifierCreator_Create_err(t *testing.T) { 51 | p := &fixedPool{getErr: errors.New("some error"), getAddresses: make(map[string]struct{})} 52 | fc := NewVerifierCreator(p) 53 | v, err := fc.Create("some address") 54 | assert.NotNil(t, err) 55 | assert.Nil(t, v) 56 | } 57 | 58 | func TestStorerCreator_Create_ok(t *testing.T) { 59 | p := &fixedPool{lc: api.NewLibrarianClient(nil), getAddresses: make(map[string]struct{})} 60 | sc := NewStorerCreator(p) 61 | s, err := sc.Create("some address") 62 | assert.Nil(t, err) 63 | assert.NotNil(t, s) 64 | } 65 | 66 | func TestStorerCreator_Create_err(t *testing.T) { 67 | p := &fixedPool{getErr: errors.New("some error"), getAddresses: make(map[string]struct{})} 68 | sc := NewStorerCreator(p) 69 | s, err := sc.Create("some address") 70 | assert.NotNil(t, err) 71 | assert.Nil(t, s) 72 | } 73 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | "time" 7 | 8 | "github.com/blang/semver" 9 | "github.com/drausin/libri/libri/common/errors" 10 | ) 11 | 12 | // Current contains the current build info. 13 | var Current BuildInfo 14 | 15 | // these variables are populated by ldflags during builds and fall back to population from git repo 16 | // when they're not set (e.g., during tests) 17 | var ( 18 | // GitBranch is the current git branch 19 | GitBranch string 20 | 21 | // GitRevision is the current git commit hash. 22 | GitRevision string 23 | 24 | // BuildDate is the date of the build. 25 | BuildDate string 26 | ) 27 | 28 | var semverString = "0.6.0" 29 | 30 | const ( 31 | develop = "develop" 32 | master = "master" 33 | snapshot = "snapshot" 34 | rc = "rc" 35 | buildDateFormat = "2006-01-02" // ISO 8601 date format 36 | ) 37 | 38 | var branchPrefixes = []string{ 39 | "feature/", 40 | "bugfix/", 41 | } 42 | var releasePrefix = "release/" 43 | 44 | // BuildInfo contains info about the current build. 45 | type BuildInfo struct { 46 | Version semver.Version 47 | GitBranch string 48 | GitRevision string 49 | BuildDate string 50 | } 51 | 52 | func init() { 53 | wd, err := os.Getwd() 54 | errors.MaybePanic(err) 55 | g := git{dir: wd} 56 | 57 | if GitBranch == "" { 58 | GitBranch = g.Branch() 59 | } 60 | if GitRevision == "" { 61 | GitRevision, err = g.Commit() 62 | errors.MaybePanic(err) 63 | } 64 | if BuildDate == "" { 65 | BuildDate = time.Now().UTC().Format(buildDateFormat) 66 | } 67 | Version := semver.MustParse(semverString) 68 | if GitBranch == master { 69 | // don't override pre-release or build flags it's actually a release 70 | } else if strings.HasPrefix(GitBranch, releasePrefix) { 71 | Version.Pre = []semver.PRVersion{{VersionStr: rc}} 72 | Version.Build = []string{GitRevision} 73 | } else if GitBranch == develop { 74 | Version.Pre = []semver.PRVersion{{VersionStr: snapshot}} 75 | } else { 76 | Version.Pre = []semver.PRVersion{{VersionStr: stripPrefixes(GitBranch)}} 77 | } 78 | Current = BuildInfo{ 79 | Version: Version, 80 | GitBranch: GitBranch, 81 | GitRevision: GitRevision, 82 | BuildDate: BuildDate, 83 | } 84 | } 85 | 86 | func stripPrefixes(branch string) string { 87 | for _, prefix := range branchPrefixes { 88 | if strings.HasPrefix(branch, prefix) { 89 | return strings.TrimPrefix(branch, prefix) 90 | } 91 | } 92 | return branch 93 | } 94 | -------------------------------------------------------------------------------- /libri/author/keychain/keychain.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // source: libri/author/keychain/keychain.proto 3 | 4 | /* 5 | Package keychain is a generated protocol buffer package. 6 | 7 | It is generated from these files: 8 | libri/author/keychain/keychain.proto 9 | 10 | It has these top-level messages: 11 | StoredKeychain 12 | */ 13 | package keychain 14 | 15 | import proto "github.com/golang/protobuf/proto" 16 | import fmt "fmt" 17 | import math "math" 18 | 19 | // Reference imports to suppress errors if they are not otherwise used. 20 | var _ = proto.Marshal 21 | var _ = fmt.Errorf 22 | var _ = math.Inf 23 | 24 | // This is a compile-time assertion to ensure that this generated file 25 | // is compatible with the proto package it is being compiled against. 26 | // A compilation error at this line likely means your copy of the 27 | // proto package needs to be updated. 28 | const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package 29 | 30 | type StoredKeychain struct { 31 | PrivateKeys [][]byte `protobuf:"bytes,1,rep,name=privateKeys,proto3" json:"privateKeys,omitempty"` 32 | } 33 | 34 | func (m *StoredKeychain) Reset() { *m = StoredKeychain{} } 35 | func (m *StoredKeychain) String() string { return proto.CompactTextString(m) } 36 | func (*StoredKeychain) ProtoMessage() {} 37 | func (*StoredKeychain) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } 38 | 39 | func (m *StoredKeychain) GetPrivateKeys() [][]byte { 40 | if m != nil { 41 | return m.PrivateKeys 42 | } 43 | return nil 44 | } 45 | 46 | func init() { 47 | proto.RegisterType((*StoredKeychain)(nil), "keychain.StoredKeychain") 48 | } 49 | 50 | func init() { proto.RegisterFile("libri/author/keychain/keychain.proto", fileDescriptor0) } 51 | 52 | var fileDescriptor0 = []byte{ 53 | // 101 bytes of a gzipped FileDescriptorProto 54 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0xc9, 0xc9, 0x4c, 0x2a, 55 | 0xca, 0xd4, 0x4f, 0x2c, 0x2d, 0xc9, 0xc8, 0x2f, 0xd2, 0xcf, 0x4e, 0xad, 0x4c, 0xce, 0x48, 0xcc, 56 | 0xcc, 0x83, 0x33, 0xf4, 0x0a, 0x8a, 0xf2, 0x4b, 0xf2, 0x85, 0x38, 0x60, 0x7c, 0x25, 0x23, 0x2e, 57 | 0xbe, 0xe0, 0x92, 0xfc, 0xa2, 0xd4, 0x14, 0x6f, 0xa8, 0x88, 0x90, 0x02, 0x17, 0x77, 0x41, 0x51, 58 | 0x66, 0x59, 0x62, 0x49, 0xaa, 0x77, 0x6a, 0x65, 0xb1, 0x04, 0xa3, 0x02, 0xb3, 0x06, 0x4f, 0x10, 59 | 0xb2, 0x50, 0x12, 0x1b, 0xd8, 0x10, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x04, 0xa8, 0x22, 60 | 0x2a, 0x6c, 0x00, 0x00, 0x00, 61 | } 62 | -------------------------------------------------------------------------------- /libri/common/subscribe/filter_test.go: -------------------------------------------------------------------------------- 1 | package subscribe 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "testing" 7 | 8 | "github.com/drausin/libri/libri/librarian/api" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestToFromAPI(t *testing.T) { 13 | rng := rand.New(rand.NewSource(0)) 14 | f1 := newFilter([][]byte{}, 0.75, rng) 15 | a, err := ToAPI(f1) 16 | assert.Nil(t, err) 17 | f2, err := FromAPI(a) 18 | assert.Nil(t, err) 19 | assert.Equal(t, f1, f2) 20 | } 21 | 22 | func TestFromAPI_err(t *testing.T) { 23 | f, err := FromAPI(&api.BloomFilter{Encoded: []byte{}}) 24 | assert.NotNil(t, err) 25 | assert.Nil(t, f) 26 | } 27 | 28 | /* 29 | // this "test" is helpful in empirically determining reasonable n and fp params for bloom filters 30 | func TestEmpiricalFilterParameters(t *testing.T) { 31 | rng := rand.New(rand.NewSource(0)) 32 | fps := []float64{0.0001, 0.001, 0.01, 0.1, 0.3, 0.5, 0.75, 0.9} 33 | ns := []uint{1, 2, 5, 10, 25, 50, 100} 34 | for _, n := range ns { 35 | for _, fp := range fps { 36 | m, k := bloom.EstimateParameters(n, fp) 37 | filter := bloom.New(m, k) 38 | for c := uint(0); c < n; c++ { 39 | filter.Add(api.RandBytes(rng, api.ECPubKeyLength)) 40 | } 41 | fpCount, nTrials := 0, 100000 42 | for c := 0; c < nTrials; c++ { 43 | if filter.Test(api.RandBytes(rng, api.ECPubKeyLength)) { 44 | fpCount++ 45 | } 46 | } 47 | actualFp := float64(fpCount) / float64(nTrials) 48 | log.Printf("n: %d, est fp: %f, m: %d, k: %d, actual fp: %f\n", n, fp, m, 49 | k, actualFp) 50 | } 51 | } 52 | } 53 | */ 54 | 55 | func TestNewFPFilter(t *testing.T) { 56 | rng := rand.New(rand.NewSource(0)) 57 | nTrials := 100000 58 | 59 | // check that for the actual FP rate is >= target FP rate - tolerance 60 | tolerance := 0.05 61 | for _, targetFP := range []float64{0.3, 0.5, 0.75, 0.9, 1.0} { 62 | filter := newFilter([][]byte{}, targetFP, rng) 63 | 64 | // measure FP rate 65 | fpCount := 0 66 | for c := 0; c < nTrials; c++ { 67 | if filter.Test(api.RandBytes(rng, api.ECPubKeyLength)) { 68 | fpCount++ 69 | } 70 | } 71 | measuredFP := float64(fpCount) / float64(nTrials) 72 | info := fmt.Sprintf("target fp: %f, measured fp: %f", targetFP, measuredFP) 73 | assert.True(t, measuredFP >= targetFP-tolerance, info) 74 | } 75 | 76 | } 77 | 78 | func TestAlwaysInFilter(t *testing.T) { 79 | rng := rand.New(rand.NewSource(0)) 80 | f := alwaysInFilter() 81 | for c := 0; c < 10000; c++ { 82 | in := f.Test(api.RandBytes(rng, api.ECPubKeyLength)) 83 | assert.True(t, in) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /libri/author/io/page/storage.go: -------------------------------------------------------------------------------- 1 | package page 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/drausin/libri/libri/common/id" 7 | "github.com/drausin/libri/libri/common/storage" 8 | "github.com/drausin/libri/libri/librarian/api" 9 | ) 10 | 11 | var ( 12 | // ErrUnexpectedDocContent indicates that a document is not of the expected content type 13 | // (e.g., an Entry when expecing a Page). 14 | ErrUnexpectedDocContent = errors.New("unexpected document content") 15 | 16 | // ErrMissingPage indicates when a page was expected to be stored but was not found. 17 | ErrMissingPage = errors.New("missing page") 18 | ) 19 | 20 | // Storer stores pages to an inner storage.DocumentStorer. 21 | type Storer interface { 22 | // Store writes pages to inner storage and returns a slice of their keys. 23 | Store(pages chan *api.Page) ([]id.ID, error) 24 | } 25 | 26 | // Loader loads pages from an inner storage.DocmentLoader. 27 | type Loader interface { 28 | // Load reads pages from inner storage and sends them on the supplied pages channel. A 29 | // signal (or closing) on the abort channel interrupts the loading. 30 | Load(keys []id.ID, pages chan *api.Page, abort chan struct{}) error 31 | } 32 | 33 | // StorerLoader stores and loads pages from an inner storage.DocumentStorerLoader. 34 | type StorerLoader interface { 35 | Storer 36 | Loader 37 | } 38 | 39 | type storerLoader struct { 40 | inner storage.DocumentSLD 41 | } 42 | 43 | // NewStorerLoader creates a new StorerLoader instance from an inner storage.DocumentSLD 44 | // instance. 45 | func NewStorerLoader(inner storage.DocumentSLD) StorerLoader { 46 | return &storerLoader{ 47 | inner: inner, 48 | } 49 | } 50 | 51 | func (s *storerLoader) Store(pages chan *api.Page) ([]id.ID, error) { 52 | keys := make([]id.ID, 0) 53 | for page := range pages { 54 | doc, key, err := api.GetPageDocument(page) 55 | if err != nil { 56 | return nil, err 57 | } 58 | if err := s.inner.Store(key, doc); err != nil { 59 | return nil, err 60 | } 61 | keys = append(keys, key) 62 | } 63 | return keys, nil 64 | } 65 | 66 | func (s *storerLoader) Load(keys []id.ID, pages chan *api.Page, abort chan struct{}) error { 67 | for _, key := range keys { 68 | doc, err := s.inner.Load(key) 69 | if err != nil { 70 | return err 71 | } 72 | if doc == nil { 73 | return ErrMissingPage 74 | } 75 | docPage, ok := doc.Contents.(*api.Document_Page) 76 | if !ok { 77 | return ErrUnexpectedDocContent 78 | } 79 | select { 80 | case <-abort: 81 | return nil 82 | default: 83 | pages <- docPage.Page 84 | if err := s.inner.Delete(key); err != nil { // no need to keep page around 85 | return err 86 | } 87 | } 88 | } 89 | return nil 90 | } 91 | -------------------------------------------------------------------------------- /libri/librarian/client/creators.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "github.com/drausin/libri/libri/librarian/api" 5 | ) 6 | 7 | // IntroducerCreator creates api.Introducers. 8 | type IntroducerCreator interface { 9 | // Create creates an api.Introducer from the api.Connector. 10 | Create(address string) (api.Introducer, error) 11 | } 12 | 13 | type introducerCreator struct { 14 | clients Pool 15 | } 16 | 17 | // NewIntroducerCreator creates a new IntroducerCreator. 18 | func NewIntroducerCreator(clients Pool) IntroducerCreator { 19 | return &introducerCreator{clients} 20 | } 21 | 22 | func (c *introducerCreator) Create(address string) (api.Introducer, error) { 23 | lc, err := c.clients.Get(address) 24 | if err != nil { 25 | return nil, err 26 | } 27 | return lc.(api.Introducer), nil 28 | } 29 | 30 | // FinderCreator creates api.Finders. 31 | type FinderCreator interface { 32 | // Create creates an api.Finder from the api.Connector. 33 | Create(address string) (api.Finder, error) 34 | } 35 | 36 | type finderCreator struct { 37 | clients Pool 38 | } 39 | 40 | // NewFinderCreator creates a new FinderCreator. 41 | func NewFinderCreator(clients Pool) FinderCreator { 42 | return &finderCreator{clients} 43 | } 44 | 45 | func (c *finderCreator) Create(address string) (api.Finder, error) { 46 | lc, err := c.clients.Get(address) 47 | if err != nil { 48 | return nil, err 49 | } 50 | return lc.(api.Finder), nil 51 | } 52 | 53 | // VerifierCreator creates api.Verifiers. 54 | type VerifierCreator interface { 55 | // Create creates an api.Verifier from the api.Connector. 56 | Create(address string) (api.Verifier, error) 57 | } 58 | 59 | type verifierCreator struct { 60 | clients Pool 61 | } 62 | 63 | // NewVerifierCreator creates a new FinderCreator. 64 | func NewVerifierCreator(clients Pool) VerifierCreator { 65 | return &verifierCreator{clients} 66 | } 67 | 68 | func (c *verifierCreator) Create(address string) (api.Verifier, error) { 69 | lc, err := c.clients.Get(address) 70 | if err != nil { 71 | return nil, err 72 | } 73 | return lc.(api.Verifier), nil 74 | } 75 | 76 | // StorerCreator creates api.Storers. 77 | type StorerCreator interface { 78 | // Create creates an api.Storer from the api.Connector. 79 | Create(address string) (api.Storer, error) 80 | } 81 | 82 | type storerCreator struct { 83 | clients Pool 84 | } 85 | 86 | // NewStorerCreator creates a new StorerCreator. 87 | func NewStorerCreator(clients Pool) StorerCreator { 88 | return &storerCreator{clients} 89 | } 90 | 91 | func (c *storerCreator) Create(address string) (api.Storer, error) { 92 | lc, err := c.clients.Get(address) 93 | if err != nil { 94 | return nil, err 95 | } 96 | return lc.(api.Storer), nil 97 | } 98 | -------------------------------------------------------------------------------- /libri/acceptance/public-testnet.md: -------------------------------------------------------------------------------- 1 | ## Libri public testnet 2 | 3 | The libri public test network has a small number of nodes running. We are actively seeking people 4 | interested in helping test it, but please reach out to coordinate before sending any large load to 5 | it or spinning up your own Libri peers to join the network. Get in touch via `contact` AT `libri.io`. 6 | 7 | The current testnet seeds addresses are: 8 | 9 | LIBRI_TESTNET_SEEDS='35.227.54.0:30100,35.237.0.109:30104,35.237.27.67:30108,35.237.27.67:30112' 10 | 11 | 12 | ### Simple file upload/download 13 | 14 | The simplest way to run Libri Author (client) commands is from within a Libri Docker container. 15 | Start and enter a new container via 16 | 17 | docker run --rm -it --entrypoint /bin/bash daedalus2718/libri:snapshot 18 | 19 | Once in, set the `LIBRI_TESTNET_SEEDS` via the expression above and then define some Libri 20 | environment variables so we don't have to pass them in as CLI args. 21 | 22 | export LIBRI_KEYCHAINSDIR='/tmp/libri/keys' 23 | export LIBRI_DATADIR='/tmp/libri/data' 24 | export LIBRI_PASSPHRASE='my super secret thingy' 25 | LOG_FILE=${LIBRI_DATADIR}/up.log 26 | 27 | Confirm we can talk to the librarians. 28 | 29 | libri test health -a ${LIBRI_TESTNET_SEEDS} 30 | 31 | Initialize the author keys and create the test file. 32 | 33 | libri author init 34 | 35 | mkdir -p ${LIBRI_DATADIR} 36 | echo 'Hello Libri!' > ${LIBRI_DATADIR}/test.up.txt 37 | 38 | Upload the file and get the resulting envelope ID. 39 | 40 | libri author upload -a ${LIBRI_TESTNET_SEEDS} -f ${LIBRI_DATADIR}/test.up.txt 2>&1 | tee ${LOG_FILE} 41 | 42 | # grab envelope key from the log 43 | ENVELOPE_KEY=$(grep 'envelope_key' ${LOG_FILE} | sed -E 's/.*"envelope_key": "([^ "]*).*/\1/g') 44 | echo "uploaded with envelope key '${ENVELOPE_KEY}'" 45 | 46 | Download the file from the envelope ID and confirm it's the same as the uploaded file. 47 | 48 | libri author download -a ${LIBRI_TESTNET_SEEDS} -f ${LIBRI_DATADIR}/test.down.txt -e ${ENVELOPE_KEY} 49 | 50 | cat ${LIBRI_DATADIR}/test.down.txt 51 | md5sum ${LIBRI_DATADIR}/test.* 52 | 53 | When you're satisfied, exit the container. 54 | 55 | exit 56 | 57 | 58 | ### Spinning up a fleet of peers to join the Libri testnet. 59 | 60 | Follow the GCP (cloud) directions on the [cloud deployment README] to spin up a stand-along cluster. 61 | After running the `LIBRI_TESTNET_PEERS` command above, change the Librarian bootstrap addresses 62 | to these testnet peers and re-apply Kubernetes config. 63 | 64 | sed -i -E "s/^(.*--bootstraps).*$/\1 '${LIBRI_TESTNET_SEEDS}'/g" libri.yml 65 | kubectl apply -f libri.yml -------------------------------------------------------------------------------- /libri/common/storage/storer_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "bytes" 5 | "math/rand" 6 | "testing" 7 | 8 | "github.com/drausin/libri/libri/common/db" 9 | "github.com/drausin/libri/libri/common/id" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestNewKVDBStorerLoader(t *testing.T) { 14 | // just test that this doesn't panic 15 | NewKVDBStorerLoader(nil, nil, nil, nil) 16 | } 17 | 18 | func TestKvdbSLD_StoreLoadDelete(t *testing.T) { 19 | rng := rand.New(rand.NewSource(int64(0))) 20 | cases := []struct { 21 | ns []byte 22 | key []byte 23 | value []byte 24 | }{ 25 | {[]byte("ns"), bytes.Repeat([]byte{0}, id.Length), []byte{0}}, 26 | {[]byte("ns"), bytes.Repeat([]byte{0}, id.Length), 27 | bytes.Repeat([]byte{255}, 1024)}, 28 | {[]byte("test namespace"), id.NewPseudoRandom(rng).Bytes(), []byte("test value")}, 29 | } 30 | 31 | kvdb, cleanup, err := db.NewTempDirRocksDB() 32 | defer cleanup() 33 | defer kvdb.Close() 34 | assert.Nil(t, err) 35 | for _, c := range cases { 36 | sld := NewKVDBStorerLoaderDeleter( 37 | c.ns, 38 | kvdb, 39 | NewMaxLengthChecker(256), 40 | NewMaxLengthChecker(1024), 41 | ) 42 | err := sld.Store(c.key, c.value) 43 | assert.Nil(t, err) 44 | 45 | loaded, err := sld.Load(c.key) 46 | assert.Nil(t, err) 47 | assert.Equal(t, c.value, loaded) 48 | 49 | err = sld.Delete(c.key) 50 | assert.Nil(t, err) 51 | } 52 | } 53 | 54 | func TestKvdbSLD_Load_err(t *testing.T) { 55 | sld := &kvdbSLD{kc: NewMaxLengthChecker(1)} 56 | value, err := sld.Load([]byte("some long key")) 57 | assert.NotNil(t, err) 58 | assert.Nil(t, value) 59 | } 60 | 61 | func TestKvdbSLD_Delete_err(t *testing.T) { 62 | sld := &kvdbSLD{kc: NewMaxLengthChecker(1)} 63 | err := sld.Delete([]byte("some long key")) 64 | assert.NotNil(t, err) 65 | } 66 | 67 | func TestKvdbSLD_Iterate(t *testing.T) { 68 | kvdb, cleanup, err := db.NewTempDirRocksDB() 69 | defer cleanup() 70 | defer kvdb.Close() 71 | assert.Nil(t, err) 72 | 73 | ns := []byte("key") 74 | sld := NewKVDBStorerLoaderDeleter(ns, kvdb, NewMaxLengthChecker(256), NewMaxLengthChecker(1024)) 75 | 76 | vals := map[string][]byte{ 77 | "1": []byte("val1"), 78 | "2": []byte("val2"), 79 | "3": []byte("val3"), 80 | } 81 | for key, val := range vals { 82 | err = sld.Store([]byte(key), val) 83 | assert.Nil(t, err) 84 | } 85 | 86 | nIters := 0 87 | callback := func(key, value []byte) { 88 | nIters++ 89 | expected, in := vals[string(key)] 90 | assert.True(t, in) 91 | assert.Equal(t, expected, value) 92 | } 93 | lb, ub := []byte("0"), []byte("9") 94 | err = sld.Iterate(lb, ub, make(chan struct{}), callback) 95 | assert.Nil(t, err) 96 | assert.Equal(t, len(vals), nIters) 97 | } 98 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing 2 | 3 | ### Developing 4 | Fork the project and clone to your development machine. Get the Golang dependencies onto your local 5 | development machine via 6 | ```bash 7 | make get-deps 8 | ``` 9 | We use [dep](https://github.com/golang/dep) for vendoring. You will also need Docker installed. 10 | 11 | ### Exploring 12 | 13 | The [acceptance tests](libri/acceptance/librarian_test.go) and 14 | [librarian type](libri/librarian/server/server.go) are good places from which to start exploring 15 | the codebase. See the [libri librarian start](libri/cmd/start.go) and 16 | [libri author upload](libri/cmd/upload.go) commands for example the CLI entrypoints. 17 | 18 | ### Testing 19 | The simplest way to run the tests is from within a build container, which has all the required 20 | binaries (e.g., RocksDB) already installed and linked. Our [CI](.circleci/config.yml) uses it. 21 | 22 | *N.B., currently the local build container is a bit slower/more laggy than I would like. Improvement 23 | suggestions are very welcome.* 24 | 25 | The build container mounts 26 | - `~/.go/src`, so your libri code and its dependencies are available 27 | - `~/.bashrc`, so your build container shell is nice and familiar 28 | - `~/.gitconfig`, so you can do all your favorite git things 29 | 30 | To start it, run 31 | ```bash 32 | ./scripts/run-build-container.sh 33 | ``` 34 | which brings you into the build container. From there you can run most things you'd care about. 35 | The most common `make` targets are 36 | - `make test`: run all tests 37 | - `make acceptance`: run the acceptance tests 38 | - `make lint-diff`: lint the uncommitted changes 39 | - `make lint`: lint the entire repo 40 | - `make fix`: run `goimports` & `go fmt` on repo 41 | Of course you can also run normal `go` tool commands or any other shell command you like. 42 | 43 | You won't be able to run things requiring `docker run` (which you can't do from within a container), 44 | including 45 | - `make demo` (or the underlying [local-demo.sh](libri/acceptance/local-demo.sh)) 46 | - starting a local Kubernetes cluster from `deloy/cloud/kubernetes/libri.yml` 47 | 48 | If you want to run tests locally (i.e., not in the build container), you'll have do the local 49 | installation (see below). 50 | 51 | ### Local OSX installation 52 | 53 | This requires a tad more setup and obviously isn't as isolated as the build container, but it's 54 | faster since it's ultimately just your local machine. 55 | 56 | First [install RocksDB](https://github.com/facebook/rocksdb/blob/master/INSTALL.md) 57 | ```$bash 58 | brew install rocksdb 59 | ``` 60 | Then build the [gorocksdb](https://github.com/tecbot/gorocksdb) driver 61 | ```$bash 62 | ./build/install-gorocksdb.sh 63 | ``` 64 | 65 | -------------------------------------------------------------------------------- /libri/common/storage/checker_test.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha256" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestEmptyChecker_Check_ok(t *testing.T) { 12 | kc := NewMaxLengthChecker(8) 13 | cases := [][]byte{ 14 | []byte("key"), 15 | []byte("value"), 16 | []byte("ns"), 17 | []byte{0}, 18 | bytes.Repeat([]byte{0}, 7), 19 | bytes.Repeat([]byte{255}, 8), 20 | } 21 | for _, c := range cases { 22 | assert.Nil(t, kc.Check(c)) 23 | } 24 | } 25 | 26 | func TestEmptyChecker_Check_err(t *testing.T) { 27 | kc := NewMaxLengthChecker(8) 28 | cases := [][]byte{ 29 | nil, 30 | []byte(nil), 31 | []byte{}, 32 | bytes.Repeat([]byte{255}, 16), 33 | } 34 | for _, c := range cases { 35 | assert.NotNil(t, kc.Check(c)) 36 | } 37 | } 38 | 39 | func TestMaxLengthChecker_Check_ok(t *testing.T) { 40 | kc := NewMaxLengthChecker(8) 41 | cases := [][]byte{ 42 | []byte("key"), 43 | []byte("value"), 44 | []byte("ns"), 45 | []byte{0}, 46 | bytes.Repeat([]byte{0}, 7), 47 | bytes.Repeat([]byte{255}, 8), 48 | } 49 | for _, c := range cases { 50 | assert.Nil(t, kc.Check(c)) 51 | } 52 | } 53 | 54 | func TestMaxLengthChecker_Check_err(t *testing.T) { 55 | kc := NewMaxLengthChecker(8) 56 | cases := [][]byte{ 57 | nil, 58 | []byte(nil), 59 | []byte{}, 60 | []byte("too long value"), 61 | []byte("too long key"), 62 | []byte("too long namespace"), 63 | bytes.Repeat([]byte{255}, 16), 64 | } 65 | for _, c := range cases { 66 | assert.NotNil(t, kc.Check(c)) 67 | } 68 | } 69 | 70 | func TestExactLengthChecker_Check_ok(t *testing.T) { 71 | kc := NewExactLengthChecker(8) 72 | cases := [][]byte{ 73 | []byte("somekey1"), 74 | []byte("someval1"), 75 | bytes.Repeat([]byte{0}, 8), 76 | bytes.Repeat([]byte{255}, 8), 77 | } 78 | for _, c := range cases { 79 | assert.Nil(t, kc.Check(c)) 80 | } 81 | } 82 | 83 | func TestExactLengthChecker_Check_err(t *testing.T) { 84 | kc := NewExactLengthChecker(8) 85 | cases := [][]byte{ 86 | nil, 87 | []byte(nil), 88 | []byte{}, 89 | []byte("short"), 90 | []byte("too long value"), 91 | bytes.Repeat([]byte{255}, 7), 92 | bytes.Repeat([]byte{255}, 9), 93 | } 94 | for _, c := range cases { 95 | assert.NotNil(t, kc.Check(c)) 96 | } 97 | } 98 | 99 | func TestHashChecker_Check_ok(t *testing.T) { 100 | c := NewHashKeyValueChecker() 101 | v := []byte("some test value") 102 | k := sha256.Sum256(v) 103 | assert.Nil(t, c.Check(k[:], v)) 104 | } 105 | 106 | func TestHashChecker_Check_err(t *testing.T) { 107 | c := NewHashKeyValueChecker() 108 | assert.NotNil(t, c.Check([]byte{0, 1, 2}, []byte{0, 1, 2})) 109 | } 110 | -------------------------------------------------------------------------------- /libri/author/io/enc/benchmarks_test.go: -------------------------------------------------------------------------------- 1 | package enc 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "github.com/drausin/libri/libri/common/errors" 8 | ) 9 | 10 | const ( 11 | KB = 1024 12 | MB = 1024 * KB 13 | ) 14 | 15 | var ( 16 | smallPlaintextSizes = []int{128} 17 | mediumPlaintextSizes = []int{KB} 18 | largePlaintextSizes = []int{MB} 19 | ) 20 | 21 | var benchmarkCases = []struct { 22 | name string 23 | plaintextSizes []int 24 | }{ 25 | {"small", smallPlaintextSizes}, 26 | {"medium", mediumPlaintextSizes}, 27 | {"large", largePlaintextSizes}, 28 | } 29 | 30 | func BenchmarkEncrypt(b *testing.B) { 31 | for _, c := range benchmarkCases { 32 | b.Run(c.name, func(b *testing.B) { benchmarkEncrypt(b, c.plaintextSizes) }) 33 | } 34 | } 35 | 36 | func BenchmarkDecrypt(b *testing.B) { 37 | for _, c := range benchmarkCases { 38 | b.Run(c.name, func(b *testing.B) { benchmarkDecrypt(b, c.plaintextSizes) }) 39 | } 40 | } 41 | 42 | func benchmarkEncrypt(b *testing.B, plaintextSizes []int) { 43 | b.StopTimer() 44 | rng := rand.New(rand.NewSource(0)) 45 | keys := NewPseudoRandomEEK(rng) 46 | encrypter, err := NewEncrypter(keys) 47 | errors.MaybePanic(err) 48 | 49 | plaintexts := make([][]byte, len(plaintextSizes)) 50 | totBytes := int64(0) 51 | for i, plaintextSize := range plaintextSizes { 52 | plaintexts[i] = make([]byte, plaintextSize) 53 | _, err = rng.Read(plaintexts[i]) 54 | errors.MaybePanic(err) 55 | totBytes += int64(plaintextSize) 56 | } 57 | 58 | b.SetBytes(totBytes) 59 | b.StartTimer() 60 | for n := 0; n < b.N; n++ { 61 | for _, plaintext := range plaintexts { 62 | _, err = encrypter.Encrypt(plaintext, 0) 63 | errors.MaybePanic(err) 64 | } 65 | } 66 | } 67 | 68 | func benchmarkDecrypt(b *testing.B, plaintextSizes []int) { 69 | b.StopTimer() 70 | rng := rand.New(rand.NewSource(0)) 71 | keys := NewPseudoRandomEEK(rng) 72 | 73 | decrypter, err := NewDecrypter(keys) 74 | errors.MaybePanic(err) 75 | 76 | encrypter, err := NewEncrypter(keys) 77 | errors.MaybePanic(err) 78 | 79 | ciphertexts := make([][]byte, len(plaintextSizes)) 80 | totBytes := int64(0) 81 | for i, plaintextSize := range plaintextSizes { 82 | 83 | plaintext := make([]byte, plaintextSize) 84 | _, err = rng.Read(plaintext) 85 | errors.MaybePanic(err) 86 | 87 | ciphertexts[i], err = encrypter.Encrypt(plaintext, 0) 88 | errors.MaybePanic(err) 89 | totBytes += int64(len(ciphertexts[i])) 90 | } 91 | 92 | b.SetBytes(totBytes) 93 | b.StartTimer() 94 | for n := 0; n < b.N; n++ { 95 | for _, ciphertext := range ciphertexts { 96 | _, err = decrypter.Decrypt(ciphertext, 0) 97 | errors.MaybePanic(err) 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /libri/common/subscribe/subscription.go: -------------------------------------------------------------------------------- 1 | package subscribe 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/drausin/libri/libri/librarian/api" 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | var ( 11 | // ErrOutOfBoundsFPRate indicates when the false positive rate is not in (0, 1]. 12 | ErrOutOfBoundsFPRate = errors.New("false positive rate out of (0, 1] bounds") 13 | 14 | // ErrNilPublicKeys indicates when either the author or reader public keys are nil. 15 | ErrNilPublicKeys = errors.New("nil public keys") 16 | ) 17 | 18 | // NewSubscription returns a new subscription with filters for the given author and reader public 19 | // keys and false positive rates. Users should recall that the overall subscription false positive 20 | // rate will be the product of the author and reader false positive rates. 21 | func NewSubscription( 22 | authorPubs [][]byte, 23 | authorFp float64, 24 | readerPubs [][]byte, 25 | readerFp float64, 26 | rng *rand.Rand, 27 | ) (*api.Subscription, error) { 28 | 29 | if readerFp <= 0.0 || readerFp > 1.0 || authorFp <= 0.0 || authorFp > 1.0 { 30 | return nil, ErrOutOfBoundsFPRate 31 | } 32 | if authorPubs == nil || readerPubs == nil { 33 | return nil, ErrNilPublicKeys 34 | } 35 | authorFilter, err := ToAPI(newFilter(authorPubs, authorFp, rng)) 36 | if err != nil { 37 | return nil, err 38 | } 39 | readerFilter, err := ToAPI(newFilter(readerPubs, readerFp, rng)) 40 | if err != nil { 41 | return nil, err 42 | } 43 | return &api.Subscription{ 44 | AuthorPublicKeys: authorFilter, 45 | ReaderPublicKeys: readerFilter, 46 | }, nil 47 | } 48 | 49 | // NewAuthorSubscription creates an *api.Subscription using the given author public keys and false 50 | // positive rate with a 1.0 false positive rate for reader keys. This is useful when one doesn't 51 | // want to filter on any reader keys. 52 | func NewAuthorSubscription(authorPubs [][]byte, fp float64, rng *rand.Rand) ( 53 | *api.Subscription, error) { 54 | return NewSubscription(authorPubs, fp, [][]byte{}, 1.0, rng) 55 | } 56 | 57 | // NewReaderSubscription creates an *api.Subscription using the given reader public keys and false 58 | // positive rate with a 1.0 false positive rate for author keys. This is useful when one doesn't 59 | // want to filter on any author keys. 60 | func NewReaderSubscription(readerPubs [][]byte, fp float64, rng *rand.Rand) ( 61 | *api.Subscription, error) { 62 | return NewSubscription([][]byte{}, 1.0, readerPubs, fp, rng) 63 | } 64 | 65 | // NewFPSubscription creates an *api.Subscription with the given false positive rate on the author 66 | // keys and a 1.0 false positive rate on the reader keys. 67 | func NewFPSubscription(fp float64, rng *rand.Rand) (*api.Subscription, error) { 68 | return NewSubscription([][]byte{}, fp, [][]byte{}, 1.0, rng) 69 | } 70 | -------------------------------------------------------------------------------- /libri/author/config_test.go: -------------------------------------------------------------------------------- 1 | package author 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | 7 | "github.com/drausin/libri/libri/author/io/print" 8 | "github.com/drausin/libri/libri/author/io/publish" 9 | "github.com/drausin/libri/libri/common/parse" 10 | "github.com/stretchr/testify/assert" 11 | "go.uber.org/zap/zapcore" 12 | ) 13 | 14 | func TestNewDefaultConfig(t *testing.T) { 15 | c := NewDefaultConfig() 16 | assert.NotEmpty(t, c.DataDir) 17 | assert.NotEmpty(t, c.DbDir) 18 | assert.NotEmpty(t, c.KeychainDir) 19 | assert.NotEmpty(t, c.LibrarianAddrs) 20 | assert.NotEmpty(t, c.Print) 21 | assert.NotEmpty(t, c.Publish) 22 | } 23 | 24 | func TestConfig_WithDataDir(t *testing.T) { 25 | c1, c2, c3 := &Config{}, &Config{}, &Config{} 26 | c1.WithDefaultDataDir() 27 | assert.Equal(t, c1.DataDir, c2.WithDataDir("").DataDir) 28 | assert.NotEqual(t, c1.DataDir, c3.WithDataDir("/some/other/dir").DataDir) 29 | } 30 | 31 | func TestConfig_WithDBDir(t *testing.T) { 32 | c1, c2, c3 := &Config{}, &Config{}, &Config{} 33 | c1.WithDefaultDBDir() 34 | assert.Equal(t, c1.DbDir, c2.WithDBDir("").DbDir) 35 | assert.NotEqual(t, c1.DbDir, c3.WithDBDir("/some/other/dir").DbDir) 36 | } 37 | 38 | func TestConfig_WithKeychainDir(t *testing.T) { 39 | c1, c2, c3 := &Config{}, &Config{}, &Config{} 40 | c1.WithDefaultKeychainDir() 41 | assert.Equal(t, c1.KeychainDir, c2.WithKeychainDir("").KeychainDir) 42 | assert.NotEqual(t, c1.KeychainDir, c3.WithKeychainDir("/some/other/dir").KeychainDir) 43 | } 44 | 45 | func TestConfig_WithBootstrapAddrs(t *testing.T) { 46 | c1, c2, c3 := &Config{}, &Config{}, &Config{} 47 | c1.WithDefaultLibrarianAddrs() 48 | assert.Equal(t, c1.LibrarianAddrs, c2.WithLibrarianAddrs(nil).LibrarianAddrs) 49 | c3Addr, err := parse.Addr("localhost", 1234) 50 | assert.Nil(t, err) 51 | assert.NotEqual(t, 52 | c1.LibrarianAddrs, 53 | c3.WithLibrarianAddrs([]*net.TCPAddr{c3Addr}).LibrarianAddrs, 54 | ) 55 | } 56 | 57 | func TestConfig_WithPrint(t *testing.T) { 58 | c1, c2, c3 := &Config{}, &Config{}, &Config{} 59 | c1.WithDefaultPrint() 60 | assert.Equal(t, c1.Print, c2.WithPrint(nil).Print) 61 | assert.NotEqual(t, 62 | c1.Print, 63 | c3.WithPrint(&print.Parameters{Parallelism: 1}).Print, 64 | ) 65 | } 66 | 67 | func TestConfig_WithPublish(t *testing.T) { 68 | c1, c2, c3 := &Config{}, &Config{}, &Config{} 69 | c1.WithDefaultPublish() 70 | assert.Equal(t, c1.Print, c2.WithPublish(nil).Print) 71 | assert.NotEqual(t, 72 | c1.Publish, 73 | c3.WithPublish(&publish.Parameters{PutParallelism: 1}).Publish, 74 | ) 75 | } 76 | 77 | func TestConfig_WithLogLevel(t *testing.T) { 78 | c1, c2, c3 := &Config{}, &Config{}, &Config{} 79 | c1.WithDefaultLogLevel() 80 | assert.Equal(t, c1.LogLevel, c2.WithLogLevel(0).LogLevel) 81 | assert.NotEqual(t, 82 | c1.LogLevel, 83 | c3.WithLogLevel(zapcore.DebugLevel).LogLevel, 84 | ) 85 | } 86 | -------------------------------------------------------------------------------- /libri/librarian/server/replicate/metrics.go: -------------------------------------------------------------------------------- 1 | package replicate 2 | 3 | import ( 4 | "github.com/drausin/libri/libri/common/errors" 5 | prom "github.com/prometheus/client_golang/prometheus" 6 | ) 7 | 8 | const ( 9 | promNamespace = "libri" 10 | promSubsystem = "server_replicator" 11 | ) 12 | 13 | type result int 14 | 15 | const ( 16 | succeeded result = iota 17 | exhausted 18 | errored 19 | ) 20 | 21 | func (r result) String() string { 22 | switch r { 23 | case succeeded: 24 | return "succeeded" 25 | case exhausted: 26 | return "exhausted" 27 | case errored: 28 | return "errored" 29 | } 30 | panic("should never get here") 31 | } 32 | 33 | type replicationStatus int 34 | 35 | const ( 36 | unknown replicationStatus = iota 37 | full 38 | under 39 | ) 40 | 41 | func (s replicationStatus) String() string { 42 | switch s { 43 | case unknown: 44 | return "unknown" 45 | case full: 46 | return "full" 47 | case under: 48 | return "under" 49 | } 50 | panic("should never get here") 51 | } 52 | 53 | type metrics struct { 54 | verification *prom.CounterVec 55 | replication *prom.CounterVec 56 | } 57 | 58 | func newMetrics() *metrics { 59 | verifications := prom.NewCounterVec( 60 | prom.CounterOpts{ 61 | Namespace: promNamespace, 62 | Subsystem: promSubsystem, 63 | Name: "verification_count", 64 | Help: "Verifications event results and status counts", 65 | }, 66 | []string{"result", "status"}, 67 | ) 68 | replications := prom.NewCounterVec( 69 | prom.CounterOpts{ 70 | Namespace: promNamespace, 71 | Subsystem: promSubsystem, 72 | Name: "replication_count", 73 | Help: "Replication event result counts", 74 | }, 75 | []string{"result"}, 76 | ) 77 | return &metrics{ 78 | verification: verifications, 79 | replication: replications, 80 | } 81 | } 82 | 83 | func (m *metrics) incVerification(result result, status replicationStatus) { 84 | m.verification.WithLabelValues(result.String(), status.String()).Inc() 85 | } 86 | 87 | func (m *metrics) incReplication(result result) { 88 | m.replication.WithLabelValues(result.String()).Inc() 89 | } 90 | 91 | func (m *metrics) register() { 92 | prom.MustRegister(m.verification) 93 | prom.MustRegister(m.replication) 94 | 95 | // populate zero counts 96 | for _, r := range []result{succeeded, exhausted, errored} { 97 | for _, s := range []replicationStatus{full, under} { 98 | _, err := m.verification.GetMetricWithLabelValues(r.String(), s.String()) 99 | errors.MaybePanic(err) // should never happen 100 | } 101 | _, err := m.replication.GetMetricWithLabelValues(r.String()) 102 | errors.MaybePanic(err) // should never happen 103 | } 104 | } 105 | 106 | func (m *metrics) unregister() { 107 | prom.Unregister(m.verification) 108 | prom.Unregister(m.replication) 109 | } 110 | -------------------------------------------------------------------------------- /libri/librarian/api/testing.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/drausin/libri/libri/common/ecid" 7 | "github.com/drausin/libri/libri/common/errors" 8 | "github.com/drausin/libri/libri/common/id" 9 | ) 10 | 11 | // NewTestDocument generates a dummy Entry document for use in testing. 12 | func NewTestDocument(rng *rand.Rand) (*Document, id.ID) { 13 | doc := &Document{&Document_Entry{NewTestSinglePageEntry(rng)}} 14 | key, err := GetKey(doc) 15 | errors.MaybePanic(err) 16 | return doc, key 17 | } 18 | 19 | // NewTestEnvelope generates a dummy Envelope document for use in testing. 20 | func NewTestEnvelope(rng *rand.Rand) *Envelope { 21 | return &Envelope{ 22 | AuthorPublicKey: fakePubKey(rng), 23 | ReaderPublicKey: fakePubKey(rng), 24 | EntryKey: RandBytes(rng, DocumentKeyLength), 25 | EekCiphertext: RandBytes(rng, EEKCiphertextLength), 26 | EekCiphertextMac: RandBytes(rng, HMAC256Length), 27 | } 28 | } 29 | 30 | // NewTestSinglePageEntry generates a dummy Entry document with a single Page for use in testing. 31 | func NewTestSinglePageEntry(rng *rand.Rand) *Entry { 32 | page := NewTestPage(rng) 33 | return &Entry{ 34 | AuthorPublicKey: page.AuthorPublicKey, 35 | CreatedTime: 1, 36 | MetadataCiphertext: RandBytes(rng, 64), 37 | MetadataCiphertextMac: RandBytes(rng, HMAC256Length), 38 | Page: page, 39 | } 40 | } 41 | 42 | // NewTestMultiPageEntry generates a dummy Entry document with two page keys for use in testing. 43 | func NewTestMultiPageEntry(rng *rand.Rand) *Entry { 44 | pageKeys := [][]byte{ 45 | id.NewPseudoRandom(rng).Bytes(), 46 | id.NewPseudoRandom(rng).Bytes(), 47 | } 48 | return &Entry{ 49 | AuthorPublicKey: ecid.NewPseudoRandom(rng).PublicKeyBytes(), 50 | CreatedTime: 1, 51 | MetadataCiphertextMac: RandBytes(rng, 32), 52 | MetadataCiphertext: RandBytes(rng, 64), 53 | PageKeys: pageKeys, 54 | } 55 | } 56 | 57 | // NewTestPage generates a dummy Page for use in testing. 58 | func NewTestPage(rng *rand.Rand) *Page { 59 | return &Page{ 60 | AuthorPublicKey: fakePubKey(rng), 61 | CiphertextMac: RandBytes(rng, 32), 62 | Ciphertext: RandBytes(rng, 64), 63 | } 64 | } 65 | 66 | // NewTestPublication generates a dummy Publication for use in testing. 67 | func NewTestPublication(rng *rand.Rand) *Publication { 68 | return &Publication{ 69 | EnvelopeKey: RandBytes(rng, id.Length), 70 | EntryKey: RandBytes(rng, id.Length), 71 | AuthorPublicKey: fakePubKey(rng), 72 | ReaderPublicKey: fakePubKey(rng), 73 | } 74 | } 75 | 76 | // RandBytes generates a random bytes slice of a given length. 77 | func RandBytes(rng *rand.Rand, length int) []byte { 78 | b := make([]byte, length) 79 | _, err := rng.Read(b) 80 | errors.MaybePanic(err) 81 | return b 82 | } 83 | func fakePubKey(rng *rand.Rand) []byte { 84 | return RandBytes(rng, ECPubKeyLength) 85 | } 86 | -------------------------------------------------------------------------------- /libri/author/io/ship/shipper.go: -------------------------------------------------------------------------------- 1 | package ship 2 | 3 | import ( 4 | "github.com/drausin/libri/libri/author/io/enc" 5 | "github.com/drausin/libri/libri/author/io/pack" 6 | "github.com/drausin/libri/libri/author/io/publish" 7 | "github.com/drausin/libri/libri/common/id" 8 | "github.com/drausin/libri/libri/librarian/api" 9 | "github.com/drausin/libri/libri/librarian/client" 10 | ) 11 | 12 | // Shipper publishes documents to libri. 13 | type Shipper interface { 14 | // ShipEntry publishes (to libri) the entry document, its page document keys (if more than one), 15 | // and the envelope document with the author and reader public keys. It returns the 16 | // published envelope document and its key. 17 | ShipEntry( 18 | entry *api.Document, authorPub []byte, readerPub []byte, kek *enc.KEK, eek *enc.EEK, 19 | ) (*api.Document, id.ID, error) 20 | 21 | ShipEnvelope(entryKey id.ID, authorPub, readerPub []byte, kek *enc.KEK, eek *enc.EEK) ( 22 | *api.Document, id.ID, error) 23 | } 24 | 25 | type shipper struct { 26 | librarians client.PutterBalancer 27 | publisher publish.Publisher 28 | mlPublisher publish.MultiLoadPublisher 29 | deletePages bool 30 | } 31 | 32 | // NewShipper creates a new Shipper from a librarian api.Balancer and two publisher variants. 33 | func NewShipper( 34 | librarians client.PutterBalancer, 35 | publisher publish.Publisher, 36 | mlPublisher publish.MultiLoadPublisher) Shipper { 37 | return &shipper{ 38 | librarians: librarians, 39 | publisher: publisher, 40 | mlPublisher: mlPublisher, 41 | deletePages: true, 42 | } 43 | } 44 | 45 | func (s *shipper) ShipEntry( 46 | entry *api.Document, authorPub []byte, readerPub []byte, kek *enc.KEK, eek *enc.EEK, 47 | ) (*api.Document, id.ID, error) { 48 | 49 | // publish separate pages, if necessary 50 | pageKeys, err := api.GetEntryPageKeys(entry) 51 | if err != nil { 52 | return nil, nil, err 53 | } 54 | if pageKeys != nil { 55 | err = s.mlPublisher.Publish(pageKeys, authorPub, s.librarians, s.deletePages) 56 | if err != nil { 57 | return nil, nil, err 58 | } 59 | } 60 | 61 | rlc := s.mlPublisher.GetRetryPutter(s.librarians) 62 | entryKey, err := s.publisher.Publish(entry, authorPub, rlc) 63 | if err != nil { 64 | return nil, nil, err 65 | } 66 | return s.ShipEnvelope(entryKey, authorPub, readerPub, kek, eek) 67 | } 68 | 69 | func (s *shipper) ShipEnvelope( 70 | entryKey id.ID, authorPub, readerPub []byte, kek *enc.KEK, eek *enc.EEK, 71 | ) (*api.Document, id.ID, error) { 72 | 73 | eekCiphertext, eekCiphertextMAC, err := kek.Encrypt(eek) 74 | if err != nil { 75 | return nil, nil, err 76 | } 77 | envelope := pack.NewEnvelopeDoc(entryKey, authorPub, readerPub, eekCiphertext, eekCiphertextMAC) 78 | rlc := s.mlPublisher.GetRetryPutter(s.librarians) 79 | envelopeKey, err := s.publisher.Publish(envelope, authorPub, rlc) 80 | if err != nil { 81 | return nil, nil, err 82 | } 83 | return envelope, envelopeKey, nil 84 | } 85 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL=/bin/bash -eou pipefail 2 | GOTOOLS= github.com/alecthomas/gometalinter \ 3 | github.com/wadey/gocovmerge 4 | LIBRI_PKGS=$(shell go list ./... | grep -v /vendor/) 5 | LIBRI_PKG_SUBDIRS=$(shell go list ./... | grep -v /vendor/ | sed -r 's|github.com/drausin/libri/||g' | sort) 6 | GIT_STATUS_SUBDIRS=$(shell git status --porcelain | grep -e '\.go$$' | sed -r 's|^...(.+)/[^/]+\.go$$|\1|' | sort | uniq) 7 | GIT_DIFF_SUBDIRS=$(shell git diff develop..HEAD --name-only | grep -e '\.go$$' | sed -r 's|^(.+)/[^/]+\.go$$|\1|' | sort | uniq) 8 | GIT_STATUS_PKG_SUBDIRS=$(shell echo $(LIBRI_PKG_SUBDIRS) $(GIT_STATUS_SUBDIRS) | tr " " "\n" | sort | uniq -d) 9 | GIT_DIFF_PKG_SUBDIRS=$(shell echo $(LIBRI_PKG_SUBDIRS) $(GIT_DIFF_SUBDIRS) | tr " " "\n" | sort | uniq -d) 10 | 11 | 12 | .PHONY: bench build 13 | 14 | acceptance: 15 | @echo "--> Running acceptance tests" 16 | @mkdir -p artifacts 17 | @go test -tags acceptance -v github.com/drausin/libri/libri/acceptance 2>&1 | tee artifacts/acceptance.log 18 | 19 | bench: 20 | @echo "--> Running benchmarks" 21 | @./scripts/run-author-benchmarks.sh 22 | 23 | build: 24 | @echo "--> Running go build" 25 | @go build $(LIBRI_PKGS) 26 | 27 | build-static: 28 | @echo "--> Running go build for static binary" 29 | @./scripts/build-static.sh deploy/bin/libri 30 | 31 | demo: 32 | @echo "--> Running demo" 33 | @./libri/acceptance/local-demo.sh 34 | 35 | docker-build-image: 36 | @docker build -t daedalus2718/libri-build:latest build 37 | 38 | docker-image: 39 | @echo "--> Building docker image" 40 | @docker build --rm=false -t daedalus2718/libri:snapshot deploy 41 | 42 | fix: 43 | @echo "--> Running goimports" 44 | @find . -name *.go | grep -v /vendor/ | xargs goimports -l -w 45 | 46 | get-deps: 47 | @echo "--> Getting dependencies" 48 | @go get -u github.com/golang/dep/cmd/dep 49 | @dep ensure 50 | @go get -u -v $(GOTOOLS) 51 | @gometalinter --install 52 | 53 | install-git-hooks: 54 | @echo "--> Installing git-hooks" 55 | @./scripts/install-git-hooks.sh 56 | 57 | lint: 58 | @echo "--> Running gometalinter" 59 | @gometalinter $(LIBRI_PKG_SUBDIRS) --config=.gometalinter.json --deadline=5m 60 | 61 | lint-diff: 62 | @echo "--> Running gometalinter on packages with uncommitted changes" 63 | @echo $(GIT_STATUS_PKG_SUBDIRS) | tr " " "\n" 64 | @echo $(GIT_STATUS_PKG_SUBDIRS) | xargs gometalinter --config=.gometalinter.json --deadline=5m 65 | 66 | proto: 67 | @echo "--> Running protoc" 68 | @protoc ./libri/author/keychain/*.proto --go_out=plugins=grpc:. 69 | @protoc ./libri/common/ecid/*.proto --go_out=plugins=grpc:. 70 | @pushd libri && protoc ./librarian/api/*.proto --go_out=plugins=grpc:. && popd 71 | 72 | test-cover: 73 | @echo "--> Running go test with coverage" 74 | @./scripts/test-cover.sh 75 | 76 | test: 77 | @echo "--> Running go test" 78 | @go test -race $(LIBRI_PKGS) 79 | 80 | test-stress: 81 | @echo "--> Running stress tests" 82 | @./scripts/stress-test.sh 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /libri/cmd/download.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | cerrors "github.com/drausin/libri/libri/common/errors" 7 | "github.com/drausin/libri/libri/common/id" 8 | "github.com/pkg/errors" 9 | "github.com/spf13/cobra" 10 | "github.com/spf13/viper" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | const ( 15 | envelopeKeyFlag = "envelopeKey" 16 | downFilepathFlag = "downFilepath" 17 | ) 18 | 19 | var ( 20 | errMissingEnvelopeKey = errors.New("missing envelope key") 21 | ) 22 | 23 | // downloadCmd represents the download command 24 | var downloadCmd = &cobra.Command{ 25 | Use: "download", 26 | Short: "download a file from a Libri network using an envelope ID", 27 | RunE: func(cmd *cobra.Command, args []string) error { 28 | return newFileDownloader().download() 29 | }, 30 | } 31 | 32 | func init() { 33 | authorCmd.AddCommand(downloadCmd) 34 | 35 | downloadCmd.Flags().Uint32P(parallelismFlag, "n", 3, 36 | "number of parallel processes") 37 | downloadCmd.Flags().StringP(downFilepathFlag, "f", "", 38 | "path of local file to write downloaded contents to") 39 | downloadCmd.Flags().StringP(envelopeKeyFlag, "e", "", 40 | "key of envelope to download") 41 | 42 | // bind viper flags 43 | viper.SetEnvPrefix(envVarPrefix) // look for env vars with "LIBRI_" prefix 44 | viper.AutomaticEnv() // read in environment variables that match 45 | cerrors.MaybePanic(viper.BindPFlags(downloadCmd.Flags())) 46 | } 47 | 48 | type fileDownloader interface { 49 | download() error 50 | } 51 | 52 | func newFileDownloader() fileDownloader { 53 | return &fileDownloaderImpl{ 54 | ag: newAuthorGetter(), 55 | ad: &authorDownloaderImpl{}, 56 | kc: &keychainsGetterImpl{ 57 | pg: &terminalPassphraseGetter{}, 58 | }, 59 | } 60 | } 61 | 62 | type fileDownloaderImpl struct { 63 | ag authorGetter 64 | ad authorDownloader 65 | kc keychainsGetter 66 | } 67 | 68 | func (d *fileDownloaderImpl) download() error { 69 | envelopeKeyStr := viper.GetString(envelopeKeyFlag) 70 | if envelopeKeyStr == "" { 71 | return errMissingEnvelopeKey 72 | } 73 | envelopeKey, err := id.FromString(envelopeKeyStr) 74 | if err != nil { 75 | return err 76 | } 77 | downFilepath := viper.GetString(downFilepathFlag) 78 | if downFilepath == "" { 79 | return errMissingFilepath 80 | } 81 | authorKeys, selfReaderKeys, err := d.kc.get() 82 | if err != nil { 83 | return err 84 | } 85 | author, logger, err := d.ag.get(authorKeys, selfReaderKeys) 86 | if err != nil { 87 | return err 88 | } 89 | file, err := os.Create(downFilepath) 90 | if err != nil { 91 | return err 92 | } 93 | logger.Info("downloading document", 94 | zap.Stringer("envelope_key", envelopeKey), 95 | zap.String("filepath", downFilepath), 96 | ) 97 | err = d.ad.download(author, file, envelopeKey) 98 | if err != nil { 99 | return err 100 | } 101 | return file.Close() 102 | } 103 | -------------------------------------------------------------------------------- /libri/common/storage/checker.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha256" 6 | "errors" 7 | "fmt" 8 | ) 9 | 10 | // Checker checks that a key or value is value. 11 | type Checker interface { 12 | // Check that a key or value is valid. 13 | Check(x []byte) error 14 | } 15 | 16 | // NewEmptyChecker creates a new Checker instance that ensures values have non-zero length. 17 | func NewEmptyChecker() Checker { 18 | return &emptyChecker{} 19 | } 20 | 21 | type emptyChecker struct{} 22 | 23 | func (ec *emptyChecker) Check(x []byte) error { 24 | if x == nil { 25 | return errors.New("must not be nil") 26 | } 27 | if len(x) == 0 { 28 | return errors.New("must have non-zero length") 29 | } 30 | return nil 31 | } 32 | 33 | // NewMaxLengthChecker creates a new Checker that ensures that values are not empty and have 34 | // length less <= max length. 35 | func NewMaxLengthChecker(max int) Checker { 36 | return &maxLengthChecker{ 37 | max: max, 38 | ec: NewEmptyChecker(), 39 | } 40 | } 41 | 42 | type maxLengthChecker struct { 43 | // max length allowed 44 | max int 45 | 46 | // empty checker 47 | ec Checker 48 | } 49 | 50 | func (lc *maxLengthChecker) Check(x []byte) error { 51 | if err := lc.ec.Check(x); err != nil { 52 | return err 53 | } 54 | if len(x) > lc.max { 55 | return fmt.Errorf("must have length <= %v, actual length = %v", lc.max, len(x)) 56 | } 57 | return nil 58 | } 59 | 60 | // NewExactLengthChecker creates a Checker instance that ensures that values are not empty and 61 | // have a specified length. 62 | func NewExactLengthChecker(length int) Checker { 63 | return &exactLengthChecker{ 64 | length: length, 65 | ec: NewEmptyChecker(), 66 | } 67 | } 68 | 69 | type exactLengthChecker struct { 70 | // exact length allowed. 71 | length int 72 | 73 | // empty checker 74 | ec Checker 75 | } 76 | 77 | func (lc *exactLengthChecker) Check(x []byte) error { 78 | if err := lc.ec.Check(x); err != nil { 79 | return err 80 | } 81 | if len(x) != lc.length { 82 | return fmt.Errorf("must have length = %v, actual length = %v", lc.length, len(x)) 83 | } 84 | return nil 85 | } 86 | 87 | // KeyValueChecker checks that a key-value combination is valid. 88 | type KeyValueChecker interface { 89 | // Check checks that a key-value combination is valid. 90 | Check(key []byte, value []byte) error 91 | } 92 | 93 | type hashChecker struct{} 94 | 95 | // NewHashKeyValueChecker returns a new KeyValueChecker that checks that the key is the SHA256 96 | // hash of the value. 97 | func NewHashKeyValueChecker() KeyValueChecker { 98 | return &hashChecker{} 99 | } 100 | 101 | func (hc *hashChecker) Check(key []byte, value []byte) error { 102 | hash := sha256.Sum256(value) 103 | if !bytes.Equal(key, hash[:]) { 104 | return errors.New("key does not equal SHA256 hash of value") 105 | } 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /libri/author/helpers_test.go: -------------------------------------------------------------------------------- 1 | package author 2 | 3 | import ( 4 | "errors" 5 | "net" 6 | "testing" 7 | 8 | "github.com/drausin/libri/libri/author/io/enc" 9 | "github.com/drausin/libri/libri/author/keychain" 10 | "github.com/drausin/libri/libri/common/ecid" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestEnvelopeKeySampler_Sample_ok(t *testing.T) { 15 | authorKeys, selfReaderKeys := keychain.New(1), keychain.New(1) 16 | s := &envelopeKeySamplerImpl{ 17 | authorKeys: authorKeys, 18 | selfReaderKeys: selfReaderKeys, 19 | } 20 | authPubBytes, srPubBytes, kek1, eek, err := s.sample() 21 | assert.Nil(t, err) 22 | assert.NotNil(t, authPubBytes) 23 | assert.NotNil(t, srPubBytes) 24 | assert.NotNil(t, kek1) 25 | assert.NotNil(t, eek) 26 | authPub, err := ecid.FromPublicKeyBytes(authPubBytes) 27 | assert.Nil(t, err) 28 | 29 | // construct keys other way from selfReader private key and author public key 30 | srPriv, in := selfReaderKeys.Get(srPubBytes) 31 | assert.True(t, in) 32 | assert.NotNil(t, srPriv) 33 | kek2, err := enc.NewKEK(srPriv.Key(), authPub) 34 | assert.Nil(t, err) 35 | 36 | // check keys constructed each way are equal 37 | assert.Equal(t, kek1, kek2) 38 | } 39 | 40 | func TestEnvelopeKeySampler_Sample_err(t *testing.T) { 41 | // authorKeys.Sample() error should bubble up 42 | s1 := &envelopeKeySamplerImpl{ 43 | authorKeys: &fixedKeychain{sampleErr: errors.New("some Sample error")}, 44 | selfReaderKeys: keychain.New(3), 45 | } 46 | aPB, srPB, kek, eek, err := s1.sample() 47 | assert.NotNil(t, err) 48 | assert.Nil(t, aPB) 49 | assert.Nil(t, srPB) 50 | assert.Nil(t, kek) 51 | assert.Nil(t, eek) 52 | 53 | // selfReaderKeys.Sample() error should bubble up 54 | s2 := &envelopeKeySamplerImpl{ 55 | authorKeys: keychain.New(3), 56 | selfReaderKeys: &fixedKeychain{sampleErr: errors.New("some Sample error")}, 57 | } 58 | aPB, srPB, kek, eek, err = s2.sample() 59 | assert.NotNil(t, err) 60 | assert.Nil(t, aPB) 61 | assert.Nil(t, srPB) 62 | assert.Nil(t, kek) 63 | assert.Nil(t, eek) 64 | 65 | // too hard/annoying to create error in NewKEK() and NewEEK() 66 | } 67 | 68 | func TestGetLibrarianHealthClients(t *testing.T) { 69 | librarianAddrs := []*net.TCPAddr{ 70 | {IP: net.ParseIP("127.0.0.1"), Port: 20100}, 71 | {IP: net.ParseIP("127.0.0.1"), Port: 20101}, 72 | } 73 | healthClients, err := getLibrarianHealthClients(librarianAddrs) 74 | assert.Nil(t, err) 75 | assert.Equal(t, 2, len(healthClients)) 76 | _, in := healthClients["127.0.0.1:20100"] 77 | assert.True(t, in) 78 | _, in = healthClients["127.0.0.1:20101"] 79 | assert.True(t, in) 80 | } 81 | 82 | type fixedKeychain struct { 83 | sampleID ecid.ID 84 | sampleErr error 85 | } 86 | 87 | func (f *fixedKeychain) Sample() (ecid.ID, error) { 88 | return f.sampleID, f.sampleErr 89 | } 90 | 91 | func (f *fixedKeychain) Get(publicKey []byte) (ecid.ID, bool) { 92 | return nil, false 93 | } 94 | 95 | func (f *fixedKeychain) Len() int { 96 | return 0 97 | } 98 | -------------------------------------------------------------------------------- /libri/cmd/io.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "math/rand" 7 | 8 | "github.com/drausin/libri/libri/author" 9 | "github.com/drausin/libri/libri/author/io/common" 10 | "github.com/drausin/libri/libri/common/errors" 11 | "github.com/spf13/cobra" 12 | "github.com/spf13/viper" 13 | "go.uber.org/zap" 14 | ) 15 | 16 | const ( 17 | maxContentSize = 1 * 1024 * 1024 // bytes 18 | minContentSize = 32 // bytes 19 | ) 20 | 21 | const ( 22 | nEntriesFlag = "nEntries" 23 | ) 24 | 25 | // ioCmd represents the io command 26 | var ioCmd = &cobra.Command{ 27 | Use: "io", 28 | Short: "check ability to upload and download entries", 29 | Long: `TODO(drausin)`, 30 | RunE: func(cmd *cobra.Command, args []string) error { 31 | author, logger, err := newTestAuthorGetter().get() 32 | if err != nil { 33 | return err 34 | } 35 | if err := newIOTester().test(author, logger); err != nil { 36 | return err 37 | } 38 | return nil 39 | }, 40 | } 41 | 42 | func init() { 43 | testCmd.AddCommand(ioCmd) 44 | 45 | ioCmd.Flags().IntP(nEntriesFlag, "n", 8, "number of entries") 46 | 47 | // bind viper flags 48 | viper.SetEnvPrefix("LIBRI") // look for env vars with "LIBRI_" prefix 49 | viper.AutomaticEnv() // read in environment variables that match 50 | errors.MaybePanic(viper.BindPFlags(ioCmd.Flags())) 51 | } 52 | 53 | type ioTester interface { 54 | test(author *author.Author, logger *zap.Logger) error 55 | } 56 | 57 | func newIOTester() ioTester { 58 | return &ioTesterImpl{ 59 | au: &authorUploaderImpl{}, 60 | ad: &authorDownloaderImpl{}, 61 | } 62 | } 63 | 64 | type ioTesterImpl struct { 65 | au authorUploader 66 | ad authorDownloader 67 | } 68 | 69 | func (t *ioTesterImpl) test(author *author.Author, logger *zap.Logger) error { 70 | rng := rand.New(rand.NewSource(0)) 71 | nEntries := viper.GetInt(nEntriesFlag) 72 | for i := 0; i < nEntries; i++ { 73 | nContentBytes := minContentSize + 74 | int(rng.Int31n(int32(maxContentSize-minContentSize))) 75 | contents := common.NewCompressableBytes(rng, nContentBytes).Bytes() 76 | mediaType := "application/x-pdf" 77 | if rng.Int()%2 == 0 { 78 | mediaType = "application/x-gzip" 79 | } 80 | 81 | uploadedBuf := bytes.NewReader(contents) 82 | envelopeKey, err := t.au.upload(author, uploadedBuf, mediaType) 83 | if err != nil { 84 | return err 85 | } 86 | downloadedBuf := new(bytes.Buffer) 87 | if err := t.ad.download(author, downloadedBuf, envelopeKey); err != nil { 88 | return err 89 | } 90 | downloaded := downloadedBuf.Bytes() 91 | if !bytes.Equal(contents, downloaded) { 92 | return fmt.Errorf( 93 | "uploaded content (%d bytes) does not equal downloaded (%d bytes)", 94 | len(contents), len(downloaded), 95 | ) 96 | } 97 | } 98 | 99 | logger.Info("successfully uploaded & downloaded all entries", 100 | zap.Int("n_entries", nEntries), 101 | ) 102 | return nil 103 | } 104 | -------------------------------------------------------------------------------- /libri/librarian/client/requests.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/drausin/libri/libri/common/ecid" 7 | "github.com/drausin/libri/libri/common/id" 8 | "github.com/drausin/libri/libri/librarian/api" 9 | ) 10 | 11 | // ErrUnexpectedRequestID indicates when the RequestID in a response is different than that in the 12 | // request. 13 | var ErrUnexpectedRequestID = errors.New("response contains unexpected RequestID") 14 | 15 | // NewRequestMetadata creates a RequestMetadata object from the peer ID and a random request ID. 16 | func NewRequestMetadata(peerID, orgID ecid.ID) *api.RequestMetadata { 17 | m := &api.RequestMetadata{ 18 | RequestId: id.NewRandom().Bytes(), 19 | PubKey: peerID.PublicKeyBytes(), 20 | } 21 | if orgID != nil { 22 | m.OrgPubKey = orgID.PublicKeyBytes() 23 | } 24 | return m 25 | } 26 | 27 | // NewIntroduceRequest creates an IntroduceRequest object. 28 | func NewIntroduceRequest( 29 | peerID, orgID ecid.ID, apiSelf *api.PeerAddress, nPeers uint, 30 | ) *api.IntroduceRequest { 31 | return &api.IntroduceRequest{ 32 | Metadata: NewRequestMetadata(peerID, orgID), 33 | Self: apiSelf, 34 | NumPeers: uint32(nPeers), 35 | } 36 | } 37 | 38 | // NewFindRequest creates a FindRequest object. 39 | func NewFindRequest(peerID, orgID ecid.ID, key id.ID, nPeers uint) *api.FindRequest { 40 | return &api.FindRequest{ 41 | Metadata: NewRequestMetadata(peerID, orgID), 42 | Key: key.Bytes(), 43 | NumPeers: uint32(nPeers), 44 | } 45 | } 46 | 47 | // NewVerifyRequest creates a VerifyRequest object. 48 | func NewVerifyRequest( 49 | peerID, orgID ecid.ID, key id.ID, macKey []byte, nPeers uint, 50 | ) *api.VerifyRequest { 51 | return &api.VerifyRequest{ 52 | Metadata: NewRequestMetadata(peerID, orgID), 53 | Key: key.Bytes(), 54 | MacKey: macKey, 55 | NumPeers: uint32(nPeers), 56 | } 57 | } 58 | 59 | // NewStoreRequest creates a StoreRequest object. 60 | func NewStoreRequest(peerID, orgID ecid.ID, key id.ID, value *api.Document) *api.StoreRequest { 61 | return &api.StoreRequest{ 62 | Metadata: NewRequestMetadata(peerID, orgID), 63 | Key: key.Bytes(), 64 | Value: value, 65 | } 66 | } 67 | 68 | // NewGetRequest creates a GetRequest object. 69 | func NewGetRequest(peerID, orgID ecid.ID, key id.ID) *api.GetRequest { 70 | return &api.GetRequest{ 71 | Metadata: NewRequestMetadata(peerID, orgID), 72 | Key: key.Bytes(), 73 | } 74 | } 75 | 76 | // NewPutRequest creates a PutRequest object. 77 | func NewPutRequest(peerID, orgID ecid.ID, key id.ID, value *api.Document) *api.PutRequest { 78 | return &api.PutRequest{ 79 | Metadata: NewRequestMetadata(peerID, orgID), 80 | Key: key.Bytes(), 81 | Value: value, 82 | } 83 | } 84 | 85 | // NewSubscribeRequest creates a SubscribeRequest object. 86 | func NewSubscribeRequest( 87 | peerID, orgID ecid.ID, subscription *api.Subscription, 88 | ) *api.SubscribeRequest { 89 | return &api.SubscribeRequest{ 90 | Metadata: NewRequestMetadata(peerID, orgID), 91 | Subscription: subscription, 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /libri/author/keychain/keychain_test.go: -------------------------------------------------------------------------------- 1 | package keychain 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | "math/rand" 9 | 10 | "github.com/drausin/libri/libri/common/ecid" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestSampler_Sample_ok(t *testing.T) { 15 | kc := New(3) 16 | k1, err := kc.Sample() 17 | assert.Nil(t, err) 18 | assert.NotNil(t, k1) 19 | } 20 | 21 | func TestSampler_Sample_err(t *testing.T) { 22 | kc := New(0) 23 | k1, err := kc.Sample() 24 | assert.NotNil(t, err) 25 | assert.Nil(t, k1) 26 | } 27 | 28 | func TestGetter_Get(t *testing.T) { 29 | rng := rand.New(rand.NewSource(0)) 30 | kc := New(3) 31 | k1, err := kc.Sample() 32 | assert.Nil(t, err) 33 | assert.NotNil(t, k1) 34 | 35 | k1, in := kc.Get(k1.PublicKeyBytes()) 36 | assert.True(t, in) 37 | assert.NotNil(t, k1) 38 | 39 | k2, in := kc.Get(ecid.NewPseudoRandom(rng).Bytes()) 40 | assert.False(t, in) 41 | assert.Nil(t, k2) 42 | } 43 | 44 | func TestUnionGetter_Get(t *testing.T) { 45 | rng := rand.New(rand.NewSource(0)) 46 | kcs := []Getter{New(3), New(3), New(3)} 47 | setKC := NewUnion(kcs...) 48 | 49 | // check we can get a key from each indiv keychain 50 | for _, kc := range kcs { 51 | k1, err := kc.(GetterSampler).Sample() 52 | assert.Nil(t, err) 53 | k1, in := setKC.Get(k1.PublicKeyBytes()) 54 | assert.True(t, in) 55 | assert.Equal(t, k1, k1) 56 | } 57 | 58 | k, in := setKC.Get(ecid.NewPseudoRandom(rng).Bytes()) 59 | assert.False(t, in) 60 | assert.Nil(t, k) 61 | } 62 | 63 | func TestSave_err(t *testing.T) { 64 | file, err := ioutil.TempFile("", "kechain-test") 65 | defer func() { assert.Nil(t, os.Remove(file.Name())) }() 66 | assert.Nil(t, err) 67 | assert.Nil(t, file.Close()) 68 | 69 | // check error from bad scrypt params bubbles up 70 | err = Save(file.Name(), "test", New(3), -1, -1) 71 | assert.NotNil(t, err) 72 | } 73 | 74 | func TestLoad_err(t *testing.T) { 75 | file, err := ioutil.TempFile("", "kechain-test") 76 | defer func() { assert.Nil(t, os.Remove(file.Name())) }() 77 | assert.Nil(t, err) 78 | n, err := file.Write([]byte("not a keychain")) 79 | assert.Nil(t, err) 80 | assert.NotZero(t, n) 81 | assert.Nil(t, file.Close()) 82 | 83 | // check that error from unmarshalling bad file bubbles up 84 | kc, err := Load(file.Name(), "test") 85 | assert.NotNil(t, err) 86 | assert.Nil(t, kc) 87 | } 88 | 89 | func TestSaveLoad(t *testing.T) { 90 | file, err := ioutil.TempFile("", "kechain-test") 91 | defer func() { assert.Nil(t, os.Remove(file.Name())) }() 92 | assert.Nil(t, err) 93 | assert.Nil(t, file.Close()) 94 | 95 | kc1, auth := New(3), "test passphrase" 96 | err = Save(file.Name(), auth, kc1, veryLightScryptN, veryLightScryptP) 97 | assert.Nil(t, err) 98 | 99 | kc2, err := Load(file.Name(), auth) 100 | assert.Nil(t, err) 101 | assert.Equal(t, kc1, kc2) 102 | 103 | kc3, err := Load(file.Name(), "wrong passphrase") 104 | assert.NotNil(t, err) 105 | assert.Nil(t, kc3) 106 | 107 | } 108 | -------------------------------------------------------------------------------- /libri/librarian/server/routing/benchmarks_test.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "github.com/drausin/libri/libri/common/id" 8 | "github.com/drausin/libri/libri/librarian/server/introduce" 9 | "github.com/drausin/libri/libri/librarian/server/peer" 10 | "github.com/drausin/libri/libri/librarian/server/search" 11 | ) 12 | 13 | var benchmarkCases = []struct { 14 | name string 15 | numPeers int 16 | }{ 17 | {"small", 16}, 18 | {"medium", 64}, 19 | {"large", 256}, 20 | } 21 | 22 | func BenchmarkTable_Push(b *testing.B) { 23 | for _, c := range benchmarkCases { 24 | b.Run(c.name, func(b *testing.B) { benchmarkPush(b, c.numPeers) }) 25 | } 26 | } 27 | 28 | func BenchmarkTable_PushPop(b *testing.B) { 29 | for _, c := range benchmarkCases { 30 | b.Run(c.name, func(b *testing.B) { benchmarkPushFind(b, c.numPeers) }) 31 | } 32 | } 33 | 34 | func BenchmarkTable_Peak(b *testing.B) { 35 | for _, c := range benchmarkCases { 36 | b.Run(c.name, func(b *testing.B) { benchmarkFind(b, c.numPeers) }) 37 | } 38 | } 39 | 40 | func BenchmarkTable_Sample(b *testing.B) { 41 | for _, c := range benchmarkCases { 42 | b.Run(c.name, func(b *testing.B) { benchmarkSample(b, c.numPeers) }) 43 | } 44 | } 45 | 46 | func benchmarkPush(b *testing.B, numPeers int) { 47 | rng := rand.New(rand.NewSource(int64(0))) 48 | for n := 0; n < b.N; n++ { 49 | rt, _, _, _ := NewTestWithPeers(rng, 0) // empty 50 | repeatedPeers := make([]peer.Peer, 0) 51 | for _, p := range peer.NewTestPeers(rng, numPeers) { 52 | status := rt.Push(p) 53 | if status == Added { 54 | // 25% of the time, we add this peer to list we'll add again later 55 | if rng.Float32() < 0.25 { 56 | repeatedPeers = append(repeatedPeers, p) 57 | } 58 | } 59 | } 60 | 61 | // add a few other peers a second time 62 | for _, p := range repeatedPeers { 63 | rt.Push(p) 64 | } 65 | } 66 | } 67 | 68 | func benchmarkPushFind(b *testing.B, numPeers int) { 69 | rng := rand.New(rand.NewSource(int64(0))) 70 | for n := 0; n < b.N; n++ { 71 | rt, _, _, _ := NewTestWithPeers(rng, numPeers) 72 | 73 | // pop half the peers every time 74 | for rt.NumPeers() > 0 { 75 | numToPop := uint(rt.NumPeers()/2 + 1) 76 | target := id.NewPseudoRandom(rng) 77 | rt.Find(target, numToPop) 78 | } 79 | } 80 | } 81 | 82 | func benchmarkFind(b *testing.B, numPeers int) { 83 | rng := rand.New(rand.NewSource(int64(0))) 84 | for n := 0; n < b.N; n++ { 85 | rt, _, _, _ := NewTestWithPeers(rng, numPeers) 86 | for c := 0; c < 100; c++ { 87 | target := id.NewPseudoRandom(rng) 88 | rt.Find(target, search.DefaultNClosestResponses) 89 | } 90 | } 91 | } 92 | 93 | func benchmarkSample(b *testing.B, numPeers int) { 94 | rng := rand.New(rand.NewSource(int64(0))) 95 | for n := 0; n < b.N; n++ { 96 | rt, _, _, _ := NewTestWithPeers(rng, numPeers) 97 | for c := 0; c < 100; c++ { 98 | rt.Sample(introduce.DefaultNumPeersPerRequest, rng) 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /libri/librarian/client/context.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | 7 | "github.com/golang/protobuf/proto" 8 | "golang.org/x/net/context" 9 | "google.golang.org/grpc/metadata" 10 | ) 11 | 12 | const ( 13 | signatureKey = "signature" 14 | orgSignatureKey = "orgsignature" 15 | ) 16 | 17 | var ( 18 | errContextMissingMetadata = errors.New("context unexpectedly missing metadata") 19 | errContextMissingSignature = errors.New("metadata signature key unexpectedly does not " + 20 | "exist") 21 | errContextMissingOrgSignature = errors.New("metadata organization signature key " + 22 | "unexpectedly does not exist") 23 | ) 24 | 25 | // NewSignatureContext creates a new context with the signed JSON web token (JWT) string. 26 | func NewSignatureContext(ctx context.Context, signedJWT, orgSignedJWT string) context.Context { 27 | return metadata.NewOutgoingContext(ctx, 28 | metadata.Pairs( 29 | signatureKey, signedJWT, 30 | orgSignatureKey, orgSignedJWT, 31 | ), 32 | ) 33 | } 34 | 35 | // NewIncomingSignatureContext creates a new context with the signed JSON web token (JWT) string 36 | // in the incoming metadata field. This function should only be used for testing. 37 | func NewIncomingSignatureContext( 38 | ctx context.Context, signedJWT, orgSignedJWT string, 39 | ) context.Context { 40 | return metadata.NewIncomingContext(ctx, 41 | metadata.Pairs( 42 | signatureKey, signedJWT, 43 | orgSignatureKey, orgSignedJWT, 44 | ), 45 | ) 46 | } 47 | 48 | // FromSignatureContext extracts the signed JSON web token from the context. 49 | func FromSignatureContext(ctx context.Context) (string, string, error) { 50 | md, ok := metadata.FromIncomingContext(ctx) 51 | if !ok { 52 | return "", "", errContextMissingMetadata 53 | } 54 | signedJWTs, exists := md[signatureKey] 55 | if !exists { 56 | return "", "", errContextMissingSignature 57 | } 58 | signedOrgJWTs, exists := md[orgSignatureKey] 59 | if !exists { 60 | return "", "", errContextMissingOrgSignature 61 | } 62 | return signedJWTs[0], signedOrgJWTs[0], nil 63 | } 64 | 65 | // NewSignedContext creates a new context with a request signature. 66 | func NewSignedContext(signer, orgSigner Signer, request proto.Message) (context.Context, error) { 67 | ctx := context.Background() 68 | 69 | // sign the message 70 | signedJWT, err := signer.Sign(request) 71 | if err != nil { 72 | return nil, err 73 | } 74 | orgSignedJWT, err := orgSigner.Sign(request) 75 | if err != nil { 76 | return nil, err 77 | } 78 | ctx = NewSignatureContext(ctx, signedJWT, orgSignedJWT) 79 | return ctx, nil 80 | } 81 | 82 | // NewSignedTimeoutContext creates a new context with a timeout and request signature. 83 | func NewSignedTimeoutContext( 84 | signer, orgSigner Signer, request proto.Message, timeout time.Duration, 85 | ) ( 86 | context.Context, context.CancelFunc, error) { 87 | 88 | ctx, err := NewSignedContext(signer, orgSigner, request) 89 | if err != nil { 90 | return nil, func() {}, err 91 | } 92 | ctx, cancel := context.WithTimeout(ctx, timeout) 93 | return ctx, cancel, nil 94 | } 95 | -------------------------------------------------------------------------------- /libri/librarian/client/context_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | "time" 7 | 8 | "context" 9 | 10 | "github.com/drausin/libri/libri/common/ecid" 11 | "github.com/drausin/libri/libri/common/id" 12 | "github.com/stretchr/testify/assert" 13 | "google.golang.org/grpc/metadata" 14 | ) 15 | 16 | func TestNewSignatureContext(t *testing.T) { 17 | ctx := context.Background() 18 | signedToken1 := "some.signed.token" 19 | signedOrgToken1 := "some.signed.org-token" 20 | signedCtx := NewSignatureContext(ctx, signedToken1, signedOrgToken1) 21 | md, ok := metadata.FromOutgoingContext(signedCtx) 22 | assert.True(t, ok) 23 | signedTokens2, in := md[signatureKey] 24 | assert.True(t, in) 25 | signedOrgTokens2, in := md[orgSignatureKey] 26 | assert.True(t, in) 27 | assert.True(t, len(signedTokens2) == 1) 28 | assert.Equal(t, signedToken1, signedTokens2[0]) 29 | assert.Equal(t, signedOrgToken1, signedOrgTokens2[0]) 30 | } 31 | 32 | func TestNewFromSignatureContext(t *testing.T) { 33 | ctx := context.Background() 34 | signedToken1 := "some.signed.token" 35 | signedOrgToken1 := "some.signed.org-token" 36 | signedCtx := NewIncomingSignatureContext(ctx, signedToken1, signedOrgToken1) 37 | signedToken2, signedOrgToken2, err := FromSignatureContext(signedCtx) 38 | assert.Equal(t, signedToken1, signedToken2) 39 | assert.Equal(t, signedOrgToken1, signedOrgToken2) 40 | assert.Nil(t, err) 41 | } 42 | 43 | func TestFromSignatureContext_missingMetadataErr(t *testing.T) { 44 | signedToken, signedOrgToken, err := FromSignatureContext(context.Background()) 45 | assert.Zero(t, signedToken) 46 | assert.Zero(t, signedOrgToken) 47 | assert.NotNil(t, err) 48 | } 49 | 50 | func TestFromSignatureContext_missingSignatureErr(t *testing.T) { 51 | ctx := metadata.NewOutgoingContext(context.Background(), metadata.MD{}) // no signature key 52 | signedToken, signedOrgToken, err := FromSignatureContext(ctx) 53 | assert.Zero(t, signedToken) 54 | assert.Zero(t, signedOrgToken) 55 | assert.NotNil(t, err) 56 | } 57 | func TestNewSignedTimeoutContext_ok(t *testing.T) { 58 | rng := rand.New(rand.NewSource(int64(0))) 59 | ctx, cancel, err := NewSignedTimeoutContext( 60 | &TestNoOpSigner{}, 61 | &TestNoOpSigner{}, 62 | NewFindRequest( 63 | ecid.NewPseudoRandom(rng), 64 | ecid.NewPseudoRandom(rng), 65 | id.NewPseudoRandom(rng), 66 | 20, 67 | ), 68 | 5*time.Second, 69 | ) 70 | assert.NotNil(t, ctx) 71 | 72 | md, in := metadata.FromOutgoingContext(ctx) 73 | assert.True(t, in) 74 | assert.NotNil(t, md[signatureKey]) 75 | assert.NotNil(t, cancel) 76 | assert.Nil(t, err) 77 | } 78 | 79 | func TestNewSignedTimeoutContext_err(t *testing.T) { 80 | rng := rand.New(rand.NewSource(int64(0))) 81 | ctx, cancel, err := NewSignedTimeoutContext( 82 | &TestErrSigner{}, 83 | &TestErrSigner{}, 84 | NewFindRequest( 85 | ecid.NewPseudoRandom(rng), 86 | ecid.NewPseudoRandom(rng), 87 | id.NewPseudoRandom(rng), 88 | 20, 89 | ), 90 | 5*time.Second, 91 | ) 92 | assert.Nil(t, ctx) 93 | assert.NotNil(t, cancel) 94 | assert.NotNil(t, err) 95 | } 96 | -------------------------------------------------------------------------------- /libri/author/logging.go: -------------------------------------------------------------------------------- 1 | package author 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/drausin/libri/libri/common/ecid" 9 | "github.com/drausin/libri/libri/common/id" 10 | "github.com/drausin/libri/libri/librarian/api" 11 | "go.uber.org/zap" 12 | "go.uber.org/zap/zapcore" 13 | ) 14 | 15 | const ( 16 | logClientIDShort = "client_id_short" 17 | logEntryKey = "entry_key" 18 | logEnvelopeKey = "envelope_key" 19 | logAuthorPubShort = "author_pub_short" 20 | logReaderPubShort = "reader_pub_short" 21 | logNPages = "n_pages" 22 | logMetadata = "metadata" 23 | logSpeedMbps = "speed_Mbps" 24 | ) 25 | 26 | func packingContentFields(authorPub []byte) []zapcore.Field { 27 | return []zapcore.Field{ 28 | zap.String(logAuthorPubShort, id.ShortHex(authorPub[1:9])), 29 | } 30 | } 31 | 32 | func shippingEntryFields(authorPub, readerPub []byte) []zapcore.Field { 33 | return []zapcore.Field{ 34 | zap.String(logAuthorPubShort, id.ShortHex(authorPub[1:9])), 35 | zap.String(logReaderPubShort, id.ShortHex(readerPub[1:9])), 36 | } 37 | } 38 | 39 | func uploadedDocFields( 40 | envKey fmt.Stringer, env *api.Document, md *api.EntryMetadata, elapsed time.Duration, 41 | ) []zapcore.Field { 42 | entryKey := id.FromBytes(env.Contents.(*api.Document_Envelope).Envelope.EntryKey) 43 | return docFields(envKey, entryKey, md, elapsed) 44 | } 45 | 46 | func downloadingDocFields(envKey fmt.Stringer) []zapcore.Field { 47 | return []zapcore.Field{ 48 | zap.Stringer(logEnvelopeKey, envKey), 49 | } 50 | } 51 | 52 | func downloadedDocFields( 53 | envKey, entryKey fmt.Stringer, md *api.EntryMetadata, elapsed time.Duration, 54 | ) []zapcore.Field { 55 | return docFields(envKey, entryKey, md, elapsed) 56 | } 57 | 58 | func docFields( 59 | envKey, entryKey fmt.Stringer, md *api.EntryMetadata, elapsed time.Duration, 60 | ) []zapcore.Field { 61 | 62 | // surprisingly hard to truncate this to 2 decimal places as float 63 | speedMbps := float32(md.UncompressedSize) * 8 / float32(2<<20) / float32(elapsed.Seconds()) 64 | return []zapcore.Field{ 65 | zap.Stringer(logEnvelopeKey, envKey), 66 | zap.Stringer(logEntryKey, entryKey), 67 | zap.String(logSpeedMbps, fmt.Sprintf("%.2f", speedMbps)), 68 | zap.Object(logMetadata, md), 69 | } 70 | } 71 | 72 | func unpackingContentFields(entryKey fmt.Stringer, nPages int) []zapcore.Field { 73 | return []zapcore.Field{ 74 | zap.Stringer(logEntryKey, entryKey), 75 | zap.Int(logNPages, nPages), 76 | } 77 | } 78 | 79 | func sharingDocFields(envKey fmt.Stringer, readerPub *ecdsa.PublicKey) []zapcore.Field { 80 | return []zapcore.Field{ 81 | zap.Stringer(logEnvelopeKey, envKey), 82 | zap.String(logReaderPubShort, id.ShortHex(ecid.ToPublicKeyBytes(readerPub)[1:9])), 83 | } 84 | } 85 | 86 | func sharedDocFields(envKey, entryKey fmt.Stringer, authorPub, readerPub []byte) []zapcore.Field { 87 | return []zapcore.Field{ 88 | zap.Stringer(logEntryKey, entryKey), 89 | zap.Stringer(logEnvelopeKey, envKey), 90 | zap.String(logAuthorPubShort, id.ShortHex(authorPub[1:9])), 91 | zap.String(logReaderPubShort, id.ShortHex(readerPub[1:9])), 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /libri/author/keychain/storage.go: -------------------------------------------------------------------------------- 1 | package keychain 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | "sync" 6 | 7 | "github.com/drausin/libri/libri/common/ecid" 8 | ethkeystore "github.com/ethereum/go-ethereum/accounts/keystore" 9 | "github.com/ethereum/go-ethereum/crypto" 10 | ) 11 | 12 | // encryptToStored encrypts the contents of Keychain using the authentication passphrase and scrypt 13 | // difficulty parameters. 14 | func encryptToStored(kc GetterSampler, auth string, scryptN, scryptP int) (*StoredKeychain, error) { 15 | storedPrivateKeys := make([][]byte, 0) 16 | var mu sync.Mutex 17 | var wg sync.WaitGroup 18 | errs, done := make(chan error, 1), make(chan struct{}, 1) 19 | 20 | // encrypt all keys in parallel b/c each can be intensive, thanks to scrypt 21 | for _, key1 := range kc.(*keychain).privs { 22 | wg.Add(1) 23 | go func(key2 ecid.ID) { 24 | var err error 25 | defer wg.Done() 26 | encryptedKeyBytes, err := encryptKey(key2.Key(), auth, scryptN, scryptP) 27 | if err != nil { 28 | errs <- err 29 | } 30 | mu.Lock() 31 | storedPrivateKeys = append(storedPrivateKeys, encryptedKeyBytes) 32 | mu.Unlock() 33 | }(key1) 34 | } 35 | 36 | go func() { 37 | wg.Wait() 38 | done <- struct{}{} 39 | }() 40 | 41 | select { 42 | case <-done: 43 | return &StoredKeychain{ 44 | PrivateKeys: storedPrivateKeys, 45 | }, nil 46 | case err := <-errs: 47 | return nil, err 48 | } 49 | } 50 | 51 | // decryptFromStored decrypts the contents of a StoredKeychain using the authentication passphrase. 52 | func decryptFromStored(stored *StoredKeychain, auth string) (GetterSampler, error) { 53 | ecids := make([]ecid.ID, 0) 54 | var mu sync.Mutex 55 | var wg sync.WaitGroup 56 | errs, done := make(chan error, 1), make(chan struct{}, 1) 57 | 58 | // decrypt all keys in parallel b/c each can be intensive, thanks to scrypt 59 | for _, keyJSON1 := range stored.PrivateKeys { 60 | wg.Add(1) 61 | go func(keyJSON2 []byte) { 62 | defer wg.Done() 63 | priv, err := decryptKey(keyJSON2, auth) 64 | if err != nil { 65 | errs <- err 66 | return 67 | } 68 | i, err := ecid.FromPrivateKey(priv) 69 | if err != nil { 70 | errs <- err 71 | return 72 | } 73 | mu.Lock() 74 | ecids = append(ecids, i) 75 | mu.Unlock() 76 | }(keyJSON1) 77 | } 78 | 79 | go func() { 80 | wg.Wait() 81 | done <- struct{}{} 82 | }() 83 | 84 | select { 85 | case <-done: 86 | return FromECIDs(ecids), nil 87 | case err := <-errs: 88 | return nil, err 89 | } 90 | } 91 | 92 | func encryptKey(key *ecdsa.PrivateKey, auth string, scryptN, scryptP int) ([]byte, error) { 93 | ethKey := ðkeystore.Key{ 94 | // Address is not not used by libri, but required for encryption & decryption 95 | Address: crypto.PubkeyToAddress(key.PublicKey), 96 | PrivateKey: key, 97 | } 98 | return ethkeystore.EncryptKey(ethKey, auth, scryptN, scryptP) 99 | } 100 | 101 | func decryptKey(keyJSON []byte, auth string) (*ecdsa.PrivateKey, error) { 102 | ethKey, err := ethkeystore.DecryptKey(keyJSON, auth) 103 | if err != nil { 104 | return nil, err 105 | } 106 | return ethKey.PrivateKey, nil 107 | } 108 | -------------------------------------------------------------------------------- /libri/author/io/print/scanner.go: -------------------------------------------------------------------------------- 1 | package print 2 | 3 | import ( 4 | "io" 5 | "sync" 6 | 7 | "github.com/drausin/libri/libri/author/io/comp" 8 | "github.com/drausin/libri/libri/author/io/enc" 9 | "github.com/drausin/libri/libri/author/io/page" 10 | "github.com/drausin/libri/libri/common/id" 11 | "github.com/drausin/libri/libri/librarian/api" 12 | ) 13 | 14 | // Scanner writes locally-stored pages to a unified content stream. 15 | type Scanner interface { 16 | // Scan loads pages with the given keys and metadata from an internal page.Loader and 17 | // writes their concatenated output to the content io.Writer. 18 | Scan(content io.Writer, pageKeys []id.ID, keys *enc.EEK, metatdata *api.EntryMetadata) error 19 | } 20 | 21 | type scanner struct { 22 | params *Parameters 23 | pageL page.Loader 24 | init scanInitializer 25 | } 26 | 27 | // NewScanner creates a new Scanner object with the given parameters, encryption keys, and page 28 | // loader. 29 | func NewScanner(params *Parameters, pageL page.Loader) Scanner { 30 | return &scanner{ 31 | params: params, 32 | pageL: pageL, 33 | init: &scanInitializerImpl{ 34 | params: params, 35 | }, 36 | } 37 | } 38 | 39 | func (s *scanner) Scan( 40 | content io.Writer, pageKeys []id.ID, keys *enc.EEK, md *api.EntryMetadata, 41 | ) error { 42 | 43 | pages := make(chan *api.Page, int(s.params.Parallelism)) 44 | if err := api.ValidateEntryMetadata(md); err != nil { 45 | return err 46 | } 47 | decompressor, unpaginator, err := s.init.Initialize(content, md.CompressionCodec, keys, pages) 48 | if err != nil { 49 | return err 50 | } 51 | errs := make(chan error, 1) 52 | abortLoad := make(chan struct{}) 53 | wg := new(sync.WaitGroup) 54 | wg.Add(1) 55 | go func() { 56 | _, wtErr := unpaginator.WriteTo(decompressor) 57 | if wtErr != nil { 58 | errs <- wtErr 59 | close(abortLoad) 60 | } 61 | wg.Done() 62 | }() 63 | 64 | err = s.pageL.Load(pageKeys, pages, abortLoad) 65 | close(pages) 66 | if err != nil { 67 | return err 68 | } 69 | wg.Wait() 70 | 71 | select { 72 | case err = <-errs: 73 | return err 74 | default: 75 | } 76 | 77 | return enc.CheckMACs(unpaginator.CiphertextMAC(), decompressor.UncompressedMAC(), md) 78 | } 79 | 80 | type scanInitializer interface { 81 | Initialize(content io.Writer, codec api.CompressionCodec, keys *enc.EEK, pages chan *api.Page) ( 82 | comp.Decompressor, page.Unpaginator, error) 83 | } 84 | 85 | type scanInitializerImpl struct { 86 | params *Parameters 87 | } 88 | 89 | func (si *scanInitializerImpl) Initialize( 90 | content io.Writer, codec api.CompressionCodec, keys *enc.EEK, pages chan *api.Page, 91 | ) (comp.Decompressor, page.Unpaginator, error) { 92 | 93 | decompressor, err := comp.NewDecompressor(content, codec, keys, 94 | si.params.CompressionBufferSize) 95 | if err != nil { 96 | return nil, nil, err 97 | } 98 | decrypter, err := enc.NewDecrypter(keys) 99 | if err != nil { 100 | return nil, nil, err 101 | } 102 | unpaginator, err := page.NewUnpaginator(pages, decrypter, keys) 103 | if err != nil { 104 | return nil, nil, err 105 | } 106 | return decompressor, unpaginator, nil 107 | } 108 | -------------------------------------------------------------------------------- /libri/cmd/author_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | "os" 7 | "strings" 8 | "testing" 9 | 10 | "path/filepath" 11 | 12 | "github.com/drausin/libri/libri/author" 13 | "github.com/drausin/libri/libri/author/keychain" 14 | "github.com/drausin/libri/libri/common/logging" 15 | "github.com/pkg/errors" 16 | "github.com/spf13/viper" 17 | "github.com/stretchr/testify/assert" 18 | "go.uber.org/zap" 19 | ) 20 | 21 | func TestAuthorGetter_get_ok(t *testing.T) { 22 | keychainDir, err := ioutil.TempDir("", "test-author") 23 | assert.Nil(t, err) 24 | config := author.NewDefaultConfig(). 25 | WithDataDir(keychainDir). 26 | WithDefaultDBDir(). 27 | WithDefaultKeychainDir() 28 | logger1 := logging.NewDevInfoLogger() 29 | 30 | ag := authorGetterImpl{ 31 | acg: &fixedAuthorConfigGetter{ 32 | config: config, 33 | logger: logger1, 34 | }, 35 | } 36 | authorKeys, selfReaderKeys := keychain.New(1), keychain.New(1) 37 | 38 | author2, logger2, err := ag.get(authorKeys, selfReaderKeys) 39 | assert.Nil(t, err) 40 | assert.NotNil(t, author2) 41 | assert.Equal(t, logger1, logger2) 42 | 43 | assert.Nil(t, os.RemoveAll(keychainDir)) 44 | assert.Nil(t, os.RemoveAll(config.DataDir)) 45 | } 46 | 47 | func TestAuthorGetter_get_err(t *testing.T) { 48 | ag := authorGetterImpl{ 49 | acg: &fixedAuthorConfigGetter{ 50 | err: errors.New("some config get error"), 51 | }, 52 | } 53 | authorKeys, selfReaderKeys := keychain.New(1), keychain.New(1) 54 | 55 | author2, logger2, err := ag.get(authorKeys, selfReaderKeys) 56 | assert.NotNil(t, err) 57 | assert.Nil(t, author2) 58 | assert.Nil(t, logger2) 59 | } 60 | 61 | func TestAuthorConfigGetter_get_ok(t *testing.T) { 62 | dataDir, logLevel := "some/data/dir", zap.DebugLevel 63 | libAddrs := []string{"127.0.0.1:1234", "127.0.0.1:5678"} 64 | libAddrsArg := strings.Join(libAddrs, " ") 65 | log.Print(libAddrsArg) 66 | viper.Set(dataDirFlag, dataDir) 67 | viper.Set(logLevelFlag, logLevel) 68 | viper.Set(authorLibrariansFlag, libAddrsArg) 69 | acg := &authorConfigGetterImpl{} 70 | 71 | config, logger, err := acg.get(authorLibrariansFlag) 72 | 73 | assert.Nil(t, err) 74 | assert.Equal(t, logLevel, config.LogLevel) 75 | assert.Equal(t, len(libAddrs), len(config.LibrarianAddrs)) 76 | for i, la := range config.LibrarianAddrs { 77 | assert.Equal(t, libAddrs[i], la.String()) 78 | } 79 | assert.NotNil(t, logger) 80 | 81 | cwd, err := os.Getwd() 82 | assert.Nil(t, err) 83 | assert.Nil(t, os.RemoveAll(filepath.Join(cwd, dataDir))) 84 | } 85 | 86 | func TestAuthorConfigGetter_get_err(t *testing.T) { 87 | viper.Set(authorLibrariansFlag, "not an address") 88 | acg := &authorConfigGetterImpl{} 89 | 90 | config, logger, err := acg.get(authorLibrariansFlag) 91 | 92 | assert.NotNil(t, err) 93 | assert.Nil(t, config) 94 | assert.NotNil(t, logger) // still should have been created 95 | } 96 | 97 | type fixedAuthorConfigGetter struct { 98 | config *author.Config 99 | logger *zap.Logger 100 | err error 101 | } 102 | 103 | func (f *fixedAuthorConfigGetter) get(librariansFlag string) (*author.Config, *zap.Logger, error) { 104 | return f.config, f.logger, f.err 105 | } 106 | -------------------------------------------------------------------------------- /libri/librarian/server/metrics_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | 7 | "github.com/drausin/libri/libri/librarian/api" 8 | prom "github.com/prometheus/client_golang/prometheus" 9 | dto "github.com/prometheus/client_model/go" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestStorageMetrics_initAdd(t *testing.T) { 14 | rng := rand.New(rand.NewSource(0)) 15 | serverSL := &fixedSL{stored: make(map[string][]byte)} 16 | 17 | sm1 := newStorageMetrics(serverSL) 18 | sm1.register() 19 | defer sm1.unregister() 20 | envDoc := &api.Document{ 21 | Contents: &api.Document_Envelope{ 22 | Envelope: api.NewTestEnvelope(rng), 23 | }, 24 | } 25 | entryDoc := &api.Document{ 26 | Contents: &api.Document_Entry{ 27 | Entry: api.NewTestSinglePageEntry(rng), 28 | }, 29 | } 30 | pageDoc := &api.Document{ 31 | Contents: &api.Document_Page{ 32 | Page: api.NewTestPage(rng), 33 | }, 34 | } 35 | for _, doc := range []*api.Document{envDoc, entryDoc, pageDoc} { 36 | err := sm1.Add(doc) 37 | assert.Nil(t, err) 38 | } 39 | 40 | // simulate server restarting and re-loading storage metrics 41 | sm2 := newStorageMetrics(serverSL) 42 | 43 | // check we have a single count for each doc type 44 | countMetrics := make(chan prom.Metric, 3) 45 | expectedLabelValues := map[string]struct{}{ 46 | "envelope": {}, 47 | "entry": {}, 48 | "page": {}, 49 | } 50 | sm2.count.Collect(countMetrics) 51 | close(countMetrics) 52 | nCountMetrics := 0 53 | actualCountLabelValues := map[string]struct{}{} 54 | for countMetric := range countMetrics { 55 | written := &dto.Metric{} 56 | countMetric.Write(written) 57 | assert.Equal(t, float64(1.0), *written.Counter.Value) 58 | assert.Equal(t, 1, len(written.Label)) 59 | assert.Equal(t, "doc_type", *written.Label[0].Name) 60 | actualCountLabelValues[*written.Label[0].Value] = struct{}{} 61 | nCountMetrics++ 62 | } 63 | assert.Equal(t, 3, nCountMetrics) 64 | assert.Equal(t, expectedLabelValues, actualCountLabelValues) 65 | 66 | sizeMetrics := make(chan prom.Metric, 3) 67 | sm2.size.Collect(sizeMetrics) 68 | close(sizeMetrics) 69 | nSizeMetrics := 0 70 | actualSizeLabelValues := map[string]struct{}{} 71 | for sizeMetric := range sizeMetrics { 72 | written := &dto.Metric{} 73 | sizeMetric.Write(written) 74 | assert.True(t, *written.Counter.Value > float64(0.0)) 75 | assert.Equal(t, 1, len(written.Label)) 76 | assert.Equal(t, "doc_type", *written.Label[0].Name) 77 | actualSizeLabelValues[*written.Label[0].Value] = struct{}{} 78 | nSizeMetrics++ 79 | } 80 | assert.Equal(t, 3, nSizeMetrics) 81 | } 82 | 83 | type fixedSL struct { 84 | stored map[string][]byte 85 | } 86 | 87 | func (f *fixedSL) Iterate( 88 | keyLB, keyUB []byte, done chan struct{}, callback func(key, value []byte), 89 | ) error { 90 | panic("implement me") 91 | } 92 | 93 | func (f *fixedSL) Load(key []byte) ([]byte, error) { 94 | if value, in := f.stored[string(key)]; in { 95 | return value, nil 96 | } 97 | return nil, nil 98 | } 99 | 100 | func (f *fixedSL) Store(key []byte, value []byte) error { 101 | f.stored[string(key)] = value 102 | return nil 103 | } 104 | -------------------------------------------------------------------------------- /libri/librarian/client/pool.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/drausin/libri/libri/librarian/api" 7 | "github.com/hashicorp/golang-lru" 8 | "google.golang.org/grpc" 9 | ) 10 | 11 | const ( 12 | defaultMaxConns = 128 13 | ) 14 | 15 | // Pool maintains a pool of librarian clients. 16 | type Pool interface { 17 | // Get the connection to the given address. 18 | Get(address string) (api.LibrarianClient, error) 19 | 20 | // CloseAll closes all active connections. Not further Get() calls may be made after this call. 21 | CloseAll() error 22 | } 23 | 24 | type lruPool struct { 25 | conns *lru.Cache 26 | dialer dialer 27 | closer closer 28 | evictionErr chan error 29 | } 30 | 31 | // NewLRUPool creates a new LRU Pool with the given number of max connections. 32 | func NewLRUPool(maxConns int) (Pool, error) { 33 | return newLRUPool(maxConns, insecureDialer{}, closerImpl{}) 34 | } 35 | 36 | // NewDefaultLRUPool creates a new LRU pool with the default number of max connections. 37 | func NewDefaultLRUPool() (Pool, error) { 38 | return NewLRUPool(defaultMaxConns) 39 | } 40 | 41 | func newLRUPool(maxConns int, dialer dialer, closer closer) (Pool, error) { 42 | evictionErrs := make(chan error, 1) 43 | onEvicted := func(key interface{}, value interface{}) { 44 | // close connection when it is evicted 45 | evictionErrs <- closer.close(value.(*grpc.ClientConn)) 46 | } 47 | conns, err := lru.NewWithEvict(maxConns, onEvicted) 48 | if err != nil { 49 | return nil, err 50 | } 51 | return &lruPool{ 52 | conns: conns, 53 | dialer: dialer, 54 | closer: closer, 55 | evictionErr: evictionErrs, 56 | }, nil 57 | } 58 | 59 | func (p *lruPool) Get(address string) (api.LibrarianClient, error) { 60 | if value, in := p.conns.Get(address); in { 61 | // return existing connection if we have it 62 | return api.NewLibrarianClient(value.(*grpc.ClientConn)), nil 63 | } 64 | // create a new connection 65 | conn, err := p.dialer.dial(address) 66 | if err != nil { 67 | return nil, err 68 | } 69 | if eviction := p.conns.Add(address, conn); eviction { 70 | if err := <-p.evictionErr; err != nil { 71 | return nil, err 72 | } 73 | } 74 | return api.NewLibrarianClient(conn), nil 75 | } 76 | 77 | func (p *lruPool) CloseAll() error { 78 | go func() { 79 | p.conns.Purge() 80 | close(p.evictionErr) 81 | }() 82 | 83 | for err := range p.evictionErr { 84 | if err != nil { 85 | return err 86 | } 87 | } 88 | return nil 89 | } 90 | 91 | // dialer is a very thin wrapper around grpc.Dial to facilitate mocking during testing 92 | type dialer interface { 93 | dial(address string) (*grpc.ClientConn, error) 94 | } 95 | 96 | type insecureDialer struct{} 97 | 98 | func (insecureDialer) dial(address string) (*grpc.ClientConn, error) { 99 | return grpc.Dial(address, grpc.WithInsecure()) 100 | } 101 | 102 | // closer is a very thin wrapper around (*grpc.ClientConn).Close() to facilitate mocking during 103 | // testing 104 | type closer interface { 105 | close(conn io.Closer) error 106 | } 107 | 108 | type closerImpl struct{} 109 | 110 | func (closerImpl) close(conn io.Closer) error { 111 | return conn.Close() 112 | } 113 | -------------------------------------------------------------------------------- /libri/common/storage/testing.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/drausin/libri/libri/common/errors" 7 | "github.com/drausin/libri/libri/common/id" 8 | "github.com/drausin/libri/libri/librarian/api" 9 | "github.com/golang/protobuf/proto" 10 | ) 11 | 12 | // TestSLD mocks StorerLoaderDeleter interface. 13 | type TestSLD struct { 14 | Bytes []byte 15 | LoadErr error 16 | IterateErr error 17 | StoreErr error 18 | DeleteErr error 19 | mu sync.Mutex 20 | } 21 | 22 | // Load mocks StorerLoaderDeleter.Load(). 23 | func (l *TestSLD) Load(key []byte) ([]byte, error) { 24 | l.mu.Lock() 25 | defer l.mu.Unlock() 26 | return l.Bytes, l.LoadErr 27 | } 28 | 29 | // Iterate mocks StorerLoaderDeleter.Iterate(). 30 | func (l *TestSLD) Iterate( 31 | keyLB, keyUB []byte, done chan struct{}, callback func(key, value []byte), 32 | ) error { 33 | // "happy path" doesn't really exist ATM, implement if needed 34 | return l.IterateErr 35 | } 36 | 37 | // Store mocks StorerLoaderDeleter.Store(). 38 | func (l *TestSLD) Store(key []byte, value []byte) error { 39 | l.mu.Lock() 40 | defer l.mu.Unlock() 41 | l.Bytes = value 42 | return l.StoreErr 43 | } 44 | 45 | // Delete mocks StorerLoaderDeleter.Delete(). 46 | func (l *TestSLD) Delete(key []byte) error { 47 | l.mu.Lock() 48 | defer l.mu.Unlock() 49 | l.Bytes = nil 50 | return l.DeleteErr 51 | } 52 | 53 | // NewTestDocSLD creates a new TestDocSLD. 54 | func NewTestDocSLD() *TestDocSLD { 55 | return &TestDocSLD{ 56 | Stored: make(map[string]*api.Document), 57 | } 58 | } 59 | 60 | // TestDocSLD mocks DocumentSLD. 61 | type TestDocSLD struct { 62 | StoreErr error 63 | Stored map[string]*api.Document 64 | IterateErr error 65 | LoadErr error 66 | MacErr error 67 | DeleteErr error 68 | mu sync.Mutex 69 | } 70 | 71 | // Store mocks DocumentSLD.Store(). 72 | func (f *TestDocSLD) Store(key id.ID, value *api.Document) error { 73 | f.mu.Lock() 74 | defer f.mu.Unlock() 75 | f.Stored[key.String()] = value 76 | return f.StoreErr 77 | } 78 | 79 | // Iterate mocks DocumentSLD.Iterate(). 80 | func (f *TestDocSLD) Iterate(done chan struct{}, callback func(key id.ID, value []byte)) error { 81 | for keyStr, value := range f.Stored { 82 | key, err := id.FromString(keyStr) 83 | errors.MaybePanic(err) 84 | valueBytes, err := proto.Marshal(value) 85 | errors.MaybePanic(err) // should never happen b/c only docs can be stored 86 | select { 87 | case <-done: 88 | return f.IterateErr 89 | default: 90 | callback(key, valueBytes) 91 | } 92 | } 93 | return f.IterateErr 94 | } 95 | 96 | // Load mocks DocumentSLD.Load(). 97 | func (f *TestDocSLD) Load(key id.ID) (*api.Document, error) { 98 | f.mu.Lock() 99 | defer f.mu.Unlock() 100 | value := f.Stored[key.String()] 101 | return value, f.LoadErr 102 | } 103 | 104 | // Mac mocks DocumentSLD.Mac(). 105 | func (f *TestDocSLD) Mac(key id.ID, macKey []byte) ([]byte, error) { 106 | return nil, f.MacErr 107 | } 108 | 109 | // Delete mocks DocumentSLD.Delete(). 110 | func (f *TestDocSLD) Delete(key id.ID) error { 111 | f.mu.Lock() 112 | defer f.mu.Unlock() 113 | delete(f.Stored, key.String()) 114 | return f.DeleteErr 115 | } 116 | -------------------------------------------------------------------------------- /libri/author/io/enc/mac.go: -------------------------------------------------------------------------------- 1 | package enc 2 | 3 | import ( 4 | "bytes" 5 | "crypto/hmac" 6 | "crypto/sha256" 7 | "errors" 8 | "hash" 9 | "io" 10 | 11 | cerrors "github.com/drausin/libri/libri/common/errors" 12 | "github.com/drausin/libri/libri/librarian/api" 13 | ) 14 | 15 | // ErrUnexpectedCiphertextSize indicates when the ciphertext size does not match the expected value. 16 | var ErrUnexpectedCiphertextSize = errors.New("unexpected ciphertext size") 17 | 18 | // ErrUnexpectedCiphertextMAC indicates when the ciphertext MAC does not match the expected value. 19 | var ErrUnexpectedCiphertextMAC = errors.New("unexpected ciphertext MAC") 20 | 21 | // ErrUnexpectedUncompressedSize indicates when the uncompressed size does not match the expected 22 | // value. 23 | var ErrUnexpectedUncompressedSize = errors.New("unexpected uncompressed size") 24 | 25 | // ErrUnexpectedUncompressedMAC indicates when the uncompressed MAC does not match the expected 26 | // value. 27 | var ErrUnexpectedUncompressedMAC = errors.New("unexpected uncompressed MAC") 28 | 29 | // MAC wraps a hash function to return a message authentication code (MAC) and the total number 30 | // of bytes it has digested. 31 | type MAC interface { 32 | io.Writer 33 | 34 | // Sum returns the MAC after writing the given bytes. 35 | Sum(in []byte) []byte 36 | 37 | // Reset resets the MAC. 38 | Reset() 39 | 40 | // MessageSize returns the total number of digested bytes. 41 | MessageSize() uint64 42 | } 43 | 44 | type sizeHMAC struct { 45 | inner hash.Hash 46 | size uint64 47 | } 48 | 49 | // NewHMAC returns a MAC internally using an HMAC-256 with a a given key. 50 | func NewHMAC(hmacKey []byte) MAC { 51 | return &sizeHMAC{ 52 | inner: hmac.New(sha256.New, hmacKey), 53 | } 54 | } 55 | 56 | func (h *sizeHMAC) Write(p []byte) (int, error) { 57 | n, err := h.inner.Write(p) 58 | h.size += uint64(n) 59 | return n, err 60 | } 61 | 62 | func (h *sizeHMAC) Sum(in []byte) []byte { 63 | return h.inner.Sum(in) 64 | } 65 | 66 | func (h *sizeHMAC) Reset() { 67 | h.inner.Reset() 68 | } 69 | 70 | func (h *sizeHMAC) MessageSize() uint64 { 71 | return h.size 72 | } 73 | 74 | // HMAC returns the HMAC sum for the given input bytes and HMAC-256 key. 75 | func HMAC(p []byte, hmacKey []byte) []byte { 76 | macer := NewHMAC(hmacKey) 77 | _, err := macer.Write(p) 78 | cerrors.MaybePanic(err) // should never happen b/c sha256.Write always returns nil error 79 | return macer.Sum(nil) 80 | } 81 | 82 | // CheckMACs checks that the ciphertext and uncompressed MACs are consistent with the *api.Metadata. 83 | func CheckMACs(ciphertextMAC, uncompressedMAC MAC, md *api.EntryMetadata) error { 84 | if err := api.ValidateEntryMetadata(md); err != nil { 85 | return err 86 | } 87 | if md.CiphertextSize != ciphertextMAC.MessageSize() { 88 | return ErrUnexpectedCiphertextSize 89 | } 90 | if !bytes.Equal(md.CiphertextMac, ciphertextMAC.Sum(nil)) { 91 | return ErrUnexpectedCiphertextMAC 92 | } 93 | if md.UncompressedSize != uncompressedMAC.MessageSize() { 94 | return ErrUnexpectedUncompressedSize 95 | } 96 | if !bytes.Equal(md.UncompressedMac, uncompressedMAC.Sum(nil)) { 97 | return ErrUnexpectedUncompressedMAC 98 | } 99 | return nil 100 | } 101 | -------------------------------------------------------------------------------- /libri/author/io/enc/metadata.go: -------------------------------------------------------------------------------- 1 | package enc 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | 7 | "github.com/drausin/libri/libri/librarian/api" 8 | "github.com/golang/protobuf/proto" 9 | ) 10 | 11 | // ErrUnexpectedMAC occurs when the calculated MAC did not match the expected MAC. 12 | var ErrUnexpectedMAC = errors.New("unexpected MAC") 13 | 14 | // EncryptedMetadata contains both the ciphertext and ciphertext MAC involved in encrypting an 15 | // *api.EntryMetadata instance. 16 | type EncryptedMetadata struct { 17 | Ciphertext []byte 18 | CiphertextMAC []byte 19 | } 20 | 21 | // NewEncryptedMetadata creates a new *EncryptedMetadata instance if the ciphertext and 22 | // ciphertextMAC are valid. 23 | func NewEncryptedMetadata(ciphertext, ciphertextMAC []byte) (*EncryptedMetadata, error) { 24 | if err := api.ValidateNotEmpty(ciphertext, "MetadataCiphertext"); err != nil { 25 | return nil, err 26 | } 27 | if err := api.ValidateHMAC256(ciphertextMAC); err != nil { 28 | return nil, err 29 | } 30 | return &EncryptedMetadata{ 31 | Ciphertext: ciphertext, 32 | CiphertextMAC: ciphertextMAC, 33 | }, nil 34 | } 35 | 36 | // EntryMetadataEncrypter encrypts *api.EntryMetadata. 37 | type EntryMetadataEncrypter interface { 38 | // EncryptMetadata encrypts an *api.EntryMetadata instance using the AES key and the MetadataIV 39 | // key for the MAC. 40 | Encrypt(m *api.EntryMetadata, keys *EEK) (*EncryptedMetadata, error) 41 | } 42 | 43 | // MetadataDecrypter decrypts *EncryptedMetadata. 44 | type MetadataDecrypter interface { 45 | // Decrypt decrypts an *EncryptedMetadata instance, using the AES key and the MetadataIV 46 | // key. It returns UnexpectedMACErr if the calculated ciphertext MAC does not match the 47 | // expected ciphertext MAC. 48 | Decrypt(em *EncryptedMetadata, keys *EEK) (*api.EntryMetadata, error) 49 | } 50 | 51 | // MetadataEncrypterDecrypter encrypts *api.EntryMetadata and decrypts *EncryptedMetadata. 52 | type MetadataEncrypterDecrypter interface { 53 | EntryMetadataEncrypter 54 | MetadataDecrypter 55 | } 56 | 57 | type metadataEncDec struct{} 58 | 59 | // NewMetadataEncrypterDecrypter creates a new MetadataEncrypterDecrypter. 60 | func NewMetadataEncrypterDecrypter() MetadataEncrypterDecrypter { 61 | return metadataEncDec{} 62 | } 63 | 64 | func (metadataEncDec) Encrypt(m *api.EntryMetadata, keys *EEK) (*EncryptedMetadata, error) { 65 | mPlaintext, err := proto.Marshal(m) 66 | if err != nil { 67 | return nil, err 68 | } 69 | cipher, err := newGCMCipher(keys.AESKey) 70 | if err != nil { 71 | return nil, err 72 | } 73 | mCiphertext := cipher.Seal(nil, keys.MetadataIV, mPlaintext, nil) 74 | return NewEncryptedMetadata(mCiphertext, HMAC(mCiphertext, keys.HMACKey)) 75 | } 76 | 77 | func (metadataEncDec) Decrypt(em *EncryptedMetadata, keys *EEK) (*api.EntryMetadata, error) { 78 | mac := HMAC(em.Ciphertext, keys.HMACKey) 79 | if !bytes.Equal(em.CiphertextMAC, mac) { 80 | return nil, ErrUnexpectedMAC 81 | } 82 | cipher, err := newGCMCipher(keys.AESKey) 83 | if err != nil { 84 | return nil, err 85 | } 86 | mPlaintext, err := cipher.Open(nil, keys.MetadataIV, em.Ciphertext, nil) 87 | if err != nil { 88 | return nil, err 89 | } 90 | m := &api.EntryMetadata{} 91 | if err := proto.Unmarshal(mPlaintext, m); err != nil { 92 | return nil, err 93 | } 94 | return m, nil 95 | } 96 | -------------------------------------------------------------------------------- /libri/librarian/client/pool_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "testing" 5 | 6 | "io" 7 | 8 | "github.com/pkg/errors" 9 | "github.com/stretchr/testify/assert" 10 | "google.golang.org/grpc" 11 | ) 12 | 13 | func TestNewDefaultLRUPool(t *testing.T) { 14 | p, err := NewDefaultLRUPool() 15 | assert.Nil(t, err) 16 | assert.NotNil(t, p) 17 | } 18 | 19 | func TestLRUPool_Get_ok(t *testing.T) { 20 | cc := &grpc.ClientConn{} 21 | dialer := &fixedDialer{conn: cc} 22 | closer := &fixedCloser{} 23 | p, err := newLRUPool(1, dialer, closer) 24 | assert.Nil(t, err) 25 | addr1, addr2 := "some address", "some other address" 26 | 27 | // creates new connection 28 | lc1, err := p.Get(addr1) 29 | assert.Nil(t, err) 30 | assert.NotNil(t, lc1) 31 | assert.Equal(t, 1, dialer.nCalls) 32 | 33 | // uses same connection 34 | lc2, err := p.Get(addr1) 35 | assert.Nil(t, err) 36 | assert.NotNil(t, lc2) 37 | assert.Equal(t, lc1, lc2) 38 | assert.Equal(t, 1, dialer.nCalls) 39 | 40 | // creates new connection, evicting old connection 41 | lc3, err := p.Get(addr2) 42 | assert.Nil(t, err) 43 | assert.NotNil(t, lc3) 44 | assert.Equal(t, 2, dialer.nCalls) 45 | } 46 | 47 | func TestLRUPool_Get_err(t *testing.T) { 48 | p, err := newLRUPool( 49 | 1, 50 | &fixedDialer{err: errors.New("some dial error")}, 51 | &fixedCloser{err: errors.New("some close error")}, 52 | ) 53 | assert.Nil(t, err) 54 | lc, err := p.Get("some address") 55 | assert.NotNil(t, err) 56 | assert.Nil(t, lc) 57 | 58 | p.(*lruPool).dialer = &fixedDialer{conn: &grpc.ClientConn{}} 59 | lc, err = p.Get("some address") 60 | assert.Nil(t, err) 61 | assert.NotNil(t, lc) 62 | 63 | lc, err = p.Get("some other address") 64 | assert.NotNil(t, err) 65 | assert.Nil(t, lc) 66 | } 67 | 68 | func TestLRUPool_CloseAll_ok(t *testing.T) { 69 | cc := &grpc.ClientConn{} 70 | dialer := &fixedDialer{conn: cc} 71 | closer := &fixedCloser{} 72 | p, err := newLRUPool(2, dialer, closer) 73 | assert.Nil(t, err) 74 | addr1, addr2 := "some address", "some other address" 75 | 76 | lc1, err := p.Get(addr1) 77 | assert.Nil(t, err) 78 | assert.NotNil(t, lc1) 79 | lc2, err := p.Get(addr2) 80 | assert.Nil(t, err) 81 | assert.NotNil(t, lc2) 82 | 83 | err = p.CloseAll() 84 | assert.Nil(t, err) 85 | assert.Equal(t, 0, p.(*lruPool).conns.Len()) 86 | } 87 | 88 | func TestLRUPool_CloseAll_err(t *testing.T) { 89 | cc := &grpc.ClientConn{} 90 | dialer := &fixedDialer{conn: cc} 91 | closer := &fixedCloser{err: errors.New("some close error")} 92 | p, err := newLRUPool(2, dialer, closer) 93 | assert.Nil(t, err) 94 | addr1, addr2 := "some address", "some other address" 95 | 96 | lc1, err := p.Get(addr1) 97 | assert.Nil(t, err) 98 | assert.NotNil(t, lc1) 99 | lc2, err := p.Get(addr2) 100 | assert.Nil(t, err) 101 | assert.NotNil(t, lc2) 102 | 103 | err = p.CloseAll() 104 | assert.NotNil(t, err) 105 | } 106 | 107 | type fixedDialer struct { 108 | conn *grpc.ClientConn 109 | err error 110 | nCalls int 111 | } 112 | 113 | func (fd *fixedDialer) dial(address string) (*grpc.ClientConn, error) { 114 | fd.nCalls++ 115 | return fd.conn, fd.err 116 | } 117 | 118 | type fixedCloser struct { 119 | err error 120 | } 121 | 122 | func (fc *fixedCloser) close(conn io.Closer) error { 123 | return fc.err 124 | } 125 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | working_directory: /go/src/github.com/drausin/libri 5 | docker: 6 | - image: daedalus2718/libri-build 7 | parallelism: 2 8 | steps: 9 | - checkout 10 | - setup_remote_docker 11 | - restore_cache: 12 | keys: 13 | - v1-vendor-{{ arch }}-{{ checksum "Gopkg.lock" }}-rdb5.15.10 14 | - run: 15 | name: Install dependencies 16 | command: make get-deps 17 | - run: 18 | name: Install RocksDB 19 | command: | 20 | # hack to make sure RocksDB runs on CircleCI machines 21 | if [[ $(md5sum /usr/local/lib/librocksdb.a | awk '{print $1}') != "c0adac05c11848a439ec1744b32efb7e" ]]; then 22 | ./build/install-rocksdb.sh 23 | fi 24 | - run: 25 | name: Install gorocksdb 26 | command: ./build/install-gorocksdb.sh 27 | - save_cache: 28 | key: v1-vendor-{{ arch }}-{{ checksum "Gopkg.lock" }}-rdb5.15.10 29 | paths: 30 | - vendor 31 | - /usr/local/lib/librocksdb.a 32 | - run: 33 | name: Build 34 | command: | 35 | if [[ ${CIRCLE_NODE_INDEX} -eq 0 ]]; then 36 | make build # ensure everything builds ok 37 | make build-static # build linux binary for Docker image 38 | make docker-image # ensure Docker image builds ok, even though only used on deployment 39 | fi 40 | - store_artifacts: 41 | path: deploy/bin 42 | - run: 43 | name: Test 44 | command: | 45 | case ${CIRCLE_NODE_INDEX} in 46 | 0) 47 | if [[ "${CIRCLE_BRANCH}" == "develop-intensive-build" ]]; then 48 | make test-stress # both unit and acceptance tests 49 | else 50 | make acceptance 51 | fi 52 | make demo 53 | ;; 54 | 1) 55 | make test-cover 56 | bash <(curl -s https://codecov.io/bash) -f artifacts/cover/test-coverage-merged.cov 57 | ;; 58 | esac 59 | - run: 60 | name: Lint 61 | command: | 62 | if [[ ${CIRCLE_NODE_INDEX} -eq 0 ]]; then 63 | make lint 64 | fi 65 | 66 | - deploy: 67 | name: Publish docker image 68 | command: | 69 | if [[ "${CIRCLE_BRANCH}" == "master" ]]; then 70 | docker login -u ${DOCKER_USER} -p ${DOCKER_PASS} 71 | LIBRI_VERSION=$(./deploy/bin/libri version) 72 | docker tag daedalus2718/libri:snapshot daedalus2718/libri:${LIBRI_VERSION} 73 | docker tag daedalus2718/libri:snapshot daedalus2718/libri:latest 74 | docker push daedalus2718/libri:${LIBRI_VERSION} 75 | docker push daedalus2718/libri:latest 76 | elif [[ "${CIRCLE_BRANCH}" == "develop" ]]; then 77 | docker login -u ${DOCKER_USER} -p ${DOCKER_PASS} 78 | docker push daedalus2718/libri:snapshot 79 | fi 80 | - run: 81 | name: Run benchmarks 82 | command: | 83 | if [[ "${CIRCLE_BRANCH}" == "develop" && ${CIRCLE_NODE_INDEX} -eq 0 ]]; then 84 | make bench 85 | fi 86 | - store_artifacts: 87 | path: artifacts 88 | -------------------------------------------------------------------------------- /libri/common/ecid/storage.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // source: libri/common/ecid/storage.proto 3 | 4 | /* 5 | Package ecid is a generated protocol buffer package. 6 | 7 | It is generated from these files: 8 | libri/common/ecid/storage.proto 9 | 10 | It has these top-level messages: 11 | ECDSAPrivateKey 12 | */ 13 | package ecid 14 | 15 | import proto "github.com/golang/protobuf/proto" 16 | import fmt "fmt" 17 | import math "math" 18 | 19 | // Reference imports to suppress errors if they are not otherwise used. 20 | var _ = proto.Marshal 21 | var _ = fmt.Errorf 22 | var _ = math.Inf 23 | 24 | // This is a compile-time assertion to ensure that this generated file 25 | // is compatible with the proto package it is being compiled against. 26 | // A compilation error at this line likely means your copy of the 27 | // proto package needs to be updated. 28 | const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package 29 | 30 | // ECDSAPrivateKey represents an ECDSA key-pair, whose public key x-value is used as the peer ID 31 | // to outside world. 32 | type ECDSAPrivateKey struct { 33 | // name of the curve used 34 | Curve string `protobuf:"bytes,1,opt,name=curve" json:"curve,omitempty"` 35 | // private key 36 | D []byte `protobuf:"bytes,2,opt,name=D,proto3" json:"D,omitempty"` 37 | // x-value of public key 38 | X []byte `protobuf:"bytes,3,opt,name=X,proto3" json:"X,omitempty"` 39 | // y-value of public key 40 | Y []byte `protobuf:"bytes,4,opt,name=Y,proto3" json:"Y,omitempty"` 41 | } 42 | 43 | func (m *ECDSAPrivateKey) Reset() { *m = ECDSAPrivateKey{} } 44 | func (m *ECDSAPrivateKey) String() string { return proto.CompactTextString(m) } 45 | func (*ECDSAPrivateKey) ProtoMessage() {} 46 | func (*ECDSAPrivateKey) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } 47 | 48 | func (m *ECDSAPrivateKey) GetCurve() string { 49 | if m != nil { 50 | return m.Curve 51 | } 52 | return "" 53 | } 54 | 55 | func (m *ECDSAPrivateKey) GetD() []byte { 56 | if m != nil { 57 | return m.D 58 | } 59 | return nil 60 | } 61 | 62 | func (m *ECDSAPrivateKey) GetX() []byte { 63 | if m != nil { 64 | return m.X 65 | } 66 | return nil 67 | } 68 | 69 | func (m *ECDSAPrivateKey) GetY() []byte { 70 | if m != nil { 71 | return m.Y 72 | } 73 | return nil 74 | } 75 | 76 | func init() { 77 | proto.RegisterType((*ECDSAPrivateKey)(nil), "ecid.ECDSAPrivateKey") 78 | } 79 | 80 | func init() { proto.RegisterFile("libri/common/ecid/storage.proto", fileDescriptor0) } 81 | 82 | var fileDescriptor0 = []byte{ 83 | // 132 bytes of a gzipped FileDescriptorProto 84 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0xcf, 0xc9, 0x4c, 0x2a, 85 | 0xca, 0xd4, 0x4f, 0xce, 0xcf, 0xcd, 0xcd, 0xcf, 0xd3, 0x4f, 0x4d, 0xce, 0x4c, 0xd1, 0x2f, 0x2e, 86 | 0xc9, 0x2f, 0x4a, 0x4c, 0x4f, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x01, 0x89, 0x29, 87 | 0x05, 0x72, 0xf1, 0xbb, 0x3a, 0xbb, 0x04, 0x3b, 0x06, 0x14, 0x65, 0x96, 0x25, 0x96, 0xa4, 0x7a, 88 | 0xa7, 0x56, 0x0a, 0x89, 0x70, 0xb1, 0x26, 0x97, 0x16, 0x95, 0xa5, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 89 | 0x70, 0x06, 0x41, 0x38, 0x42, 0x3c, 0x5c, 0x8c, 0x2e, 0x12, 0x4c, 0x0a, 0x8c, 0x1a, 0x3c, 0x41, 90 | 0x8c, 0x2e, 0x20, 0x5e, 0x84, 0x04, 0x33, 0x84, 0x17, 0x01, 0xe2, 0x45, 0x4a, 0xb0, 0x40, 0x78, 91 | 0x91, 0x49, 0x6c, 0x60, 0xf3, 0x8d, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0xb6, 0x4a, 0x7c, 0x21, 92 | 0x82, 0x00, 0x00, 0x00, 93 | } 94 | -------------------------------------------------------------------------------- /libri/librarian/api/librarian.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "golang.org/x/net/context" 5 | "google.golang.org/grpc" 6 | ) 7 | 8 | // Endpoint defines an enum for the librarian API endpoints. 9 | type Endpoint int 10 | 11 | const ( 12 | // All represents all endpoints, usually used when aggregating counts across all endpoints. 13 | All Endpoint = iota - 1 14 | 15 | // Introduce represents the Introduce endpoint. 16 | Introduce 17 | 18 | // Find represents the Introduce endpoint. 19 | Find 20 | 21 | // Store represents the Introduce endpoint. 22 | Store 23 | 24 | // Verify represents the Introduce endpoint. 25 | Verify 26 | 27 | // Get represents the Introduce endpoint. 28 | Get 29 | 30 | // Put represents the Introduce endpoint. 31 | Put 32 | 33 | // Subscribe represents the Introduce endpoint. 34 | Subscribe 35 | ) 36 | 37 | var ( 38 | // Endpoints is a list of all the librarian endpoints (not including All). 39 | Endpoints = []Endpoint{Introduce, Find, Store, Verify, Get, Put, Subscribe} 40 | ) 41 | 42 | func (e Endpoint) String() string { 43 | switch e { 44 | case All: 45 | return "All" 46 | case Introduce: 47 | return "Introduce" 48 | case Find: 49 | return "Find" 50 | case Store: 51 | return "Store" 52 | case Verify: 53 | return "Verify" 54 | case Get: 55 | return "Get" 56 | case Put: 57 | return "Put" 58 | case Subscribe: 59 | return "Subscribe" 60 | default: 61 | panic("unknown endpoint") 62 | } 63 | } 64 | 65 | // These interfaces split up the methods of LibrarianClient, mostly to allow for narrow interface 66 | // usage and testing. 67 | 68 | // Introducer issues Introduce queries. 69 | type Introducer interface { 70 | // Introduce identifies the node by name and ID. 71 | Introduce(ctx context.Context, in *IntroduceRequest, opts ...grpc.CallOption) ( 72 | *IntroduceResponse, error) 73 | } 74 | 75 | // Finder issues Find queries. 76 | type Finder interface { 77 | // Find returns the value for a key or the closest peers to it. 78 | Find(ctx context.Context, in *FindRequest, opts ...grpc.CallOption) (*FindResponse, error) 79 | } 80 | 81 | // Storer issues Store queries. 82 | type Storer interface { 83 | // Store stores a value in a given key. 84 | Store(ctx context.Context, in *StoreRequest, opts ...grpc.CallOption) (*StoreResponse, 85 | error) 86 | } 87 | 88 | // Verifier issues Verify queries. 89 | type Verifier interface { 90 | // Verify verifies that a peer has a given value. 91 | Verify(ctx context.Context, in *VerifyRequest, opts ...grpc.CallOption) (*VerifyResponse, error) 92 | } 93 | 94 | // Getter issues Get queries. 95 | type Getter interface { 96 | // Get retrieves a value, if it exists. 97 | Get(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetResponse, error) 98 | } 99 | 100 | // Putter issues Put queries. 101 | type Putter interface { 102 | // Put stores a value. 103 | Put(ctx context.Context, in *PutRequest, opts ...grpc.CallOption) (*PutResponse, error) 104 | } 105 | 106 | // PutterGetter issues Put and Get queries. 107 | type PutterGetter interface { 108 | Getter 109 | Putter 110 | } 111 | 112 | // Subscriber issues Subscribe queries. 113 | type Subscriber interface { 114 | // Subscribe subscribes to a defined publication stream. 115 | Subscribe(ctx context.Context, in *SubscribeRequest, opts ...grpc.CallOption) ( 116 | Librarian_SubscribeClient, error) 117 | } 118 | --------------------------------------------------------------------------------