├── .circleci └── config.yml ├── .dockerignore ├── .gitignore ├── .goreleaser.yaml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── docker ├── circleci │ ├── Dockerfile │ ├── Makefile │ └── README.md ├── dev-cluster │ ├── Dockerfile.compose │ ├── Dockerfile.natsboard │ ├── docker-compose.yml │ └── wait-for.sh └── dev-standalone │ ├── Dockerfile │ ├── LICENSE │ ├── README.md │ ├── liftbridge.yaml │ └── nats-server.conf ├── documentation ├── activity.md ├── assets │ ├── cluster.png │ ├── consumer_group_aggregation.png │ ├── consumer_group_fault_tolerance.png │ ├── consumer_group_load_balancing.png │ ├── consumer_group_publish_subscribe.png │ └── high-level.png ├── authentication_authorization.md ├── client_implementation.md ├── clients.md ├── concepts.md ├── configuration.md ├── consumer_groups.md ├── cursors.md ├── deployment.md ├── embedded_nats.md ├── envelope_protocol.md ├── faq.md ├── feature_comparison.md ├── ha_and_consistency_configuration.md ├── overview.md ├── pausing_streams.md ├── quick_start.md ├── replication_protocol.md ├── roadmap.md └── scalability_configuration.md ├── go.mod ├── go.sum ├── k8s ├── Dockerfile.k8s ├── base │ ├── config.yaml │ ├── kustomization.yaml │ ├── statefulset.yaml │ └── svc.yaml ├── dev │ ├── kind.yaml │ ├── kustomization.yaml │ ├── namespace.yaml │ ├── nats-operator.yaml │ ├── nats.yaml │ ├── natsboard.yaml │ ├── patches.yaml │ ├── storageclass.yaml │ └── svc.yaml ├── empty │ └── kustomization.yaml └── entrypoint.sh ├── main.go ├── main_test.go ├── server ├── activity.go ├── activity_test.go ├── api.go ├── api_test.go ├── authz.go ├── commitlog │ ├── commitlog.go │ ├── commitlog_test.go │ ├── compact_cleaner.go │ ├── compact_cleaner_test.go │ ├── delete_cleaner.go │ ├── delete_cleaner_test.go │ ├── encoder.go │ ├── index.go │ ├── index_test.go │ ├── interface.go │ ├── leader_epoch_cache.go │ ├── leader_epoch_cache_test.go │ ├── message.go │ ├── message_set.go │ ├── reader.go │ ├── reader_test.go │ ├── segment.go │ ├── segment_test.go │ ├── util.go │ └── util_test.go ├── common_test.go ├── config.go ├── config_test.go ├── configs │ ├── authz │ │ ├── model.conf │ │ └── policy.csv │ ├── certs │ │ ├── ca-cert.pem │ │ ├── ca-cert.srl │ │ ├── ca-key.pem │ │ ├── cert_maker │ │ │ ├── Readme.md │ │ │ ├── client-ext.cnf │ │ │ ├── gen.sh │ │ │ └── server-ext.cnf │ │ ├── client │ │ │ ├── client-cert.pem │ │ │ ├── client-key.pem │ │ │ └── client-req.pem │ │ └── server │ │ │ ├── server-cert.pem │ │ │ ├── server-key.pem │ │ │ └── server-req.pem │ ├── full.yaml │ ├── host.yaml │ ├── invalid-listen.yaml │ ├── listen-host.yaml │ ├── listen.yaml │ ├── nats-auth.yaml │ ├── simple.yaml │ ├── tls-authz.yaml │ ├── tls-nats.yaml │ ├── tls.yaml │ └── unknown-setting.yaml ├── cursors.go ├── encryption │ ├── handler_interface.go │ ├── localkey_handler.go │ └── localkey_handler_test.go ├── failover.go ├── fsm.go ├── fsm_test.go ├── groups.go ├── groups_test.go ├── health │ └── health.go ├── logger │ └── logger.go ├── metadata.go ├── metadata_test.go ├── partition.go ├── partition_test.go ├── propagation.go ├── protocol │ ├── Makefile │ ├── envelope.go │ ├── envelope_test.go │ ├── internal.pb.go │ └── internal.proto ├── raft.go ├── replicator.go ├── replicator_test.go ├── server.go ├── server_test.go ├── signal.go ├── stream.go └── version.go ├── skaffold.yaml └── website ├── .gcloudignore ├── README.md ├── app.yaml ├── core └── Footer.js ├── i18n └── en.json ├── package-lock.json ├── package.json ├── pages └── en │ ├── acknowledgements.js │ ├── help.js │ ├── index.js │ └── versions.js ├── sidebars.json ├── siteConfig.js ├── static ├── css │ └── custom.css └── img │ ├── asterisk.png │ ├── favicon.png │ ├── key_value.png │ ├── liftbridge.png │ ├── liftbridge_full.png │ ├── liftbridge_icon.png │ ├── log_retention.png │ ├── nats.png │ ├── scalability.png │ ├── use_cases.svg │ └── zen.png ├── versioned_docs ├── version-v0.0.1-alpha │ ├── client_implementation.md │ ├── clients.md │ ├── concepts.md │ ├── configuration.md │ ├── deployment.md │ ├── faq.md │ ├── ha_and_consistency_configuration.md │ ├── overview.md │ ├── quick_start.md │ └── replication_protocol.md ├── version-v1.0.0-alpha │ ├── client_implementation.md │ ├── clients.md │ ├── concepts.md │ ├── configuration.md │ ├── deployment.md │ ├── envelope_protocol.md │ ├── faq.md │ ├── ha_and_consistency_configuration.md │ ├── overview.md │ ├── quick_start.md │ └── replication_protocol.md ├── version-v1.0.0-beta │ ├── client_implementation.md │ ├── clients.md │ ├── concepts.md │ ├── configuration.md │ ├── deployment.md │ ├── envelope_protocol.md │ ├── faq.md │ ├── ha_and_consistency_configuration.md │ ├── overview.md │ ├── quick_start.md │ └── replication_protocol.md ├── version-v1.0.0 │ ├── activity.md │ ├── client_implementation.md │ ├── clients.md │ ├── concepts.md │ ├── configuration.md │ ├── deployment.md │ ├── faq.md │ ├── feature_comparison.md │ ├── ha_and_consistency_configuration.md │ ├── overview.md │ ├── pausing_streams.md │ └── quick_start.md ├── version-v1.1.0 │ ├── configuration.md │ └── quick_start.md ├── version-v1.2.0 │ ├── client_implementation.md │ ├── configuration.md │ ├── quick_start.md │ └── roadmap.md ├── version-v1.3.0 │ ├── client_implementation.md │ ├── clients.md │ ├── concepts.md │ ├── configuration.md │ ├── cursors.md │ ├── feature_comparison.md │ ├── pausing_streams.md │ ├── quick_start.md │ └── roadmap.md ├── version-v1.4.0 │ ├── configuration.md │ └── quick_start.md ├── version-v1.4.1 │ ├── configuration.md │ └── quick_start.md ├── version-v1.5.0 │ ├── client_implementation.md │ ├── concepts.md │ ├── configuration.md │ ├── embedded_nats.md │ ├── quick_start.md │ ├── replication_protocol.md │ ├── roadmap.md │ └── scalability_configuration.md ├── version-v1.5.1 │ ├── configuration.md │ └── quick_start.md ├── version-v1.6.0 │ ├── client_implementation.md │ ├── concepts.md │ ├── configuration.md │ ├── feature_comparison.md │ ├── quick_start.md │ └── roadmap.md ├── version-v1.7.0 │ ├── configuration.md │ └── quick_start.md ├── version-v1.7.1 │ ├── authentication_authorization.md │ ├── configuration.md │ └── quick_start.md ├── version-v1.8.0 │ ├── concepts.md │ ├── configuration.md │ ├── consumer_groups.md │ ├── cursors.md │ ├── faq.md │ ├── feature_comparison.md │ ├── overview.md │ ├── quick_start.md │ ├── roadmap.md │ └── scalability_configuration.md └── version-v1.9.0 │ ├── activity.md │ ├── authentication_authorization.md │ ├── configuration.md │ ├── consumer_groups.md │ ├── quick_start.md │ └── roadmap.md ├── versioned_sidebars ├── version-v0.0.1-alpha-sidebars.json ├── version-v1.0.0-alpha-sidebars.json ├── version-v1.0.0-beta-sidebars.json ├── version-v1.0.0-sidebars.json ├── version-v1.2.0-sidebars.json ├── version-v1.3.0-sidebars.json ├── version-v1.5.0-sidebars.json ├── version-v1.7.1-sidebars.json └── version-v1.8.0-sidebars.json ├── versions.json └── yarn.lock /.dockerignore: -------------------------------------------------------------------------------- 1 | dev/data 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | 16 | # Ignore jetbrains goland/idea setting path 17 | .idea/ 18 | 19 | # Ignore website node modules and build 20 | website/node_modules/ 21 | website/build/ 22 | website/deploy/build/ 23 | 24 | dev/data 25 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod download 4 | 5 | builds: 6 | - env: 7 | - CGO_ENABLED=0 8 | goos: 9 | - darwin 10 | - linux 11 | - windows 12 | goarch: 13 | - amd64 14 | - 386 15 | - arm64 16 | 17 | changelog: 18 | skip: true 19 | 20 | checksum: 21 | name_template: 'checksums.txt' 22 | 23 | snapshot: 24 | name_template: "{{ .Tag }}-next" 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17-alpine as build-base 2 | RUN apk update && apk upgrade && \ 3 | apk add --no-cache bash git openssh make 4 | ADD . /go/src/github.com/liftbridge-io/liftbridge 5 | WORKDIR /go/src/github.com/liftbridge-io/liftbridge 6 | ENV GO111MODULE on 7 | ENV CGO_ENABLED 0 8 | ENV GOARCH amd64 9 | ENV GOOS linux 10 | RUN go build -mod=readonly -o liftbridge 11 | 12 | FROM alpine:latest 13 | RUN addgroup -g 1001 -S liftbridge && adduser -u 1001 -S liftbridge -G liftbridge 14 | COPY --chown=liftbridge:liftbridge --from=build-base /go/src/github.com/liftbridge-io/liftbridge/liftbridge /usr/local/bin/liftbridge 15 | EXPOSE 9292 16 | VOLUME "/tmp/liftbridge/liftbridge-default" 17 | ENTRYPOINT ["liftbridge"] 18 | USER liftbridge 19 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | KIND_CLUSTER_NAME=kind 2 | KIND_KUBECONFIG:=~/.kube/kind-config-$(KIND_CLUSTER_NAME) 3 | 4 | .PHONY: compose-up 5 | compose-up: 6 | cd docker/dev-cluster/; docker-compose up --build 7 | 8 | .PHONY: compose-down 9 | compose-down: 10 | cd docker/dev-cluster/; docker-compose down --rmi local 11 | 12 | .PHONY: push-k8s-image 13 | push-k8s-image: 14 | skaffold run -p deploy-k8s-image 15 | 16 | .PHONY: kind-up 17 | kind-up: 18 | kind create cluster --config k8s/dev/kind.yaml --name=$(KIND_CLUSTER_NAME) 19 | 20 | .PHONY: kind-down 21 | kind-down: 22 | kind delete cluster --name=$(KIND_CLUSTER_NAME) 23 | 24 | .PHONY: kind-apply 25 | kind-apply: 26 | KUBECONFIG=$(KIND_KUBECONFIG) kubectl apply -R -f k8s/dev/nats-operator.yaml 27 | KUBECONFIG=$(KIND_KUBECONFIG) skaffold dev -p dev 28 | 29 | .PHONY: kind-export 30 | kind-export: 31 | echo export KUBECONFIG="$$(kind get kubeconfig-path --name="$(KIND_CLUSTER_NAME)")" 32 | 33 | build: liftbridge 34 | liftbridge: 35 | GO111MODULE=on CGO_ENABLED=0 go build -mod=readonly -o liftbridge 36 | 37 | build-dev: liftbridge-dev 38 | liftbridge-dev: 39 | CGO_ENABLED=1 go build -tags netgo -ldflags '-extldflags "-static"' -mod=readonly -o liftbridge-dev 40 | 41 | .PHONY: clean 42 | clean: 43 | rm -f liftbridge 44 | rm -f liftbridge-dev 45 | 46 | .PHONY: website-deploy 47 | website-deploy: website/build 48 | gcloud app deploy $(WEBSITE_DEPLOY_FLAGS) website/app.yaml 49 | 50 | website/build: 51 | yarn --cwd website run build 52 | 53 | .PHONY: website-clean 54 | website-clean: 55 | rm -rf website/build 56 | 57 | .PHONY: website-up 58 | website-up: 59 | yarn --cwd website start 60 | 61 | 62 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Liftbridge Logo](./website/static/img/liftbridge_full.png) 2 | --- 3 | [![Build][Build-Status-Image]][Build-Status-Url] [![License][License-Image]][License-Url] [![ReportCard][ReportCard-Image]][ReportCard-Url] [![Coverage][Coverage-Image]][Coverage-Url] 4 | 5 | Liftbridge provides lightweight, fault-tolerant message streams by implementing 6 | a durable, replicated, and scalable message log. The vision for Liftbridge is 7 | to deliver a "Kafka-lite" solution designed with the Go community first in 8 | mind. Unlike Kafka, which is built on the JVM and whose canonical client 9 | library is Java (or the C-based librdkafka), Liftbridge and its canonical 10 | client, [go-liftbridge](https://github.com/liftbridge-io/go-liftbridge), are 11 | implemented in Go. The ultimate goal of Liftbridge is to provide a lightweight 12 | message-streaming solution with a focus on simplicity and usability. Use it as 13 | a simpler and lighter alternative to systems like Kafka and Pulsar or to add 14 | streaming semantics to an existing NATS deployment. 15 | 16 | See the [introduction](https://bravenewgeek.com/introducing-liftbridge-lightweight-fault-tolerant-message-streams/) 17 | post on Liftbridge and [this post](https://bravenewgeek.com/building-a-distributed-log-from-scratch-part-5-sketching-a-new-system/) 18 | for more context and some of the inspiration behind it. 19 | 20 | ## Documentation 21 | 22 | - [Official documentation](https://liftbridge.io/docs/overview.html) 23 | - [Getting started](https://liftbridge.io/docs/quick-start.html) 24 | - [FAQ](https://liftbridge.io/docs/faq.html) 25 | - [Website](https://liftbridge.io) 26 | 27 | ## Community 28 | 29 | - [Slack](https://liftbridge.slack.com) - click [here](https://liftbridge.io/help.html) to request an invite 30 | - [Twitter](https://twitter.com/liftbridge_io) 31 | 32 | 33 | [License-Url]: https://www.apache.org/licenses/LICENSE-2.0 34 | [License-Image]: https://img.shields.io/badge/License-Apache2-blue.svg 35 | [Build-Status-Url]: https://circleci.com/gh/liftbridge-io/liftbridge 36 | [Build-Status-Image]: https://circleci.com/gh/liftbridge-io/liftbridge.svg?style=svg 37 | [ReportCard-Url]: https://goreportcard.com/report/github.com/liftbridge-io/liftbridge 38 | [ReportCard-Image]: https://goreportcard.com/badge/github.com/liftbridge-io/liftbridge 39 | [Coverage-Url]: https://coveralls.io/github/liftbridge-io/liftbridge?branch=master 40 | [Coverage-image]: https://coveralls.io/repos/github/liftbridge-io/liftbridge/badge.svg?branch=master 41 | -------------------------------------------------------------------------------- /docker/circleci/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM circleci/golang:1.17 2 | 3 | RUN sudo apt-get update -yqq && sudo apt-get install -yqq bzr 4 | 5 | WORKDIR /project 6 | 7 | ENV GO111MODULE="on" 8 | 9 | COPY go.mod /project/go.mod 10 | COPY go.sum /project/go.sum 11 | # cache deps before building and copying source so that we don't need to 12 | # re-download as much and so that source changes don't invalidate our downloaded 13 | # layer 14 | RUN go mod download 15 | -------------------------------------------------------------------------------- /docker/circleci/Makefile: -------------------------------------------------------------------------------- 1 | VERSION:=0.2.0 2 | build: 3 | @ docker build -t liftbridge/liftbridge-circleci:$(VERSION) -f Dockerfile ../../ 4 | publish: 5 | @ docker push liftbridge/liftbridge-circleci:$(VERSION) 6 | -------------------------------------------------------------------------------- /docker/circleci/README.md: -------------------------------------------------------------------------------- 1 | # Liftbridge build image 2 | 3 | This image is used in CircleCI to build the project. 4 | -------------------------------------------------------------------------------- /docker/dev-cluster/Dockerfile.compose: -------------------------------------------------------------------------------- 1 | FROM golang:1.17 as build-base 2 | WORKDIR /workspace 3 | ENV GO111MODULE on 4 | 5 | # Copy the Go Modules manifests 6 | COPY go.mod go.mod 7 | COPY go.sum go.sum 8 | # cache deps before building and copying source so that we don't need to re-download as much 9 | # and so that source changes don't invalidate our downloaded layer 10 | RUN go mod download 11 | 12 | # Copy the go source 13 | COPY main.go main.go 14 | COPY server/ server/ 15 | 16 | # Build 17 | RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=readonly -o liftbridge 18 | 19 | FROM debian:stretch-slim 20 | COPY --from=build-base /workspace/liftbridge /liftbridge 21 | RUN apt-get update && apt-get install -y netcat 22 | COPY docker/dev-cluster/wait-for.sh /wait-for.sh 23 | -------------------------------------------------------------------------------- /docker/dev-cluster/Dockerfile.natsboard: -------------------------------------------------------------------------------- 1 | FROM mhart/alpine-node:12 2 | 3 | RUN npm install natsboard -g 4 | RUN apk add --no-cache netcat-openbsd 5 | COPY wait-for.sh /wait-for.sh 6 | ENTRYPOINT ["/wait-for.sh"] 7 | -------------------------------------------------------------------------------- /docker/dev-cluster/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | nats: 4 | image: nats:2.6.4 5 | ports: 6 | - "4222:4222" 7 | networks: 8 | - overlay 9 | natsboard: 10 | build: 11 | context: . 12 | dockerfile: Dockerfile.natsboard 13 | ports: 14 | - "3000:3000" 15 | command: '/wait-for.sh natsboard --nats-mon-url http://nats:8222' 16 | environment: 17 | - WAIT_FOR_HOST=nats 18 | - WAIT_FOR_PORT=8222 19 | networks: 20 | - overlay 21 | liftbridge1: 22 | build: 23 | context: ../../ 24 | dockerfile: docker/dev-cluster/Dockerfile.compose 25 | ports: 26 | - "9292:9292" 27 | command: [ 28 | '/wait-for.sh', 29 | '/liftbridge', 30 | '--data-dir=/data1', 31 | '--port=9292', 32 | '--level=debug', 33 | '--nats-servers=nats:4222', 34 | '--raft-bootstrap-seed', 35 | '--id=server-1' 36 | ] 37 | environment: 38 | - WAIT_FOR_HOST=nats 39 | - WAIT_FOR_PORT=4222 40 | networks: 41 | - overlay 42 | volumes: 43 | - ./data/data1:/data1 44 | liftbridge2: 45 | build: 46 | context: ../../ 47 | dockerfile: docker/dev-cluster/Dockerfile.compose 48 | ports: 49 | - "9293:9293" 50 | command: [ 51 | '/wait-for.sh', 52 | '/liftbridge', 53 | '--data-dir=/data2', 54 | '--port=9293', 55 | '--level=debug', 56 | '--nats-servers=nats:4222', 57 | '--id=server-2' 58 | ] 59 | environment: 60 | - WAIT_FOR_HOST=nats 61 | - WAIT_FOR_PORT=4222 62 | networks: 63 | - overlay 64 | volumes: 65 | - ./data/data2:/data2 66 | liftbridge3: 67 | build: 68 | context: ../../ 69 | dockerfile: docker/dev-cluster/Dockerfile.compose 70 | ports: 71 | - "9294:9294" 72 | command: [ 73 | '/wait-for.sh', 74 | '/liftbridge', 75 | '--data-dir=/data3', 76 | '--port=9294', 77 | '--level=debug', 78 | '--nats-servers=nats:4222', 79 | '--id=server-3' 80 | ] 81 | environment: 82 | - WAIT_FOR_HOST=nats 83 | - WAIT_FOR_PORT=4222 84 | networks: 85 | - overlay 86 | volumes: 87 | - ./data/data3:/data3 88 | networks: 89 | overlay: 90 | driver: bridge 91 | -------------------------------------------------------------------------------- /docker/dev-cluster/wait-for.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | while ! nc -z $WAIT_FOR_HOST $WAIT_FOR_PORT; do sleep 1 ; echo waiting ; done 4 | 5 | exec "$@" 6 | -------------------------------------------------------------------------------- /docker/dev-standalone/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17-alpine as build-base 2 | 3 | ENV GO111MODULE on 4 | RUN go get github.com/liftbridge-io/liftbridge@master 5 | 6 | FROM alpine:latest 7 | COPY --from=build-base /go/bin/liftbridge /usr/local/bin/liftbridge 8 | 9 | EXPOSE 9292 4222 8222 6222 10 | 11 | VOLUME "/tmp/liftbridge/liftbridge-default" 12 | 13 | COPY docker/dev-standalone/nats-server.conf nats-server.conf 14 | COPY docker/dev-standalone/liftbridge.yaml liftbridge.yaml 15 | 16 | ENTRYPOINT ["liftbridge", "-c", "liftbridge.yaml"] 17 | -------------------------------------------------------------------------------- /docker/dev-standalone/liftbridge.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | host: 0.0.0.0 3 | clustering.raft.bootstrap.seed: true 4 | nats.embedded.config: nats-server.conf 5 | logging.nats: true 6 | -------------------------------------------------------------------------------- /docker/dev-standalone/nats-server.conf: -------------------------------------------------------------------------------- 1 | 2 | # Client port of 4222 on all interfaces 3 | port: 4222 4 | 5 | # HTTP monitoring port 6 | monitor_port: 8222 7 | 8 | # This is for clustering multiple servers together. 9 | cluster { 10 | 11 | # Route connections to be received on any interface on port 6222 12 | port: 6222 13 | 14 | # Routes are protected, so need to use them with --routes flag 15 | # e.g. --routes=nats-route://ruser:T0pS3cr3t@otherdockerhost:6222 16 | authorization { 17 | user: ruser 18 | password: T0pS3cr3t 19 | timeout: 0.75 20 | } 21 | 22 | # Routes are actively solicited and connected to from this server. 23 | # This Docker image has none by default, but you can pass a 24 | # flag to the nats-server docker image to create one to an existing server. 25 | routes = [] 26 | } -------------------------------------------------------------------------------- /documentation/assets/cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liftbridge-io/liftbridge/9d7e049a824aaeb014a8650bc1fe26817bbd8d9d/documentation/assets/cluster.png -------------------------------------------------------------------------------- /documentation/assets/consumer_group_aggregation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liftbridge-io/liftbridge/9d7e049a824aaeb014a8650bc1fe26817bbd8d9d/documentation/assets/consumer_group_aggregation.png -------------------------------------------------------------------------------- /documentation/assets/consumer_group_fault_tolerance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liftbridge-io/liftbridge/9d7e049a824aaeb014a8650bc1fe26817bbd8d9d/documentation/assets/consumer_group_fault_tolerance.png -------------------------------------------------------------------------------- /documentation/assets/consumer_group_load_balancing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liftbridge-io/liftbridge/9d7e049a824aaeb014a8650bc1fe26817bbd8d9d/documentation/assets/consumer_group_load_balancing.png -------------------------------------------------------------------------------- /documentation/assets/consumer_group_publish_subscribe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liftbridge-io/liftbridge/9d7e049a824aaeb014a8650bc1fe26817bbd8d9d/documentation/assets/consumer_group_publish_subscribe.png -------------------------------------------------------------------------------- /documentation/assets/high-level.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liftbridge-io/liftbridge/9d7e049a824aaeb014a8650bc1fe26817bbd8d9d/documentation/assets/high-level.png -------------------------------------------------------------------------------- /documentation/clients.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: clients 3 | title: Clients 4 | --- 5 | 6 | Currently, there are limited client libraries available for Liftbridge. 7 | However, Liftbridge uses gRPC for its client API, so low-level client libraries 8 | can be generated quite easily using the [Liftbridge protobuf 9 | definitions](https://github.com/liftbridge-io/liftbridge-api). See the [client 10 | implementation guidance](client_implementation.md) documentation for help on 11 | implementing a Liftbridge client library. 12 | 13 | Available client libraries are listed below. Libraries with an asterisk are 14 | official clients. 15 | 16 | ## Go 17 | 18 | [go-liftbridge](https://github.com/liftbridge-io/go-liftbridge)* 19 | 20 | ## Java 21 | [java-liftbridge](https://github.com/liftbridge-io/java-liftbridge)* 22 | 23 | ## Rust 24 | 25 | [liftbridge-rs](https://github.com/liftbridge-io/liftbridge-rs)* 26 | 27 | ## Node.js 28 | 29 | [node-liftbridge](https://github.com/paambaati/node-liftbridge) 30 | 31 | ## Python 32 | 33 | [python-liftbridge](https://github.com/dgzlopes/python-liftbridge) 34 | -------------------------------------------------------------------------------- /documentation/deployment.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: deployment 3 | title: Deployment 4 | --- 5 | 6 | ## Development / Testing 7 | 8 | In addition to running a native binary as described in the [quick start 9 | guide](./quick_start.md), there are three options for running a Liftbridge 10 | cluster locally for development and testing purposes: 11 | 12 | - [Docker](https://www.docker.com) - single-node Liftbridge cluster backed by a 13 | single-node NATS cluster in one container 14 | - [Docker Compose](https://docs.docker.com/compose) - three-node Liftbridge 15 | cluster backed by a single-node NATS cluster in separate containers 16 | - [Kind](https://kind.sigs.k8s.io) (Kubernetes in Docker) - three-node 17 | Liftbridge cluster backed by a three-node NATS cluster running inside a local 18 | Kubernetes cluster 19 | 20 | ### Docker 21 | 22 | There is a [container image](https://hub.docker.com/r/liftbridge/standalone-dev) 23 | available which runs an instance of Liftbridge and NATS inside a single Docker 24 | container for development and testing purposes. In effect, this runs a 25 | single-node Liftbridge cluster on your machine. 26 | 27 | Use the following Docker commands to run the container: 28 | 29 | ```shell 30 | $ docker pull liftbridge/standalone-dev 31 | $ docker run -d --name=liftbridge-main -p 4222:4222 -p 9292:9292 -p 8222:8222 -p 6222:6222 liftbridge/standalone-dev 32 | ``` 33 | 34 | This will run the container which will start both the NATS and Liftbridge 35 | servers. To check the logs to see if the container started properly, run: 36 | 37 | ```shell 38 | $ docker logs liftbridge-main 39 | ``` 40 | 41 | When running the container, you can optionally specify the mount point with: 42 | 43 | `--volume=/tmp/host/liftbridge:/tmp/liftbridge/liftbridge-default` 44 | 45 | This container exposes several ports described below. 46 | 47 | Liftbridge server exposes: 48 | - 9292 for clients connections 49 | 50 | NATS server exposes: 51 | - 4222 for clients connections 52 | - 8222 as an HTTP management port for information reporting and monitoring 53 | - 6222 is a routing port for clustering 54 | 55 | ### Docker Compose 56 | 57 | This will bring up three Liftbridge containers and one NATS node using Docker 58 | Compose: 59 | 60 | ```shell 61 | $ make compose-up 62 | ``` 63 | 64 | To tear it down, run: 65 | 66 | ```shell 67 | $ make compose-down 68 | ``` 69 | 70 | ### Kind 71 | 72 | This will deploy a three-node Liftbridge cluster backed by a three-node NATS 73 | cluster locally using Kind. For this you'll also need 74 | [Skaffold](https://skaffold.dev) and [Kustomize](https://kustomize.io) in 75 | addition to Kind. 76 | 77 | Download them: 78 | - [kind](https://github.com/kubernetes-sigs/kind/releases) version 0.5.1 or 79 | above 80 | - [kustomize](https://github.com/kubernetes-sigs/kustomize/releases) version 81 | 3.3.0 or above 82 | - [skaffold](https://skaffold.dev/docs/install/) version 0.41.0 or later 83 | 84 | To provision a local Kind cluster, run: 85 | 86 | ```shell 87 | $ make kind-up 88 | ``` 89 | 90 | Then deploy the manifests including the NATS operator and the Liftbridge 91 | cluster with: 92 | 93 | ```shell 94 | $ make kind-apply 95 | ``` 96 | 97 | To export the `KUBECONFIG` environment variable and point kubectl to the right 98 | context, run: 99 | 100 | ```shell 101 | $ make kind-export 102 | ``` 103 | 104 | After running this, you can then use kubectl as normal to interact with the 105 | local Kubernetes cluster. 106 | 107 | To tear down your local environment, run: 108 | 109 | ```shell 110 | $ make kind-down 111 | ``` 112 | -------------------------------------------------------------------------------- /documentation/embedded_nats.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: embedded-nats 3 | title: Embedded NATS 4 | --- 5 | 6 | Liftbridge relies on [NATS](https://github.com/nats-io/nats-server) for 7 | intra-cluster communication. By default, it will connect to a NATS server 8 | running on localhost. The 9 | [`nats.servers`](./configuration.md#nats-configuration-settings) setting allows 10 | configuring the NATS server(s) to connect to. However, Liftbridge can also run 11 | a NATS server embedded in the process, allowing it to run as a complete, 12 | self-contained system with no external dependencies. The below examples show 13 | how this can be done. 14 | 15 | ## Default Configuration 16 | 17 | To run an embedded NATS server with the [default configuration](https://docs.nats.io/nats-server/configuration#configuration-properties), 18 | simply enable the `embedded` setting in the [`nats`](./configuration.md#nats-configuration-settings) 19 | configuration (or equivalent `--embedded-nats` command-line flag). Set 20 | `logging.nats` to enable logging for the NATS server. 21 | 22 | ```yaml 23 | nats: 24 | embedded: true 25 | 26 | logging.nats: true 27 | ``` 28 | 29 | This will start a NATS server bound to `0.0.0.0:4222` with default settings. 30 | 31 | ## Custom Configuration 32 | 33 | To run an embedded NATS server with custom configuration, use the 34 | [`embedded.config`](./configuration.md#nats-configuration-settings) setting (or 35 | equivalent `--embedded-nats-config` command-line flag) to specify a [NATS 36 | configuration file](https://docs.nats.io/nats-server/configuration) to use. By 37 | specifying this, the `embedded` setting will be automatically enabled. Set 38 | `logging.nats` to enable logging for the NATS server. 39 | 40 | ```yaml 41 | nats: 42 | embedded.config: nats.conf 43 | 44 | logging.nats: true 45 | ``` 46 | 47 | This will start a NATS server using the specified configuration file. 48 | -------------------------------------------------------------------------------- /documentation/faq.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: faq 3 | title: Frequently Asked Questions 4 | --- 5 | 6 | ## What is Liftbridge? 7 | 8 | Liftbridge is a server that implements a durable, replicated, and scalable 9 | message log. Clients create streams which are partitioned for horizontal 10 | scalability and replicated for high availability. Streams record messages to a 11 | durable write-ahead log. 12 | 13 | Liftbridge is implemented on top of [NATS](https://nats.io), a lightweight, 14 | high-performance pub/sub messaging system. This means it can be added to an 15 | existing NATS deployment to provide message durability with no code changes. If 16 | you are not already using NATS or not familiar with it, Liftbridge can be 17 | deployed with NATS as an implementation detail. 18 | 19 | ## Why was it created? 20 | 21 | The vision for Liftbridge is to provide a "Kafka-lite" solution 22 | designed with the [Go](https://go.dev) community first in mind. Unlike Kafka, 23 | which is built on the JVM and whose canonical client library is Java (or the 24 | C-based librdkafka), Liftbridge and its canonical client, 25 | [go-liftbridge](https://github.com/liftbridge-io/go-liftbridge), are 26 | implemented in Go. The ultimate goal of Liftbridge is to provide a lightweight 27 | message-streaming solution with a focus on simplicity and usability. 28 | 29 | Liftbridge was designed to bridge the gap between sophisticated but complex 30 | log-based messaging systems like Apache Kafka and Apache Pulsar and simpler, 31 | cloud-native solutions. There is no ZooKeeper or other unwieldy dependencies, 32 | no JVM, no complicated API or configuration, and client libraries are 33 | implemented using [gRPC](https://grpc.io/). 34 | 35 | ## How does it scale? 36 | 37 | Liftbridge has several mechanisms for horizontal scaling of message consumption. 38 | Brokers can be added to the cluster and additional streams can be created which 39 | are then distributed among the cluster. In effect, this splits out message 40 | routing from storage and consumption, which allows Liftbridge to scale 41 | independently and eschew subject partitioning. 42 | 43 | Streams can also join a load-balance group, which effectively load balances a 44 | NATS subject among the streams in the group without affecting delivery to 45 | other streams. 46 | 47 | Additionally, streams can be partitioned, allowing messages to be divided up 48 | among the brokers in a cluster. In fact, all streams are partitioned with the 49 | default case being a single partition. 50 | 51 | [Consumer groups](./consumer_groups.md) allow for load balancing of stream 52 | consumption. In combination with stream partitioning, this allows for increased 53 | parallelism and higher throughput. 54 | 55 | ## What about HA? 56 | 57 | High availability is achieved by replicating the streams. When a stream is 58 | created, the client specifies a `replicationFactor`, which determines the 59 | number of brokers to replicate the stream's partitions. Each stream partition 60 | has a leader who is responsible for handling reads and writes. Followers then 61 | replicate the log from the leader. If the leader fails, one of the followers 62 | will step up to replace it. The replication protocol closely resembles that of 63 | Kafka, so there is much more nuance to avoid data consistency problems. See the 64 | [replication protocol documentation](replication_protocol.md) 65 | for more details. 66 | -------------------------------------------------------------------------------- /documentation/feature_comparison.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: feature-comparison 3 | title: Feature Comparison 4 | --- 5 | 6 | Liftbridge shares some similarities with other stream-oriented pub/sub 7 | messaging systems. The below feature-comparison matrix shows how it compares to 8 | a few of these systems. 9 | 10 | > **Architect's Note** 11 | > 12 | > These are highly nuanced pieces of infrastructure, so a straight comparison 13 | > of features is often of minimal value when evaluating solutions. This matrix 14 | > is primarily meant to serve as a reference point for those attempting to 15 | > develop a mental model of Liftbridge. 16 | 17 | | | Liftbridge | NATS Streaming | Apache Kafka | Apache Pulsar | 18 | |:----|:----|:----|:----|:----| 19 | | Runtime | self-contained binary | self-contained binary | JVM | JVM | 20 | | At-least-once delivery | ✓ | ✓ | ✓ | ✓ | 21 | | Transactions | | | ✓ | | 22 | | Partitioning | ✓ | | ✓ | ✓ | 23 | | Linear Horizontal Scalability | ✓ | | ✓ | ✓ | 24 | | Clustering/Replication | ✓ | ✓ | ✓ | ✓ | 25 | | External Coordination | | | Apache ZooKeeper | Apache ZooKeeper | 26 | | Message Replay | ✓ | ✓ | ✓ | ✓ | 27 | | Message Queueing | | ✓ | | ✓ | 28 | | Wildcard Topic Matching | ✓ | | | | 29 | | Topic Pausing | ✓ | | | | 30 | | Rate Limiting | | opt-in rate matching | quotas | publish rate limiting | 31 | | Log Retention | ✓ | ✓ | ✓ | ✓ | 32 | | Log Compaction | ✓ | | ✓ | ✓ | 33 | | Message Headers | ✓ | | ✓ | ✓ | 34 | | Multiple Consumers | consumer groups | queue subscriptions | consumer groups | subscriptions | 35 | | Consumer Position Tracking | consumer groups & cursors | durable subscriptions | consumer groups | cursors | 36 | | Event-enabled | activity stream | | | | 37 | | Multitenancy | namespaced clusters | namespaced clusters | topic-level ACLs | multitenant shared cluster | 38 | | Authentication | TLS | User authentication, TLS | TLS, Kerberos, SASL | TLS, Kerberos, JWT, Athenz | 39 | | Authorization | | | ACLs | ACLs | 40 | | Storage | filesystem | memory, filesystem, SQL| filesystem | filesystem (Apache BookKeeper), tiered storage (Amazon S3, Google Cloud Storage) | 41 | | Encryption of data-at-rest | ✓ | ✓ | ✓ | ✓ | 42 | -------------------------------------------------------------------------------- /documentation/ha_and_consistency_configuration.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: ha-and-consistency-configuration 3 | title: Configuring for High Availability and Consistency 4 | --- 5 | 6 | Liftbridge provides several parameters, outlined below, for controlling high 7 | availability and data consistency (guaranteed delivery). It should be pointed 8 | out that these characteristics are often at odds with each other. 9 | 10 | ## Replication Factor 11 | 12 | The replication factor of a stream controls the number of nodes the stream's 13 | partitions should be replicated to for redundancy. This value is set by the 14 | client when creating a stream. The default replication factor is 1. 15 | 16 | For high availability and durability of data, it is recommended to use a 17 | replication factor of at least 3. 18 | 19 | ```go 20 | // Create a stream with a replication factor of 3. 21 | if err := client.CreateStream(context.Background(), "foo", "foo-stream", lift.ReplicationFactor(3)); err != nil { 22 | if err != lift.ErrStreamExists { 23 | panic(err) 24 | } 25 | } 26 | ``` 27 | 28 | ## Ack Policy 29 | 30 | When publishing a message to Liftbridge, you can choose how many replicas must 31 | store a message before it is acked. The Ack Policy controls this behavior and 32 | is configured on each message. Valid values are: 33 | 34 | - `AckPolicy_LEADER`: the ack is sent once the leader has written the message 35 | to its log 36 | - `AckPolicy_ALL`: the ack is sent after the ISR replicas have written the 37 | message to their logs (i.e. the message is committed). 38 | - `AckPolicy_NONE`: no ack is sent 39 | 40 | The default value is `AckPolicy_LEADER`. `AckPolicy_ALL` provides the highest 41 | consistency guarantee at the expense of slower writes. 42 | 43 | ## Minimum In-Sync Replica Set 44 | 45 | You can set the minimum number of in-sync replicas (ISR) that must acknowledge 46 | a stream write before it can be committed. If the ISR drops below this size, 47 | messages cannot be committed. This is controlled with the `min.insync.replicas` 48 | setting in [`clustering`](./configuration.md#clustering-configuration-settings) 49 | configuration. 50 | 51 | By default, this value is 1, favoring availability over consistency. This 52 | setting can be used in conjunction with the replication factor and ack policy 53 | to enforce a quorum on writes which provides greater durability guarantees. For 54 | example, for a stream with a replication factor of 3, `AckPolicy_ALL` and a 55 | `min.insync.replicas` value of 2 will guarantee the message is written to at 56 | least 2 replicas. 57 | 58 | ```yaml 59 | clustering: 60 | min.insync.replicas: 2 61 | ``` 62 | -------------------------------------------------------------------------------- /documentation/overview.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: overview 3 | title: Liftbridge Overview 4 | --- 5 | 6 | Liftbridge is a server that implements a durable, replicated, and scalable 7 | message log. The vision for Liftbridge is to provide a "Kafka-lite" solution 8 | designed with the [Go](https://go.dev) community first in mind. Unlike Kafka, 9 | which is built on the JVM and whose canonical client library is Java (or the 10 | C-based librdkafka), Liftbridge and its canonical client, 11 | [go-liftbridge](https://github.com/liftbridge-io/go-liftbridge), are 12 | implemented in Go. The ultimate goal of Liftbridge is to provide a lightweight 13 | message-streaming solution with a focus on simplicity and usability. 14 | 15 | To this end, Liftbridge was designed to bridge the gap between sophisticated 16 | but complex log-based messaging systems like Apache Kafka and Apache Pulsar and 17 | simpler, cloud-native solutions. There is no ZooKeeper or other unwieldy 18 | dependencies, no JVM, no complicated API or configuration, and client libraries 19 | are implemented using [gRPC](https://grpc.io). 20 | 21 | Liftbridge is implemented on top of [NATS](https://nats.io), a lightweight, 22 | high-performance pub/sub messaging system. This means it can be added to an 23 | existing NATS deployment to provide message durability with no code changes. If 24 | you are not already using NATS or not familiar with it, Liftbridge can be 25 | deployed with NATS as an implementation detail. 26 | 27 | Streams in Liftbridge are partitioned for horizontal scalability. By default, 28 | streams have a single partition. Each partition has a leader and is replicated 29 | to some set of followers for fault-tolerance. [Consumer groups](./consumer_groups.md) 30 | provide means for implementing distributed, scalable, and fault-tolerant stream 31 | processing with consumer balancing, consumer position tracking, automatic 32 | consumer failover. 33 | -------------------------------------------------------------------------------- /documentation/pausing_streams.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: pausing-streams 3 | title: Pausing Streams 4 | --- 5 | 6 | Liftbridge streams can be *paused* to conserve system resources such as CPU, 7 | memory, and file descriptors. While pausing is performed on streams, the 8 | granularity is specified at the partition level. We can pause all or a subset 9 | of a stream's partitions. A partition is resumed when it is published to via 10 | the Liftbridge `Publish` or `PublishAsync` gRPC endpoints or if the stream was 11 | paused with `ResumeAll` enabled and another partition in the stream was 12 | published to. 13 | 14 | > **Use Case Note** 15 | > 16 | > Stream pausing can be useful for cases that involve a significant number of 17 | > streams with only a small fraction being active at any given point in time, 18 | > i.e. "sparse" streams. With the [activity stream](./activity.md), consumers 19 | > can dynamically spin down when partitions are paused and spin back up once 20 | > they are resumed. 21 | 22 | Pause functionality is exposed through the Liftbridge [gRPC 23 | API](https://github.com/liftbridge-io/liftbridge-api/blob/master/api.proto). 24 | The `PauseStream` endpoint takes a `PauseStreamRequest` which specifies the 25 | stream and set of partitions to pause. If no partitions are specified, the 26 | operation will pause _all_ of the stream's partitions. Additionally, the 27 | request includes a `ResumeAll` flag which indicates if all partitions should be 28 | resumed when one is published to or only the partition that was published to. 29 | 30 | When a partition is paused, the server will step down as leader or follower, 31 | unsubscribe from the NATS subject, and close the commit log. This means 32 | replication will stop, messages will not be received on the NATS subject, and 33 | any file handles associated with the partition will be closed. 34 | 35 | Pausing is maintained across server restarts. 36 | 37 | ## Auto Pausing 38 | 39 | In addition to the pause API, streams can be configured to automatically pause 40 | partitions when they go idle, meaning no messages are received on the partition 41 | within a specified period of time. This is configured globally using the 42 | `streams.auto.pause.time` setting which applies the pause timeout for all 43 | streams. By default, this is disabled. This can also be overridden on 44 | individual streams when they are created. 45 | 46 | The `auto.pause.disable.if.subscribers` setting controls if the automatic 47 | partition pausing should be disabled if there is any subscribers to the 48 | partition. This is disabled by default. 49 | 50 | Only the idle partitions within a stream are paused. These partitions are 51 | resumed when published to via the Liftbridge API. 52 | -------------------------------------------------------------------------------- /documentation/scalability_configuration.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: scalability-configuration 3 | title: Configuring for Scalability 4 | --- 5 | 6 | Liftbridge is designed to be clustered and horizontally scalable, meaning nodes 7 | can be added to handle additional load. There are two dimensions to this 8 | scalability, the control plane and the data plane. The _control plane_ refers 9 | to the [metadata controller](./concepts.md#controller) which performs 10 | operations and coordination responsibilies for the cluster. This is implemented 11 | using the [Raft consensus algorithm](https://raft.github.io). The _data plane_ 12 | refers to the processes around actual messages and 13 | [streams](./concepts.md#streams-and-partitions). It is important to understand 14 | how the scalability of these two concerns interrelate. 15 | 16 | ## Scaling the Data Plane 17 | 18 | There are a few different ways streams can be scaled in Liftbridge. These 19 | different approaches are discussed [here](./concepts.md#scalability). To 20 | summarize, both stream data and consumption can be scaled horizontally by 21 | adding additional nodes to the cluster along with additional streams or 22 | [stream partitioning](./concepts.md#streams-and-partitions). However, adding 23 | nodes to the cluster has implications with the metadata Raft cluster used by 24 | the control plane. This is discussed below. 25 | 26 | Stream partitioning provides increased parallelism which allows for greater 27 | throughput of messages. [Consumer groups](./consumer_groups.md) provide a means 28 | for coordinating and balancing the consumption of partitions across a set of 29 | consumers. 30 | 31 | ## Scaling the Control Plane 32 | 33 | By default, new servers that are added to a Liftbridge cluster participate in 34 | the Raft consensus group used by the control plane. These are referred to as 35 | _voters_. This means they are involved in elections for the metadata leader and 36 | committing entries to the Raft log. However, because Raft requires a minimum of 37 | `N/2+1` nodes to perform operations, this can severely limit the scalability of 38 | the control plane. For example, in a 100-node cluster, 51 nodes have to respond 39 | to commit an operation. Additionally, the Raft protocol requires exchanging 40 | `N^2` messages to arrive at consensus for a given operation. 41 | 42 | To address this, Liftbridge has a setting to limit the number of voters who 43 | participate in the Raft group. The [`clustering.raft.max.quorum.size`](./configuration.md#clustering-configuration-settings) 44 | setting restricts the number of servers who participate in the Raft quorum. Any 45 | servers added to the cluster beyond this number participate as _non-voters_. 46 | Non-voter servers operate as normal but are not involved in the Raft election 47 | or commitment processes. By default, `clustering.raft.max.quorum.size` is set 48 | to `0`, which means there is no limit. Limiting this number allows the control 49 | plane to better scale as nodes are added. This is typically not a concern for 50 | smaller clusters, such as single-digit or low-double-digit clusters but can be 51 | an issue for clusters beyond these sizes. This configuration should be set to 52 | the same value on all servers in the cluster. 53 | 54 | The configuration example below shows how to limit the Raft quorum to five 55 | servers. 56 | 57 | ```yaml 58 | clustering: 59 | raft.max.quorum.size: 5 60 | ``` 61 | 62 | Guidance on cluster and quorum size is use-case specific, but it is recommended 63 | to specify an odd number for `clustering.raft.max.quorum.size` (or to run an 64 | odd number of servers if not limiting quorum size), e.g. 3 or 5, depending on 65 | scaling needs. Ideally, cluster members are run in different availability zones 66 | or racks for improved fault-tolerance. 67 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/liftbridge-io/liftbridge 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/Workiva/go-datastructures v1.0.52 7 | github.com/casbin/casbin/v2 v2.40.6 8 | github.com/dustin/go-humanize v1.0.0 9 | github.com/golang/protobuf v1.5.3 10 | github.com/google/tink/go v1.5.0 11 | github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 12 | github.com/hako/durafmt v0.0.0-20200605151348-3a43fc422dd9 13 | github.com/hashicorp/golang-lru v0.5.4 14 | github.com/hashicorp/raft v1.1.2 15 | github.com/hashicorp/raft-boltdb/v2 v2.2.0 16 | github.com/liftbridge-io/go-liftbridge/v2 v2.2.1-0.20220311002120-3d391a19ff0e 17 | github.com/liftbridge-io/liftbridge-api v1.9.0 18 | github.com/liftbridge-io/nats-on-a-log v0.0.0-20200818183806-bb17516cf3a3 19 | github.com/natefinch/atomic v1.0.1 20 | github.com/nats-io/nats-server/v2 v2.9.23 21 | github.com/nats-io/nats.go v1.28.0 22 | github.com/nats-io/nuid v1.0.1 23 | github.com/pkg/errors v0.9.1 24 | github.com/sirupsen/logrus v1.6.0 25 | github.com/spf13/viper v1.7.0 26 | github.com/stretchr/testify v1.7.1 27 | github.com/tysonmote/gommap v0.0.2-0.20220314171410-078b7adc9d18 28 | github.com/urfave/cli v1.22.4 29 | google.golang.org/grpc v1.56.3 30 | ) 31 | 32 | require ( 33 | github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect 34 | github.com/armon/go-metrics v0.3.4 // indirect 35 | github.com/boltdb/bolt v1.3.1 // indirect 36 | github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect 37 | github.com/davecgh/go-spew v1.1.1 // indirect 38 | github.com/fatih/color v1.9.0 // indirect 39 | github.com/fsnotify/fsnotify v1.4.9 // indirect 40 | github.com/hashicorp/go-hclog v0.14.1 // indirect 41 | github.com/hashicorp/go-immutable-radix v1.2.0 // indirect 42 | github.com/hashicorp/go-msgpack v1.1.5 // indirect 43 | github.com/hashicorp/hcl v1.0.0 // indirect 44 | github.com/klauspost/compress v1.16.7 // indirect 45 | github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect 46 | github.com/magiconair/properties v1.8.1 // indirect 47 | github.com/mattn/go-colorable v0.1.7 // indirect 48 | github.com/mattn/go-isatty v0.0.12 // indirect 49 | github.com/minio/highwayhash v1.0.2 // indirect 50 | github.com/mitchellh/mapstructure v1.3.2 // indirect 51 | github.com/nats-io/jwt/v2 v2.5.0 // indirect 52 | github.com/nats-io/nkeys v0.4.6 // indirect 53 | github.com/pelletier/go-toml v1.8.0 // indirect 54 | github.com/pmezard/go-difflib v1.0.0 // indirect 55 | github.com/russross/blackfriday/v2 v2.0.1 // indirect 56 | github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b // indirect 57 | github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect 58 | github.com/spf13/afero v1.3.1 // indirect 59 | github.com/spf13/cast v1.3.1 // indirect 60 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 61 | github.com/spf13/pflag v1.0.5 // indirect 62 | github.com/subosito/gotenv v1.2.0 // indirect 63 | go.etcd.io/bbolt v1.3.5 // indirect 64 | golang.org/x/crypto v0.17.0 // indirect 65 | golang.org/x/net v0.17.0 // indirect 66 | golang.org/x/sys v0.15.0 // indirect 67 | golang.org/x/text v0.14.0 // indirect 68 | golang.org/x/time v0.3.0 // indirect 69 | google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect 70 | google.golang.org/protobuf v1.30.0 // indirect 71 | gopkg.in/ini.v1 v1.57.0 // indirect 72 | gopkg.in/yaml.v2 v2.3.0 // indirect 73 | gopkg.in/yaml.v3 v3.0.1 // indirect 74 | ) 75 | -------------------------------------------------------------------------------- /k8s/Dockerfile.k8s: -------------------------------------------------------------------------------- 1 | FROM golang:1.17-alpine as build-base 2 | RUN apk update && apk upgrade && \ 3 | apk add --no-cache bash git openssh make wget 4 | WORKDIR /workspace 5 | ENV GO111MODULE on 6 | 7 | # Copy the Go Modules manifests 8 | COPY go.mod go.mod 9 | COPY go.sum go.sum 10 | # cache deps before building and copying source so that we don't need to re-download as much 11 | # and so that source changes don't invalidate our downloaded layer 12 | RUN go mod download 13 | 14 | # Copy the go source 15 | COPY main.go main.go 16 | COPY server/ server/ 17 | 18 | # Build 19 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=readonly -o liftbridge 20 | 21 | RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \ 22 | wget -qO /bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ 23 | chmod +x /bin/grpc_health_probe 24 | 25 | FROM alpine:latest 26 | RUN apk update && apk add --no-cache bash 27 | RUN addgroup -g 1001 -S liftbridge && adduser -u 1001 -S liftbridge -G liftbridge 28 | COPY --chown=liftbridge:liftbridge --from=build-base /workspace/liftbridge /usr/local/bin/liftbridge 29 | COPY --chown=liftbridge:liftbridge --from=build-base /bin/grpc_health_probe /bin/grpc_health_probe 30 | COPY k8s/entrypoint.sh /entrypoint.sh 31 | ENTRYPOINT ["/entrypoint.sh"] 32 | USER liftbridge 33 | -------------------------------------------------------------------------------- /k8s/base/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | data: 3 | liftbridge.yaml: | 4 | listen: 0.0.0.0:9292 5 | logging.level: debug 6 | 7 | nats.servers: 8 | - "nats://nats.liftbridge.svc:4222" 9 | 10 | clustering.min.insync.replicas: 1 11 | kind: ConfigMap 12 | metadata: 13 | labels: 14 | app: liftbridge 15 | component: config 16 | environment: dev 17 | k8s-app: liftbridge 18 | name: config 19 | -------------------------------------------------------------------------------- /k8s/base/kustomization.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | namespace: default 3 | 4 | resources: 5 | - svc.yaml 6 | - statefulset.yaml 7 | - config.yaml 8 | -------------------------------------------------------------------------------- /k8s/base/statefulset.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: liftbridge 6 | spec: 7 | serviceName: "liftbridge-headless" 8 | podManagementPolicy: "Parallel" 9 | replicas: 3 10 | selector: 11 | matchLabels: 12 | app: liftbridge 13 | template: 14 | metadata: 15 | labels: 16 | app: liftbridge 17 | spec: 18 | containers: 19 | - name: liftbridge 20 | image: liftbridge 21 | ports: 22 | - name: grpc 23 | containerPort: 9292 24 | readinessProbe: 25 | exec: 26 | command: ["/bin/grpc_health_probe", "-service=proto.API", "-addr=:9292"] 27 | initialDelaySeconds: 5 28 | volumeMounts: 29 | - name: liftbridge-data 30 | mountPath: /data 31 | - mountPath: /etc/liftbridge.yaml 32 | name: liftbridge-config 33 | subPath: liftbridge.yaml 34 | volumes: 35 | - configMap: 36 | name: config 37 | name: liftbridge-config 38 | -------------------------------------------------------------------------------- /k8s/base/svc.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: liftbridge-headless 6 | spec: 7 | selector: 8 | app: liftbridge 9 | clusterIP: "None" 10 | ports: 11 | - name: grpc 12 | protocol: TCP 13 | port: 9292 14 | targetPort: grpc 15 | --- 16 | apiVersion: v1 17 | kind: Service 18 | metadata: 19 | name: liftbridge 20 | spec: 21 | selector: 22 | app: liftbridge 23 | ports: 24 | - name: grpc 25 | protocol: TCP 26 | port: 9292 27 | targetPort: grpc 28 | -------------------------------------------------------------------------------- /k8s/dev/kind.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: Cluster 3 | apiVersion: kind.sigs.k8s.io/v1alpha3 4 | nodes: 5 | - role: control-plane 6 | extraPortMappings: 7 | - containerPort: 32767 8 | hostPort: 9292 9 | listenAddress: "0.0.0.0" 10 | - containerPort: 32767 11 | hostPort: 9293 12 | listenAddress: "0.0.0.0" 13 | - containerPort: 32767 14 | hostPort: 9294 15 | listenAddress: "0.0.0.0" 16 | - containerPort: 32766 17 | hostPort: 3000 18 | listenAddress: "0.0.0.0" 19 | - containerPort: 32765 20 | hostPort: 4222 21 | listenAddress: "0.0.0.0" 22 | - role: worker 23 | - role: worker 24 | -------------------------------------------------------------------------------- /k8s/dev/kustomization.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | namespace: liftbridge 3 | 4 | bases: 5 | - ../base 6 | 7 | resources: 8 | - namespace.yaml 9 | - nats.yaml 10 | - natsboard.yaml 11 | - svc.yaml 12 | - storageclass.yaml 13 | 14 | patches: 15 | - patches.yaml 16 | -------------------------------------------------------------------------------- /k8s/dev/namespace.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: liftbridge 6 | labels: 7 | name: liftbridge 8 | environment: dev 9 | -------------------------------------------------------------------------------- /k8s/dev/nats.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: nats.io/v1alpha2 3 | kind: NatsCluster 4 | metadata: 5 | labels: 6 | app: liftbridge 7 | component: nats 8 | environment: dev 9 | k8s-app: liftbridge 10 | name: nats 11 | spec: 12 | natsConfig: 13 | writeDeadline: 5s 14 | pod: 15 | annotations: 16 | sidecar.istio.io/inject: "false" 17 | enableConfigReload: true 18 | enableMetrics: true 19 | metricsImage: synadia/prometheus-nats-exporter 20 | metricsImageTag: 0.5.0 21 | reloaderImage: connecteverything/nats-server-config-reloader 22 | reloaderImagePullPolicy: IfNotPresent 23 | reloaderImageTag: 0.6.0 24 | size: 3 25 | version: 2.1.0 26 | --- 27 | apiVersion: v1 28 | kind: Service 29 | metadata: 30 | name: nats-external 31 | spec: 32 | selector: 33 | app: nats 34 | nats_cluster: nats 35 | type: NodePort 36 | ports: 37 | - name: nats 38 | protocol: TCP 39 | port: 4222 40 | nodePort: 32765 41 | targetPort: 4222 42 | -------------------------------------------------------------------------------- /k8s/dev/natsboard.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: natsboard 6 | labels: 7 | app: natsboard 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: natsboard 13 | template: 14 | metadata: 15 | labels: 16 | app: natsboard 17 | spec: 18 | containers: 19 | - name: natsboard 20 | image: natsboard 21 | command: ['/usr/bin/natsboard'] 22 | env: 23 | - name: "WAIT_FOR_HOST" 24 | value: nats-mgmt.default.svc 25 | - name: "WAIT_FOR_PORT" 26 | value: "8222" 27 | - name: "NATS_MON_URL" 28 | value: "http://nats-mgmt.default.svc:8222" 29 | ports: 30 | - name: http 31 | containerPort: 3000 32 | --- 33 | apiVersion: v1 34 | kind: Service 35 | metadata: 36 | name: natsboard-external 37 | spec: 38 | selector: 39 | app: natsboard 40 | type: NodePort 41 | ports: 42 | - name: http 43 | protocol: TCP 44 | port: 3000 45 | nodePort: 32766 46 | targetPort: http 47 | -------------------------------------------------------------------------------- /k8s/dev/patches.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: StatefulSet 4 | metadata: 5 | name: liftbridge 6 | spec: 7 | volumeClaimTemplates: 8 | - metadata: 9 | name: liftbridge-data 10 | spec: 11 | accessModes: ["ReadWriteOnce"] 12 | storageClassName: liftbridge 13 | resources: 14 | requests: 15 | storage: 5Gi 16 | -------------------------------------------------------------------------------- /k8s/dev/storageclass.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.k8s.io/v1 2 | kind: StorageClass 3 | metadata: 4 | name: liftbridge 5 | provisioner: kubernetes.io/host-path 6 | reclaimPolicy: Retain 7 | -------------------------------------------------------------------------------- /k8s/dev/svc.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: liftbridge-external 6 | spec: 7 | selector: 8 | app: liftbridge 9 | type: NodePort 10 | ports: 11 | - name: grpc 12 | protocol: TCP 13 | port: 9292 14 | nodePort: 32767 15 | targetPort: grpc 16 | -------------------------------------------------------------------------------- /k8s/empty/kustomization.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liftbridge-io/liftbridge/9d7e049a824aaeb014a8650bc1fe26817bbd8d9d/k8s/empty/kustomization.yaml -------------------------------------------------------------------------------- /k8s/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CLUSTER_NAME=${CLUSTER_NAME:=cluster} 4 | 5 | if [[ "$HOSTNAME" =~ -0$ ]] && [[ "$SKIP_SEED" != "true" ]]; 6 | then 7 | echo "Running in $HOSTNAME as a bootstrap seed..." 8 | LIFTBRIDGE_HOST=$(hostname -i) liftbridge --data-dir=/data --config /etc/liftbridge.yaml --raft-bootstrap-seed --id="${CLUSTER_NAME}-$HOSTNAME" 9 | else 10 | echo "Running in $HOSTNAME..." 11 | LIFTBRIDGE_HOST=$(hostname -i) liftbridge --data-dir=/data --config /etc/liftbridge.yaml --id="${CLUSTER_NAME}-$HOSTNAME" 12 | fi 13 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestNormalizeNatsServers(t *testing.T) { 9 | testCases := []struct { 10 | testCase string 11 | natsServers []string 12 | want []string 13 | }{ 14 | { 15 | "No servers (should not happen but test it anyway)", 16 | nil, 17 | nil, 18 | }, 19 | { 20 | "Single server no spaces", 21 | []string{"nats://localhost:9876"}, 22 | []string{"nats://localhost:9876"}, 23 | }, 24 | { 25 | "Single server with spaces", 26 | []string{" nats://localhost:9876 "}, 27 | []string{"nats://localhost:9876"}, 28 | }, 29 | { 30 | "2 instances of --nats-servers", 31 | []string{"nats://localhost:9876", "nats://localhost:6789"}, 32 | []string{"nats://localhost:9876", "nats://localhost:6789"}, 33 | }, 34 | { 35 | "2 instances of --nats-servers with leading+trailing spaces", 36 | []string{" nats://localhost:9876 ", " nats://localhost:6789 "}, 37 | []string{"nats://localhost:9876", "nats://localhost:6789"}, 38 | }, 39 | { 40 | "Single instance of --nats-servers with multiple servers, no spaces", 41 | []string{"nats://localhost:3434,nats://localhost:2121"}, 42 | []string{"nats://localhost:3434", "nats://localhost:2121"}, 43 | }, 44 | { 45 | "Single instance of --nats-servers with multiple servers and spaces", 46 | []string{" nats://localhost:1111, nats://localhost:2222 ,nats://localhost:3333"}, 47 | []string{"nats://localhost:1111", "nats://localhost:2222", "nats://localhost:3333"}, 48 | }, 49 | { 50 | "Multiple instances of --nats-servers with multiple servers and spaces", 51 | []string{"nats://localhost:9999", " nats://localhost:8888 , nats://localhost:7777 ,nats://localhost:6666 ", " nats://localhost:5555 "}, 52 | []string{"nats://localhost:9999", "nats://localhost:8888", "nats://localhost:7777", "nats://localhost:6666", "nats://localhost:5555"}, 53 | }, 54 | } 55 | for _, tc := range testCases { 56 | t.Run(tc.testCase, func(t *testing.T) { 57 | natsServers, err := normalizeNatsServers(tc.natsServers) 58 | if err != nil { 59 | t.Fatalf("returned error: %#v", tc) 60 | } 61 | if !reflect.DeepEqual(natsServers, tc.want) { 62 | t.Fatalf("\n wanted: %#v\n got: %#v", tc.want, natsServers) 63 | } 64 | }) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /server/authz.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | 6 | grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" 7 | grpc "google.golang.org/grpc" 8 | credentials "google.golang.org/grpc/credentials" 9 | peer "google.golang.org/grpc/peer" 10 | ) 11 | 12 | // addUserContext parses client ID from context and set client ID in context 13 | func addUserContext(ctx context.Context) context.Context { 14 | p, ok := peer.FromContext(ctx) 15 | 16 | if !ok || p.AuthInfo == nil { 17 | return ctx 18 | } 19 | 20 | tlsInfo := p.AuthInfo.(credentials.TLSInfo) 21 | 22 | if len(tlsInfo.State.VerifiedChains) == 0 || len(tlsInfo.State.VerifiedChains[0]) == 0 { 23 | return ctx 24 | } 25 | 26 | clientName := tlsInfo.State.VerifiedChains[0][0].Subject.CommonName 27 | return context.WithValue(ctx, "clientID", clientName) 28 | 29 | } 30 | 31 | // AuthzUnaryInterceptor gets user from TLS-authenticated request and add user to ctx 32 | func AuthzUnaryInterceptor( 33 | ctx context.Context, 34 | req interface{}, 35 | info *grpc.UnaryServerInfo, 36 | handler grpc.UnaryHandler, 37 | ) (interface{}, error) { 38 | return handler(addUserContext(ctx), req) 39 | } 40 | 41 | // AuthzStreamInterceptor gets user from TLS-authenticated stream request and add user to ctx 42 | func AuthzStreamInterceptor(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { 43 | 44 | newStream := grpc_middleware.WrapServerStream(ss) 45 | newStream.WrappedContext = addUserContext(ss.Context()) 46 | return handler(srv, newStream) 47 | } 48 | -------------------------------------------------------------------------------- /server/commitlog/index_test.go: -------------------------------------------------------------------------------- 1 | package commitlog 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestIndexInitialSize(t *testing.T) { 11 | dir := tempDir(t) 12 | defer os.RemoveAll(dir) 13 | 14 | // Create a new index. 15 | idx, err := newIndex(options{path: dir + "test.idx"}) 16 | require.NoError(t, err) 17 | entry, err := idx.InitializePosition() 18 | require.NoError(t, err) 19 | require.Nil(t, entry) 20 | 21 | // Verify the index is preallocated and the position is 0. 22 | require.Equal(t, int64(10*1024*1024), idx.size) 23 | require.Equal(t, int64(0), idx.position) 24 | 25 | // Verify the recorded size matches the file size. 26 | finfo, err := idx.file.Stat() 27 | require.NoError(t, err) 28 | require.Equal(t, idx.size, finfo.Size()) 29 | } 30 | 31 | func TestIndexExistingSize(t *testing.T) { 32 | dir := tempDir(t) 33 | defer os.RemoveAll(dir) 34 | 35 | // Create a new index. 36 | idx, err := newIndex(options{path: dir + "test.idx"}) 37 | require.NoError(t, err) 38 | e, err := idx.InitializePosition() 39 | require.NoError(t, err) 40 | require.Nil(t, e) 41 | 42 | // Write some entries. 43 | err = idx.writeEntries([]*entry{ 44 | {}, {}, {}, {}, 45 | }) 46 | require.NoError(t, err) 47 | 48 | // Close the index so that it will be truncated. 49 | err = idx.Close() 50 | require.NoError(t, err) 51 | 52 | // Reopen the index and verify the size and position are correct. 53 | idx, err = newIndex(options{path: dir + "test.idx"}) 54 | require.NoError(t, err) 55 | require.Equal(t, idx.size, int64(entryWidth*4)) 56 | require.Equal(t, idx.size, idx.position) 57 | } 58 | 59 | func TestIndexExpansion(t *testing.T) { 60 | dir := tempDir(t) 61 | defer os.RemoveAll(dir) 62 | 63 | // Create an index with enough room for a single entry. 64 | idx, err := newIndex(options{path: dir + "test.idx", bytes: entryWidth}) 65 | require.NoError(t, err) 66 | e, err := idx.InitializePosition() 67 | require.NoError(t, err) 68 | require.Nil(t, e) 69 | 70 | // Write two entries. 71 | writeEntry := entry{Offset: 123, Timestamp: 456, Position: 789, Size: 987} 72 | err = idx.writeEntries([]*entry{{}, &writeEntry}) 73 | require.NoError(t, err) 74 | 75 | // Check the second entry can be retrieved. 76 | var readEntry entry 77 | err = idx.ReadEntryAtLogOffset(&readEntry, 1) 78 | require.NoError(t, err) 79 | require.Equal(t, writeEntry, readEntry) 80 | } 81 | -------------------------------------------------------------------------------- /server/commitlog/util.go: -------------------------------------------------------------------------------- 1 | package commitlog 2 | 3 | import ( 4 | "os" 5 | "sort" 6 | ) 7 | 8 | // findSegment returns the first segment whose next assignable offset is 9 | // greater than the given offset. Returns nil and the index where the segment 10 | // would be if there is no such segment. 11 | func findSegment(segments []*segment, offset int64) (*segment, int) { 12 | n := len(segments) 13 | idx := sort.Search(n, func(i int) bool { 14 | return segments[i].NextOffset() > offset 15 | }) 16 | if idx == n { 17 | return nil, idx 18 | } 19 | return segments[idx], idx 20 | } 21 | 22 | // findSegmentContains returns the first segment whose next assignable offset 23 | // is greater than the given offset and a bool indicating if the returned 24 | // segment contains the offset, meaning the offset is between the segment's 25 | // base offset and next assignable offset. Note that because the segment could 26 | // be compacted, "contains" does not guarantee the offset is actually present, 27 | // only that it's within the bounds. 28 | func findSegmentContains(segments []*segment, offset int64) (*segment, bool) { 29 | seg, _ := findSegment(segments, offset) 30 | if seg == nil { 31 | return nil, false 32 | } 33 | return seg, seg.BaseOffset <= offset 34 | } 35 | 36 | // findSegmentIndexByTimestamp returns the index of the first segment whose 37 | // base timestamp is greater than the given timestamp. Returns the index where 38 | // the segment would be if there is no segment whose base timestamp is greater, 39 | // i.e. the length of the slice. 40 | func findSegmentIndexByTimestamp(segments []*segment, timestamp int64) (int, error) { 41 | var ( 42 | n = len(segments) 43 | err error 44 | ) 45 | idx := sort.Search(n, func(i int) bool { 46 | // Read the first entry in the segment to determine the base timestamp. 47 | var entry entry 48 | if e := segments[i].Index.ReadEntryAtLogOffset(&entry, 0); e != nil { 49 | err = e 50 | return true 51 | } 52 | return entry.Timestamp > timestamp 53 | }) 54 | return idx, err 55 | } 56 | 57 | // findSegmentByBaseOffset returns the first segment whose base offset is 58 | // greater than or equal to the given offset. Returns nil if there is no such 59 | // segment. 60 | func findSegmentByBaseOffset(segments []*segment, offset int64) *segment { 61 | n := len(segments) 62 | idx := sort.Search(n, func(i int) bool { 63 | return segments[i].BaseOffset >= offset 64 | }) 65 | if idx == n { 66 | return nil 67 | } 68 | return segments[idx] 69 | } 70 | 71 | func roundDown(total, factor int64) int64 { 72 | return factor * (total / factor) 73 | } 74 | 75 | func exists(path string) bool { 76 | _, err := os.Stat(path) 77 | return err == nil 78 | } 79 | -------------------------------------------------------------------------------- /server/commitlog/util_test.go: -------------------------------------------------------------------------------- 1 | package commitlog 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestFindSegment(t *testing.T) { 10 | segments := []*segment{ 11 | {BaseOffset: 0, lastOffset: 9}, 12 | {BaseOffset: 10, lastOffset: 19}, 13 | {BaseOffset: 20, lastOffset: 29}, 14 | {BaseOffset: 30, lastOffset: 39}, 15 | {BaseOffset: 40, lastOffset: 49}, 16 | } 17 | seg, idx := findSegment(segments, 0) 18 | require.Equal(t, 0, idx) 19 | require.Equal(t, segments[0], seg) 20 | 21 | seg, idx = findSegment(segments, 1) 22 | require.Equal(t, 0, idx) 23 | require.Equal(t, segments[0], seg) 24 | 25 | seg, idx = findSegment(segments, 15) 26 | require.Equal(t, 1, idx) 27 | require.Equal(t, segments[1], seg) 28 | 29 | seg, idx = findSegment(segments, 42) 30 | require.Equal(t, 4, idx) 31 | require.Equal(t, segments[4], seg) 32 | 33 | seg, idx = findSegment(segments, 60) 34 | require.Equal(t, 5, idx) 35 | require.Nil(t, seg) 36 | } 37 | 38 | func TestFindSegmentByBaseOffset(t *testing.T) { 39 | segments := []*segment{ 40 | {BaseOffset: 0}, 41 | {BaseOffset: 10}, 42 | {BaseOffset: 20}, 43 | {BaseOffset: 30}, 44 | {BaseOffset: 40}, 45 | } 46 | require.Equal(t, segments[0], findSegmentByBaseOffset(segments, 0)) 47 | require.Equal(t, segments[1], findSegmentByBaseOffset(segments, 1)) 48 | require.Equal(t, segments[4], findSegmentByBaseOffset(segments, 39)) 49 | require.Nil(t, findSegmentByBaseOffset(segments, 41)) 50 | } 51 | -------------------------------------------------------------------------------- /server/common_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "strings" 7 | "sync" 8 | 9 | "github.com/liftbridge-io/liftbridge/server/logger" 10 | ) 11 | 12 | // Used by both testing.B and testing.T so need to use 13 | // a common interface: tLogger 14 | type tLogger interface { 15 | Fatalf(format string, args ...interface{}) 16 | Errorf(format string, args ...interface{}) 17 | } 18 | 19 | func stackFatalf(t tLogger, f string, args ...interface{}) { 20 | lines := make([]string, 0, 32) 21 | msg := fmt.Sprintf(f, args...) 22 | lines = append(lines, msg) 23 | 24 | // Generate the Stack of callers: 25 | for i := 1; true; i++ { 26 | _, file, line, ok := runtime.Caller(i) 27 | if !ok { 28 | break 29 | } 30 | msg := fmt.Sprintf("%d - %s:%d", i, file, line) 31 | lines = append(lines, msg) 32 | } 33 | 34 | t.Fatalf("%s", strings.Join(lines, "\n")) 35 | } 36 | 37 | type dummyLogger struct { 38 | sync.Mutex 39 | msg string 40 | } 41 | 42 | func (d *dummyLogger) logf(format string, args ...interface{}) { 43 | d.Lock() 44 | d.msg = fmt.Sprintf(format, args...) 45 | d.Unlock() 46 | } 47 | 48 | func (d *dummyLogger) log(args ...interface{}) { 49 | d.Lock() 50 | d.msg = fmt.Sprint(args...) 51 | d.Unlock() 52 | } 53 | 54 | func (d *dummyLogger) Infof(format string, args ...interface{}) { d.logf(format, args...) } 55 | func (d *dummyLogger) Debugf(format string, args ...interface{}) { d.logf(format, args...) } 56 | func (d *dummyLogger) Errorf(format string, args ...interface{}) { d.logf(format, args...) } 57 | func (d *dummyLogger) Warnf(format string, args ...interface{}) { d.logf(format, args...) } 58 | func (d *dummyLogger) Fatalf(format string, args ...interface{}) { d.logf(format, args...) } 59 | func (d *dummyLogger) Debug(args ...interface{}) { d.log(args...) } 60 | func (d *dummyLogger) Warn(args ...interface{}) { d.log(args...) } 61 | func (d *dummyLogger) Info(args ...interface{}) { d.log(args...) } 62 | func (d *dummyLogger) Fatal(args ...interface{}) { d.log(args...) } 63 | 64 | type captureFatalLogger struct { 65 | dummyLogger 66 | fatal string 67 | } 68 | 69 | func (c *captureFatalLogger) Fatalf(format string, args ...interface{}) { 70 | c.Lock() 71 | // Normally the server would stop after first fatal error, so capture only 72 | // one. 73 | if c.fatal == "" { 74 | c.fatal = fmt.Sprintf(format, args...) 75 | } 76 | c.Unlock() 77 | } 78 | 79 | func (c *captureFatalLogger) Silent(enable bool) { 80 | } 81 | 82 | func (c *captureFatalLogger) Prefix(prefix string) { 83 | } 84 | 85 | func noopLogger() logger.Logger { 86 | log := logger.NewLogger(0) 87 | log.Silent(true) 88 | return log 89 | } 90 | -------------------------------------------------------------------------------- /server/configs/authz/model.conf: -------------------------------------------------------------------------------- 1 | [request_definition] 2 | r = sub, obj, act 3 | 4 | [policy_definition] 5 | p = sub, obj, act 6 | 7 | [policy_effect] 8 | e = some(where (p.eft == allow)) 9 | 10 | [matchers] 11 | m = r.sub == p.sub && r.obj == p.obj && r.act == p.act -------------------------------------------------------------------------------- /server/configs/authz/policy.csv: -------------------------------------------------------------------------------- 1 | p, client1, *, FetchMetadata 2 | p, client1, bar, CreateStream 3 | p, client1, foo, CreateStream 4 | p, client1, foo, DeleteStream 5 | p, client1, foo, PauseStream 6 | p, client1, foo, Subscribe 7 | p, client1, foo, PublishToSubject 8 | p, client1, foo, Publish 9 | p, client1, __cursors, Publish 10 | p, client1, foo, SetStreamReadonly 11 | p, client1, foo, FetchPartitionMetadata 12 | p, client1, foo, SetCursor 13 | p, client1, foo, FetchCursor -------------------------------------------------------------------------------- /server/configs/certs/ca-cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIGKzCCBBOgAwIBAgIUDDjsUMBjb2qd9QcnIOBHyl6nG7QwDQYJKoZIhvcNAQEL 3 | BQAwgaQxCzAJBgNVBAYTAkZSMRIwEAYDVQQIDAlPY2NpdGFuaWUxETAPBgNVBAcM 4 | CFRvdWxvdXNlMRQwEgYDVQQKDAtUZWNoIFNjaG9vbDESMBAGA1UECwwJRWR1Y2F0 5 | aW9uMRowGAYDVQQDDBEqLnRlY2hzY2hvb2wuZ3VydTEoMCYGCSqGSIb3DQEJARYZ 6 | dGVjaHNjaG9vbC5ndXJ1QGdtYWlsLmNvbTAeFw0yMjAzMjcxMzEzNDFaFw0zMjAz 7 | MjQxMzEzNDFaMIGkMQswCQYDVQQGEwJGUjESMBAGA1UECAwJT2NjaXRhbmllMREw 8 | DwYDVQQHDAhUb3Vsb3VzZTEUMBIGA1UECgwLVGVjaCBTY2hvb2wxEjAQBgNVBAsM 9 | CUVkdWNhdGlvbjEaMBgGA1UEAwwRKi50ZWNoc2Nob29sLmd1cnUxKDAmBgkqhkiG 10 | 9w0BCQEWGXRlY2hzY2hvb2wuZ3VydUBnbWFpbC5jb20wggIiMA0GCSqGSIb3DQEB 11 | AQUAA4ICDwAwggIKAoICAQDq2aoLOtHW5WPeQMQWw365IOXLO5yhTxyWRNnSESkw 12 | V0Pkw8p1Q1x7gVEGjvLudPfi5pz/irXLifuDfbnQEiw8hInknTMFtMbUGuAFcMu/ 13 | AQeoDRwaCzyDq6tbny7jMgmbr1EKUxFPYD0pMgEZ6andlfjePABKzVjQCgu/tZQg 14 | AEwf7vnxTWc04oZZf0FGHB4aVkZFxrZmGptzKqnf6VXIQYXp3XoDEZlmCmZhmIod 15 | Rc8P+SzyWZm8wJ8NhizL66aBHgItDEDzw9I+LUiHeyOS0X+srcb3mcNb1o+pRAo+ 16 | o8o621Ey1yJmIz6EkJxvo9IvaWVGUoir4RVqzIbp3k4FGsEgP7z3K4HhvGU+Gsv8 17 | pXp2HgRwCmG9H9dC8gy2EVyoaxpOoPAsmUCH1lU36eJOFYzqtgFsMhE0jfrUCU7n 18 | Pq+PTAaFEbfeyNKg3bVyWDwVP0qKl++PaoG1ZwR70y33kVJQeerMhjFZY10421P1 19 | oPpO43uToR26cwMRgzngQhXutIi7WIkgkxBPI5tiHcDNX8WiHXBNcFDKkPOn/rO1 20 | 50khpfGc+hq4eaBAtDEBH+R9PLcYCXxe1WEXpG3UkZ0N2jFweooOdQXCwM040lWy 21 | fbetBGc2a59Tp+UAe55VspOuWHF0fWzZpLjuzo3TAhTPngL+xkH0ZxVwPkQe8uCm 22 | awIDAQABo1MwUTAdBgNVHQ4EFgQUVkz235XIc4ITxphuHJ4nAlAojhAwHwYDVR0j 23 | BBgwFoAUVkz235XIc4ITxphuHJ4nAlAojhAwDwYDVR0TAQH/BAUwAwEB/zANBgkq 24 | hkiG9w0BAQsFAAOCAgEA1U/Js68OyPUcUX+jFUSK12oHbvI5TA4YCYbFGIj1V74K 25 | S9v5qHf5PhneM61sNiE1rqKSTn7RZ8Q7BtrI5mNKMjqVVK11JyjRpA7oMYZ0wISC 26 | /SJsIXiFgOdwnompx5qNw0Ng7yDukdoOzebBG5Dj0knOPLoWLve8ebH5iOLR+OjL 27 | vAFxT2x43QzXVFrLVfyo5OTC9eMRATe4ux5Zm5KVy5UaplH59T2/uuTzb74hUvN9 28 | PsgOYXwXXVP2m8Z33qro8+qk61iHn50z7PEDsDhqMkwGl6ASEr5+S3cWhoZRVXZq 29 | 2QzhJHgSu6MzRq6g0cNiCtcJgZqSvOhK+uVXR27utaB88ZzmUIdsJr9z8eXHTHYB 30 | f+RcCDIRWA75TZI+k98Hzj2Y+MxJwUdMMYtv3QYMzOQX4OXP9+hwVxHwaVcVSvvA 31 | xAcQDOYB5Us/NbH7Xa2Uejvo7P5fURz6PasmFoLVhVnNlBZ3RkoxFrz28PESGrzK 32 | CTJ96eAc066CUkwElwpYOGZ7ofiEkMqrnPNwV0hZvA02jKGXFRND2bx6EloTtWGs 33 | tqD2k949vB4mJNWwchadfKSNV7gZKWaFXLffGxu5YoZSobDgTIQnycmppjOjQ2kR 34 | kUNH/ASpysVGwS9JjH+Y2jocHEOKsgd3FCoWqdqNHYxG3bBphe16cLODAF4ewZI= 35 | -----END CERTIFICATE----- 36 | -------------------------------------------------------------------------------- /server/configs/certs/ca-cert.srl: -------------------------------------------------------------------------------- 1 | 6CD183238D13A1E750E9467A3C62FEA83000749D 2 | -------------------------------------------------------------------------------- /server/configs/certs/ca-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQDq2aoLOtHW5WPe 3 | QMQWw365IOXLO5yhTxyWRNnSESkwV0Pkw8p1Q1x7gVEGjvLudPfi5pz/irXLifuD 4 | fbnQEiw8hInknTMFtMbUGuAFcMu/AQeoDRwaCzyDq6tbny7jMgmbr1EKUxFPYD0p 5 | MgEZ6andlfjePABKzVjQCgu/tZQgAEwf7vnxTWc04oZZf0FGHB4aVkZFxrZmGptz 6 | Kqnf6VXIQYXp3XoDEZlmCmZhmIodRc8P+SzyWZm8wJ8NhizL66aBHgItDEDzw9I+ 7 | LUiHeyOS0X+srcb3mcNb1o+pRAo+o8o621Ey1yJmIz6EkJxvo9IvaWVGUoir4RVq 8 | zIbp3k4FGsEgP7z3K4HhvGU+Gsv8pXp2HgRwCmG9H9dC8gy2EVyoaxpOoPAsmUCH 9 | 1lU36eJOFYzqtgFsMhE0jfrUCU7nPq+PTAaFEbfeyNKg3bVyWDwVP0qKl++PaoG1 10 | ZwR70y33kVJQeerMhjFZY10421P1oPpO43uToR26cwMRgzngQhXutIi7WIkgkxBP 11 | I5tiHcDNX8WiHXBNcFDKkPOn/rO150khpfGc+hq4eaBAtDEBH+R9PLcYCXxe1WEX 12 | pG3UkZ0N2jFweooOdQXCwM040lWyfbetBGc2a59Tp+UAe55VspOuWHF0fWzZpLju 13 | zo3TAhTPngL+xkH0ZxVwPkQe8uCmawIDAQABAoICAQCpDOuJT9rSzKaZypccJ7cl 14 | fdfL9mol93OFe6QrwWybvoVBu+oVhNqikJCJnwahvZLeQtCKZge8ruNcYHkLnLk5 15 | CtLTvJCIr/tbnMjbQbl4ET64qk8rNCFoSn974Vb39gHSjl1QiEDymT0NVVBTnc5H 16 | kfflyN0Q+0XN4x1X71XjvzMM3ZIPL392IwVAkQikdgL5RFcrTQHo94gkW3aSljzI 17 | fNnvYRy97sI0IJJX+Fya+7A0OFIxVWwD4cfcDVRbqZJeaQJ5+NMcZT4GwcPuuiX+ 18 | Pk2gOCr7+jQU/JrG5hJVysL8oPiPFytMyFvtQsdI2Wyn8j8TYr+8sbYf/X+0AvC2 19 | /Nr9Tbh7vN5OpZy/i+6l3fKRhfnRFRVX6gYGog4gjcqtiI3wKXWlIbND5eiZ+vMK 20 | VTitjKGqnLq7KRqr+U57fPj3WJdk3VZ8wDj0TEEFn2bVWQRb5pL20cm5dIOkqk7N 21 | xBnsr2Pdt0mxMq2J10r5UghGmUGjhp2lDWZ1kfDwLzxPexZNVWDOkXoMWsHmRdGQ 22 | xF9B+vRjz97u12p5zSdq9815t1Qtw2GXIkzqyXuOObxJdP7cu9vR6lBtTdyCbI5g 23 | QYrcgokcNf+EWKkivtSdzrXavMqNFYiW8qAlPurADbo+BTuCg18ogsCCwQbuuZqf 24 | MEgy41UaHqTaG1kN80GM4QKCAQEA+ju0mJMzZ9O5/klBxkvuUo9QsdBUeyZAdoRf 25 | rR8WsPDeasIrtIEeX+NYnlXIroVPvroskMiFzJdF0ZYduuGEQlL255H3x/eLLVgV 26 | TQq+hCy5AWKKx7KHW29Yl0RpW9zOTNfQ41lLk9mPofmfi2a3LX3HLnwXH9zSl8hp 27 | 5heW4rWdZAPt7KQz2jvJEHAijgngvFW6Qo+KAJry+QJLdma1dZyyhO6isiXQ8/ik 28 | hhvNuhSuqsX6wzFEDWIMkj2d0Ib6yKwUOa2Xgr7hQAldrkFI9DR7qOzY1zre+SbJ 29 | lbKSdhSXgVo8LZHwKEyjmj4bfxZu0kJq4Y3ifq8619n0ca8m+wKCAQEA8EM0SXPJ 30 | WLBrt3huso2Ug9//VJRnBFdx4HP1mC3VDppXlejl5NJC8nPSLFriXBXnSnupzggo 31 | K4uifjdfWFxlpzikblaqReV/k0OwtCIjIZDRl8nl+kz9Lgyb6Sg38UWftCCK8b+j 32 | 3bKqL9aQZxPfY0pMTJKUvoCvtQG8OjAbhE6YjCGUg7Bqy/YF563DayPfAHAoqIiL 33 | XgbCgmAy2kpG5Bi1UrhKc198SZ60XTz2jDD65J6uJ9HrP7dTy/xCDrhlY/SxOJDj 34 | w/ZZJHCMzt/q0ZtSjcUqoPp85xkI5uMMNIt8JZmURvfXh6aKSSKNdUHOBemYss5L 35 | VWBe9LXREpMjUQKCAQBNE/UGBlmQQsSI0lHjXeI0JhcKHozXPiofF5lM/0WDDT7F 36 | jbfequMLUEEszGm7cC7nJfuyQUINig8khuPze6G8uEd7fxfezZ5eQkKVb5jNp+T4 37 | yzzKVHCjNoIIXjdB55rYSqX6UbgY+6vljbmaO/Jyncqrw+dvlhp8TGxqRpvgi2bG 38 | tvsjqFSchUvit1e9fsdt28460HIGy7PKBe3us6ZzaugGUGdnDoT1kYJEGO5ewh4n 39 | VkJuu48lvCz5Iueoots+0tqMBa+kw10o13m2wj9RkZUBrKsCaQzjnBH9/XplyuP8 40 | ISpmMwzRrQG78iOQGv3Z4EGB5q18rkcm0+ka14PfAoIBAQCTa2jluNnORVGSnZ6u 41 | iBicYhC0wOoEy/LfmccTvOuBrkoXfXOx1yGkylQnSwyhG/9ywDYMaQzcyyzE7Qzf 42 | lrH4zNR3r02C3bJNlbcSj++mZMl1rTgjQKIIY0w236qTq/i1+VHHy6KsITgzah/o 43 | X1UuAySVx0rlKOim042+1k3L/L21HdWWh+S/iRFOelvxnWzzQ95uqnl5FAS9InGR 44 | ZwngYxi+zL1B1VDZizt4CjPtCRCovoR1gmQqED6mZv3RMmtjzXwADUbzsnA336dG 45 | ODZIrlkVC+mAJLIGymGf6ahPhVaDa5yDfwcMAK/Q+BZ74AidsBs1e0bV9+/LjtPd 46 | 5dJBAoIBAQCe/y7Z5iujLwP+XLnasKiDZqsK6wKUhu6Zw3K8Yv7wXD6/AWhTr6NL 47 | RlsGLgyIH3grG8A6XLfbkSBJeZY82vEj4DghecIDe9je4y60Ah8NIjhwB7O8HTq3 48 | SCPcYAUNJaiXW4CQdMY7MzDultvx/YF4daWCsHo7jnezjALHjcmdyO7MQwWW2dfn 49 | MLqCe394vwJuIPmNt+N+3Pcph32pKKXtVMzowSJCNT2eNQI3GVaXhZKLf5rzt+Yw 50 | oudsbyb4QTIF9EMiuzzTXsg4U3kzcjPHpvmkbZuhmpMU5mxvYw9Hd5ENxeFZz47G 51 | 2P3PROE4cKO8sMWKoeYWFZ93soEVkYoi 52 | -----END PRIVATE KEY----- 53 | -------------------------------------------------------------------------------- /server/configs/certs/cert_maker/Readme.md: -------------------------------------------------------------------------------- 1 | Inspired from [an available script](https://github.com/techschool/pcbook-go)m `gen.sh` is a bootstrap script to generate pairs of self-signed client/server key pairs used for testing or local usage. 2 | 3 | To generate different key pairs, just execute `gen.sh`. -------------------------------------------------------------------------------- /server/configs/certs/cert_maker/client-ext.cnf: -------------------------------------------------------------------------------- 1 | subjectAltName=DNS:localhost,IP:0.0.0.0 2 | -------------------------------------------------------------------------------- /server/configs/certs/cert_maker/gen.sh: -------------------------------------------------------------------------------- 1 | rm *.pem 2 | 3 | # 1. Generate CA's private key and self-signed certificate 4 | openssl req -x509 -newkey rsa:4096 -days 3650 -nodes -keyout ca-key.pem -out ca-cert.pem -subj "/C=FR/ST=Occitanie/L=Toulouse/O=Tech School/OU=Education/CN=*.techschool.guru/emailAddress=techschool.guru@gmail.com" 5 | 6 | echo "CA's self-signed certificate" 7 | openssl x509 -in ca-cert.pem -noout -text 8 | 9 | # 2. Generate web server's private key and certificate signing request (CSR) 10 | openssl req -newkey rsa:4096 -nodes -keyout server-key.pem -out server-req.pem -subj "/C=FR/ST=Ile de France/L=Paris/O=PC Book/OU=Computer/CN=*.liftbridge.com/emailAddress=liftbridge@gmail.com" 11 | 12 | # 3. Use CA's private key to sign web server's CSR and get back the signed certificate 13 | openssl x509 -req -in server-req.pem -days 3650 -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out server-cert.pem -extfile server-ext.cnf 14 | 15 | echo "Server's signed certificate" 16 | openssl x509 -in server-cert.pem -noout -text 17 | 18 | # 4. Generate client's private key and certificate signing request (CSR) 19 | openssl req -newkey rsa:4096 -nodes -keyout client-key.pem -out client-req.pem -subj "/C=FR/ST=Alsace/L=Strasbourg/O=PC Client/OU=Computer/CN=client1/emailAddress=liftbridge@gmail.com" 20 | 21 | # 5. Use CA's private key to sign client's CSR and get back the signed certificate 22 | openssl x509 -req -in client-req.pem -days 3650 -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client-cert.pem -extfile client-ext.cnf 23 | 24 | echo "Client's signed certificate" 25 | openssl x509 -in client-cert.pem -noout -text 26 | -------------------------------------------------------------------------------- /server/configs/certs/cert_maker/server-ext.cnf: -------------------------------------------------------------------------------- 1 | subjectAltName=DNS:localhost,IP:0.0.0.0 2 | -------------------------------------------------------------------------------- /server/configs/certs/client/client-cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIF4zCCA8ugAwIBAgIUbNGDI40ToedQ6UZ6PGL+qDAAdJ0wDQYJKoZIhvcNAQEL 3 | BQAwgaQxCzAJBgNVBAYTAkZSMRIwEAYDVQQIDAlPY2NpdGFuaWUxETAPBgNVBAcM 4 | CFRvdWxvdXNlMRQwEgYDVQQKDAtUZWNoIFNjaG9vbDESMBAGA1UECwwJRWR1Y2F0 5 | aW9uMRowGAYDVQQDDBEqLnRlY2hzY2hvb2wuZ3VydTEoMCYGCSqGSIb3DQEJARYZ 6 | dGVjaHNjaG9vbC5ndXJ1QGdtYWlsLmNvbTAeFw0yMjAzMjcxMzEzNDJaFw0zMjAz 7 | MjQxMzEzNDJaMIGRMQswCQYDVQQGEwJGUjEPMA0GA1UECAwGQWxzYWNlMRMwEQYD 8 | VQQHDApTdHJhc2JvdXJnMRIwEAYDVQQKDAlQQyBDbGllbnQxETAPBgNVBAsMCENv 9 | bXB1dGVyMRAwDgYDVQQDDAdjbGllbnQxMSMwIQYJKoZIhvcNAQkBFhRsaWZ0YnJp 10 | ZGdlQGdtYWlsLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAL/n 11 | CNxuyhslYyJEItNrMokJ5FVCaHy7Fl8UsZDwoU5C3GEMD2n4wYTaX7yetRuOFBPH 12 | m7IyO57ZFgPOubnNDTQp5Ca6hZOoZTp46x9hc8+zLf/7DNM0O/PHWb58ornqcW9/ 13 | a6fGeWTIFACRj0uXpg1mmwPGx6xlx9Dqx52zPO8dPZtv1i9FqQZvs2tn66mlHnVo 14 | ENfJewX2dwMCRD6kAL9y878da5FzajO7evocnveSBa7S8i0zNocqJYmyvRBg7IWu 15 | sUdXnUvQu6eRl2l3AWxFKYt6AtZb6zGr8YZou1v7ADGhhC33IMOOg4alhuw2Tp1h 16 | z5WbsektO6MQ0n6dDVhM+7OSrX0oia/HopxjjrQZYBCChXv5IB0Fh6JHxH3MhiMe 17 | rtHFxs606OeNF8x8fGQvxI6kfROs6WnJzw/TZQUkhoKer/qkHxiKz6NCK9MosZ5t 18 | drjgOhed6WPNkE/KuA4c1EYXzvVY8CYFnWS9lhKr1v06rOuV7ksRWLnoC12WqkDR 19 | 6oDi0tU6u1m9IUkidBSAdtfX4ko15Au6hJgneezVMEin5Mix8wmqBb3h5vv9ddUG 20 | sCWut5S00QOlBjGtIedWECtuODwgls2JpUcLMdtOMEVmYo1eJg10oXV4a6r5ZZE+ 21 | TO2OQ8Pl7aUUMifzVU15OetO+iZLEP6QC0iA4YsZAgMBAAGjHjAcMBoGA1UdEQQT 22 | MBGCCWxvY2FsaG9zdIcEAAAAADANBgkqhkiG9w0BAQsFAAOCAgEAljMU3jJtSJnD 23 | yzWh0fnF7N4VFqEXIBwhUPpRzLMxK+BqX8x3Ldqlb51qo/mAIjeVU8xEkRjaUHGp 24 | rNf14DkLaHFfhQ2qmLbTdIpcvUcomdRBj8rtMPM9nzPgT8rLD229w8HTWY0ZFOkA 25 | SOTyalp4eh0gKhJzrIHJPuYHbY5g9FsAcUDZTuy3zLpHjBJSXVM6ErMXd4dG6zsq 26 | Fr0c+guMqZmg/NJOMAHsHq3ncojDVT3M8H9uSWnRlsBv8LoYmxIYVyGwONFKy01N 27 | 65dMhxZDM+OPsxdXzkX+y06V57ujbqPuuCojjt8ucNy6+nG0xk0DNf5ghaIoEpju 28 | JRnkK91ykMiAqbD5z7HuDGNluX3OokwOGsVbub/bCNSkqwLAjEBdk3YKQyUu3Rrl 29 | zNa6sbYJfyjitQQnsNP6/TvyhDVtT7s8n6JuVks22N89APBG22ph3kPv20NzeTUN 30 | cHuCHHgLrIJi2NAOuiZPAsWbj4WKlz0aaiYNqPZuQX/ZwWb0s5hWbRB2bPpDsBcu 31 | oHtNmR58XBPIqZGguS8aL9br09ypArKublh4f1tRW9j2jBgzyYC/nQ14iBnxJ7Av 32 | J47txMfrPHmA95TUzw6+6qYh8ywhwigsvTS69C3AHw7RsameWblEfJXYI6cn5MGL 33 | 0ND9mgMWVfybVbRBQ8nKK1/VNAY9qR0= 34 | -----END CERTIFICATE----- 35 | -------------------------------------------------------------------------------- /server/configs/certs/client/client-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC/5wjcbsobJWMi 3 | RCLTazKJCeRVQmh8uxZfFLGQ8KFOQtxhDA9p+MGE2l+8nrUbjhQTx5uyMjue2RYD 4 | zrm5zQ00KeQmuoWTqGU6eOsfYXPPsy3/+wzTNDvzx1m+fKK56nFvf2unxnlkyBQA 5 | kY9Ll6YNZpsDxsesZcfQ6sedszzvHT2bb9YvRakGb7NrZ+uppR51aBDXyXsF9ncD 6 | AkQ+pAC/cvO/HWuRc2ozu3r6HJ73kgWu0vItMzaHKiWJsr0QYOyFrrFHV51L0Lun 7 | kZdpdwFsRSmLegLWW+sxq/GGaLtb+wAxoYQt9yDDjoOGpYbsNk6dYc+Vm7HpLTuj 8 | ENJ+nQ1YTPuzkq19KImvx6KcY460GWAQgoV7+SAdBYeiR8R9zIYjHq7RxcbOtOjn 9 | jRfMfHxkL8SOpH0TrOlpyc8P02UFJIaCnq/6pB8Yis+jQivTKLGebXa44DoXnelj 10 | zZBPyrgOHNRGF871WPAmBZ1kvZYSq9b9Oqzrle5LEVi56AtdlqpA0eqA4tLVOrtZ 11 | vSFJInQUgHbX1+JKNeQLuoSYJ3ns1TBIp+TIsfMJqgW94eb7/XXVBrAlrreUtNED 12 | pQYxrSHnVhArbjg8IJbNiaVHCzHbTjBFZmKNXiYNdKF1eGuq+WWRPkztjkPD5e2l 13 | FDIn81VNeTnrTvomSxD+kAtIgOGLGQIDAQABAoICAQCYuUjfDbkBA68yrSE+Otj8 14 | IQg8Jl67rbUvNGvgmvD1NNbo0U1PHC7/CUAaAADIjjlCSKCLn9i9Ia2YmCRNT3iL 15 | pC8z90upaOIvN95/mfIuQT6Bs43QZIi2CVhN9ikXZxKiFrlZy+X+pBDvJujw0x7P 16 | GXKS/dcZR+NPTBpTUjtTXWUTWF0QQBM10R3sg8cUuxlTfN+yrGGhLDYpCdvAMot5 17 | 0gVUexiJqF3EEBfeB3soMmkdajpcaJ6j0ZIQVVSMPWbfOVlTGtJPbS57PK5Qu+pA 18 | /YYDv7WUXZD/dgUA4EcD++16kj18y2mi2L+qIAAR62KDdD1dpoxfs6T7sL6lD153 19 | HObvtLWxdfr0nI2dwwmRcK4yueM9dUX40xKXJWVYHBb0vCCbn20hxIT6hhSIiXGu 20 | ZswyQa7CKnfZ5xyTA/ywOwwVe5Yo3Q4/UeLH5MHfT0ikCd2a1Lc9fOopopj8GqaC 21 | uRWQxAT+XuzXvLChllPUcCGPK5zbA/pJXr8+7y0PWHtGTbLlUi5H3w5QKUq+9loH 22 | 1fgp53BAH4tIAfi0zUVcisn1OxBhXBiWb5IGLeUuRUr1NzCbNZIR9iBl0ABOZKuI 23 | FEVLAsDTP2BChodnYrp31ZEFHx+nC0LWcPWd5e8i4wLfJsA+kFMUF6VAOxqsJAfX 24 | podDZMsIYTcq7GHGVbeD0QKCAQEA/OV7N4iqxrp0qg1QNt5mmwhCGvKJoKgA47rn 25 | Q43QA8EpYPlYSvUknDoppz5asPrjLlFdZWYTlKNaIvOa3RjwXDkQlpiNJnqvuNgV 26 | UtvhQHASmifoPtMc05y05I71BVhX3jsR42kJRO28zJ+dRkMtA+k7RmT0bORmj+OM 27 | 8HpzPtvW+StzjotUAaUp7jFnwzCMU8IsuumpaAuggE1LnuWP0F8Dq9CkeWsQoMF8 28 | AdMrcMP+a5565BsL7OT8AushdSDDVgAYrzNzGjvz+g7K8of0x3d7jwUx9WaTUMzZ 29 | NhDIX3n5wevz39aCEW/EBvfND/a9ANntwFhTS7H+dicTI8Nk5QKCAQEAwkHuGwuB 30 | EY1ykkuoTKKJoNcKgXretcSk1Ooska6QwnUEF5eVn6CSMydwx21/w5K0Bk+lgQg/ 31 | ND7ps4mwiSibrqpHEj0XaeLNJ3VjdgVD9nX1u7r4TCU+CtwM/iqp96923v+38cFH 32 | FSMkFRDj+7jWfVmAkgrONOVAMRjBNYcnvNwAJcA/5MpKOq5ScWVadVdXOr+OOF+N 33 | MlSFfYE1Z+oOuzr7wh504XvDaujExOTJrjyRVbOULF9txRQOlNQLOOctWzoYqlED 34 | 8UvEasp8+CgxLkGQQIFtUeBFSK+zc/KowHYLBzE05EvymjCTHJifMCJ222tLaCEx 35 | UX0MTiaxM5q+JQKCAQB+wEmsgMwPRH6aiIeuqHNGurHfLbZ7Fhk0CoZEnnwmlZa2 36 | quJs10cdIi2kTCVKVMSuf0tPOgUQV3siz3PL4ub9YgCFajo8kfsmqu6tE9Vm5YT9 37 | TIkji84D5aPodhoqLqgDkxmZIBjsOjeJW2qJP7v9HNV/p7O2LBeXgJYwJT9Dvw2U 38 | 3wlQ5VYaaPyGDK8T7m42wLiIifpFUqaEB7miDy6wYh7F65Gz5Ux1NeSASaWZJDKN 39 | H+Y6E7A7cF063TxspxogLXYxwZisCmZy6x0ex7OQkbsU6Kasd1fYhINNjMXQzKK4 40 | ZhlaE/om4Ryf3W0i1ijOl2uJHvvIkZKXB2iZYSJdAoIBAEmBpzbmqIVaz9LZ/Cpb 41 | itao6JnMQ7/mVDUZE5pgwvhCTTUcMAsCOLBQqVVdcu4vch29P7ROyZPchpRgcsPD 42 | 8P0sA867/UMdBmJ7AhLjtS7qvfy2qEQwB4UWdXgr7rsB02pYu2MortwpuvqZtJtI 43 | +yjdmFAq9JKBeUPaySmXJgtJ+GhQkhziCyqfUiUEpDEoxqI9X/Sm+4fjAcxW+z2C 44 | DOb+T8vJuJKmQXEP+X1D0akz1A7o8BXGWoQrrcTVZBW5LKmLl0/Dbkl9USrTymwg 45 | 0VNejdZK43IK+kyh57blSMPjJxMmpIwKzRdZcCFvAzW0pOMse5FAlifuuJxN+dm8 46 | IV0CggEAZ11U3sGvdwVEGBl++DBrxHrMnbZjxtBs0rk9V+ZwfmNPAs5XQb5H/cwd 47 | VogY+8zQXxfcsiEO58+o6uT3p1N8RQMng7feZXmC1K0scqcw4GsywUfY/cqXqhMf 48 | eO4aMUOhd9M+qHWQTeUt5fFnOVQ30KT3SsoofijCxBOSAojVi+XPyeAYgSgAdmuN 49 | d2KJWNt4A084HXizX2UcfHnD0qcRz/0koY2hZId76U+Lx0XmRq0DXZjG5ys2RIRr 50 | +FfT6iELVJWhF1MnsUwM7HW9gqV8YkVL6vKBCzEEByx1CmEtOFe7uBw1xZfv7pf3 51 | XBoHfFBiXCLFYxOh5LVnkvzPf71w3g== 52 | -----END PRIVATE KEY----- 53 | -------------------------------------------------------------------------------- /server/configs/certs/client/client-req.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIIE1zCCAr8CAQAwgZExCzAJBgNVBAYTAkZSMQ8wDQYDVQQIDAZBbHNhY2UxEzAR 3 | BgNVBAcMClN0cmFzYm91cmcxEjAQBgNVBAoMCVBDIENsaWVudDERMA8GA1UECwwI 4 | Q29tcHV0ZXIxEDAOBgNVBAMMB2NsaWVudDExIzAhBgkqhkiG9w0BCQEWFGxpZnRi 5 | cmlkZ2VAZ21haWwuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA 6 | v+cI3G7KGyVjIkQi02syiQnkVUJofLsWXxSxkPChTkLcYQwPafjBhNpfvJ61G44U 7 | E8ebsjI7ntkWA865uc0NNCnkJrqFk6hlOnjrH2Fzz7Mt//sM0zQ788dZvnyiuepx 8 | b39rp8Z5ZMgUAJGPS5emDWabA8bHrGXH0OrHnbM87x09m2/WL0WpBm+za2frqaUe 9 | dWgQ18l7BfZ3AwJEPqQAv3Lzvx1rkXNqM7t6+hye95IFrtLyLTM2hyolibK9EGDs 10 | ha6xR1edS9C7p5GXaXcBbEUpi3oC1lvrMavxhmi7W/sAMaGELfcgw46DhqWG7DZO 11 | nWHPlZux6S07oxDSfp0NWEz7s5KtfSiJr8einGOOtBlgEIKFe/kgHQWHokfEfcyG 12 | Ix6u0cXGzrTo540XzHx8ZC/EjqR9E6zpacnPD9NlBSSGgp6v+qQfGIrPo0Ir0yix 13 | nm12uOA6F53pY82QT8q4DhzURhfO9VjwJgWdZL2WEqvW/Tqs65XuSxFYuegLXZaq 14 | QNHqgOLS1Tq7Wb0hSSJ0FIB219fiSjXkC7qEmCd57NUwSKfkyLHzCaoFveHm+/11 15 | 1QawJa63lLTRA6UGMa0h51YQK244PCCWzYmlRwsx204wRWZijV4mDXShdXhrqvll 16 | kT5M7Y5Dw+XtpRQyJ/NVTXk56076JksQ/pALSIDhixkCAwEAAaAAMA0GCSqGSIb3 17 | DQEBCwUAA4ICAQA5AmGzWS2t2X+cyHOl5MF/mLOY/M1lXNg08Aq5vT7U8PJta195 18 | B7tQcbRDK0hGloSjxanjZnM0O38c3l4mD68cCN1HfnQl29EOp+XP4hYJr9tzdu5l 19 | Ej2NOSWmR6P1rn+DZrNqw1tepTerAogJfFEIAg21vkP/wUZPfzrqAcuSDNrzWgJ9 20 | qPBuWX+QnsEH7gf8MprVb3LHNcdZxuEN2sDuq30B3xfAef6rktO2SQnu7l6AAMht 21 | YjRC03AmAvTsUpPXKPQZo0w0UzxXds08/6Dz9MUmfVIhAQ4+wLSQIb/fpIUG4+Tc 22 | ZZH8rRgEJtVIA7MjDYrAG6sBkS2+g+e44skZxYDPdGdgZDEVaqv4/QqAeq2y7T4M 23 | unrzbqE+Bu7vyx0HNQy25EQIjlORwYE3SP9ycWX5kb//wtn2pkvDcNYFJefiTt9c 24 | JwMkEmYyvd8xjowimkh0Gw3QEfPh5sLTpAWOMH5RLPaMo6VoLOEmBj3MmVB/GoLA 25 | 9ntAjBXvpqGC5xXY2fPxIjTiDvx3FjlrYRnc07nAxtuvp77ZsRlCkNxDbB27Pz+z 26 | WqVI6PJFgSBNj0HvDbbok0kW+nyX3pHJYnY4valZmWmxwkYdzL6D70N43z4b+FL7 27 | Vsq3bnxbogmCroFttmLB2DevPZ0BHNtEjDsUKawjuUxycaXiLMsx47wLIw== 28 | -----END CERTIFICATE REQUEST----- 29 | -------------------------------------------------------------------------------- /server/configs/certs/server/server-cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIF7DCCA9SgAwIBAgIUbNGDI40ToedQ6UZ6PGL+qDAAdJwwDQYJKoZIhvcNAQEL 3 | BQAwgaQxCzAJBgNVBAYTAkZSMRIwEAYDVQQIDAlPY2NpdGFuaWUxETAPBgNVBAcM 4 | CFRvdWxvdXNlMRQwEgYDVQQKDAtUZWNoIFNjaG9vbDESMBAGA1UECwwJRWR1Y2F0 5 | aW9uMRowGAYDVQQDDBEqLnRlY2hzY2hvb2wuZ3VydTEoMCYGCSqGSIb3DQEJARYZ 6 | dGVjaHNjaG9vbC5ndXJ1QGdtYWlsLmNvbTAeFw0yMjAzMjcxMzEzNDFaFw0zMjAz 7 | MjQxMzEzNDFaMIGaMQswCQYDVQQGEwJGUjEWMBQGA1UECAwNSWxlIGRlIEZyYW5j 8 | ZTEOMAwGA1UEBwwFUGFyaXMxEDAOBgNVBAoMB1BDIEJvb2sxETAPBgNVBAsMCENv 9 | bXB1dGVyMRkwFwYDVQQDDBAqLmxpZnRicmlkZ2UuY29tMSMwIQYJKoZIhvcNAQkB 10 | FhRsaWZ0YnJpZGdlQGdtYWlsLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC 11 | AgoCggIBAMkr6nn3TWXLcCOXxsUQTCVcgt9PDTjrarFUMFmKOKvKjwf+yrADJxmz 12 | QSzrmNR6MG0DlaPNUlqmybniNpNn3yEG8Vd0T0/vYMFIMk/uoikDB0v6YSeNY9Yd 13 | j1ZwFPfajvFWRJiRqieO56IyZei96cDDxA1afzkl3Tu/pA3YmkonIM20ZhFCtKkM 14 | ktXNbZsmFKtp5lZvAJzsOoqc+bwt7gMZ8E8oT8o4D+nD4eqCw7BIJlSui4OiHcBD 15 | gkUk1Ms5AUCHf08cDPkAPb4bw75fcFmiAKUXQBb3gmHOyPF928WTsILXDUM9M/Sz 16 | gxj6mSoM4L5DK3KC6XpyfWr0Jgx1HxvhEEN+5op6X2UX4U9WIXzzJVsVLf9fbWq4 17 | 6Uo7ZALO+EiUiRRtFFodC6Wn8FU3aaJGNjV3SYmiDZIqlxViQDq3Nb2Oyxy5oNON 18 | Ys/i/t0DSfp0mmSnPWBXpn3gYZes5zdA6NySLnmj+QbGXCYw3OutpJZcZY+XyXri 19 | 6bslsntqZVOci5PEDkqT6MLSkW7YbYE7fU6YOytv7Qxj1ziskt2p12UUC4vuVsCB 20 | gCHjoprpxlUlu5uFA/I9hyRNR8brWbPAL0OAc5BCT1MEGtN7xGm7ks8ZKFg5PoXb 21 | ueSi/QpHqVBm8HdcoCBU/SxXsKpy6oy1bjzxonZv1HpMKCmTFgpnAgMBAAGjHjAc 22 | MBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEAAAAADANBgkqhkiG9w0BAQsFAAOCAgEA 23 | aCNWVnoOpT4knkreGE7GNsa1eYPqTEU+7rlBzHvHpoATfU+0YYSnydkb6zfDLXW0 24 | 2a/529Tbx+tY3n9YrZuj52AoG7HnlI3D1krKfXRCX80bwOTqxdc8EryqyJi7DESh 25 | SVJUHcKJM4sBq+uXQsUezMTkNq5/Y/lgW3nJ3eYGv3OGVVw+z8cnDuDjJGAmIAtG 26 | bwsHXQUPSSn9TRPSrASM+8ZVbEtnpu1EioGrN08Xc41EzcpOjZY7JpeAeVMV4TZq 27 | efT30luBT8TbUwEQiOk3ab8VAM9899IUo5z9Ndbfi1fmyUli+XiSwFnIH2V/r5sH 28 | KEI6LfA/feo/lKhGamSvJeRiIc3vf2Mtl38WQnn9mat4ZTCPLFsjTI8wYMXy1KE2 29 | zH/+ZhE32ILzlk9u5nED01NzAJkfbUbXl6O4CfO2eiYv/NSUW0WoRX4W28aHRWxn 30 | IK72Y2VGv6ShtKFNrTAmnezARJqZ5mkFUdaHCs0VTXGJh5eZv3c+lU878sCov3iH 31 | jlwEgjqPw3/EwO64RAUauRYLHmUdUWFI6AHeXDXKpYxHY5oTml91cZUDT9yQ3Db2 32 | jlqGuO44BnM2Gh3txrH6iVEsOOQRNvc5+L5OwUCPM5j4/Y0crN612CIMnJYcSqLD 33 | BCCS6laznpt+4hi+WgalcXPI2qm4opNrO5j1dCxDxmg= 34 | -----END CERTIFICATE----- 35 | -------------------------------------------------------------------------------- /server/configs/certs/server/server-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQDJK+p5901ly3Aj 3 | l8bFEEwlXILfTw0462qxVDBZijiryo8H/sqwAycZs0Es65jUejBtA5WjzVJapsm5 4 | 4jaTZ98hBvFXdE9P72DBSDJP7qIpAwdL+mEnjWPWHY9WcBT32o7xVkSYkaonjuei 5 | MmXovenAw8QNWn85Jd07v6QN2JpKJyDNtGYRQrSpDJLVzW2bJhSraeZWbwCc7DqK 6 | nPm8Le4DGfBPKE/KOA/pw+HqgsOwSCZUrouDoh3AQ4JFJNTLOQFAh39PHAz5AD2+ 7 | G8O+X3BZogClF0AW94JhzsjxfdvFk7CC1w1DPTP0s4MY+pkqDOC+Qytygul6cn1q 8 | 9CYMdR8b4RBDfuaKel9lF+FPViF88yVbFS3/X21quOlKO2QCzvhIlIkUbRRaHQul 9 | p/BVN2miRjY1d0mJog2SKpcVYkA6tzW9jsscuaDTjWLP4v7dA0n6dJpkpz1gV6Z9 10 | 4GGXrOc3QOjcki55o/kGxlwmMNzrraSWXGWPl8l64um7JbJ7amVTnIuTxA5Kk+jC 11 | 0pFu2G2BO31OmDsrb+0MY9c4rJLdqddlFAuL7lbAgYAh46Ka6cZVJbubhQPyPYck 12 | TUfG61mzwC9DgHOQQk9TBBrTe8Rpu5LPGShYOT6F27nkov0KR6lQZvB3XKAgVP0s 13 | V7CqcuqMtW488aJ2b9R6TCgpkxYKZwIDAQABAoICAQCISJYKlnz6jz2i/zmLWyUa 14 | 0nderQE6JFIdW/l9T2RhDVKkifnqD9i3Una+5cSdfUG9OIZxd7Fo9EEJCrUKW34P 15 | s9Jf+s2YS4Tyq+RZvkJhIkoZMMAMuX5/GXt3hWrPFmNsH5NNALGTJv7JJPdhGvd2 16 | vajdmwdBQeOEjKmpL6omvic89A+JdfVJ6Ni4uvib+Mpb7dw7heuWY3qtWPbegwcR 17 | Vssz9Q1I4330Ud2Er52+mMJO8AhX3sXk8FcaXH9ERZQRkTmv8ELhlUY+ujR7jdBh 18 | CJ0DOqMSbA8KD7qVzzvJt+oGWYYEnTvLdKlFjM+iayL+Aic25nUhnNpWUJxOKDIv 19 | zw5blb/aMgvYXpUsAhJ7GjjOKqy8CuAqfhFzAoR/lw8HB9UKw17Y1lhYiI4uGnrM 20 | YTc+5I5yqX9C5SoQ4YGYN3kClD9FqWsmUbBF1ViloM2BnZ4C1y/9UFpRH9XvgMHE 21 | mHyRRakr4o0g6ZhPNfqumt6zgFtgfW2PJ3KZpJhq7nvc2vTqVvK7gcSbY8ykK4G3 22 | 4c9c/46x+umQx3oRCwgGTOLU3ItQ+ipFNWuKdETiqAyh9t60EpcNZwhk08p5UGi0 23 | TFytTstJU184rJU/kDlMhEKuOzIGgSku6LDy+G+TBT6Iixe+LppRNr6KINcCTyWT 24 | QEJNfwddUJEhUZc6essqsQKCAQEA7geyneZI90lxbYVWzmpfanG0NplVC8ZRd2c5 25 | Oykm95uHshvjp/pclhzxiKIGLHOc1IPvwKGLsUSBGGd8YxBlgXC47WbXI7kIyNnF 26 | YGlTsSpXkGxXs5BP/LbgTu5FWfiMSPX0anTCQcqNaxyraqNJao9NGRhrKYNHpcyF 27 | Cu7Ayoi7dF4R9sEQTcyO0y3Z9G1ma085Ppra8mgOhNhhQ4cxFaeYohsy4bOwbvCm 28 | /YsPmuiA81oZWMvbRzVjUFVRQqYWFK3cAMUsLojQNsuItSkI6QoX1BZUrDzZ0vro 29 | xMISmeRNF7Mr/EGZD2yUFPxgDsoZ02i/9zwK8B3H2soOR0WKRQKCAQEA2FveqXdE 30 | WXpV/y4dsNX08trV43z5cFrUTe2tYyGx+I89gaM57IsbZlYGlTGl1zwM6rRQF+7Q 31 | ldgXNxU8MHja+z+4BAmnSutbYubMpJD5wu9iP2l+5KUdi8DYMjxaI4FTHazODoxl 32 | 41uUONFYBs4j5XtDEGfD0nbiltuiQbsj7hFGzh0fwdsOxadsQpj8mPZFfAj1AkjE 33 | CD28pry8moAbBw/+ottW1pQ8dg3/ud7QC5TDAWCagmopcpQQX6dqB5NhNnj84F/A 34 | 0G/HMyBszGQcw0UubmNiwKkPL/2MFHZiPeNgx2nNAZiR/YxwLcFoTeqsKd5iaYGl 35 | JeNQsWnYiXeCuwKCAQEA3/0gaRP2H7Xd5ijuielxhnIfGq3paN4jdVAOfAx6ndCe 36 | vc0eysh+7ceoxmFpm9TjhNvu9f9Ou+5x6OwhEfuw+UCA1O3Mj9IkYRUEdnhHCFWG 37 | 9uHtGY04p9/TbpMrccHBCTth1/etgUnBEEV3TS8A/CSDcZUX6oWeG3g8zg/kHfLT 38 | K2sGToRY+kz/Ldxc2HVGRr7TaIVCeY/P8dTImkoSt4TxzcH4fImiApO1IKwGcEhQ 39 | aC+l4HhdDUJBBaxzfltaNfVxLMxeih+2h4m6SHen6dvUloC5Bydv9uijt9vEs0ox 40 | /ZZeUs/L38bWBnWDUwd9jAJrepTm2hO8KLew/gKDXQKCAQEAipsoZUar+eq+pxDj 41 | IoOfPenl9qv8nPcDZb0rRAO1ITiava9VD236qq/X3cKFrQKif8XuPbbX1/cswDQR 42 | cDgsiaNDfwq0KvHmhNC5L4BVEYYMWfV6vn3tFLgBiQVS9cYG+k7XX0igiWwE4/Vx 43 | QELVilFSIHNpZy6UcPLZ2uRJ210kEC1mR+nPZ96fI3cg89lpoFGYrNLzCxiKAAOP 44 | jCcfqYGcrrZ3xlYG+dZ0Cp5sh87QstQxh/T9ApNKfg0hhyLqt1wBHLkbLC0/30gZ 45 | i0NjjdGHMl7nR+fGfAchs4Y339AIExh7plcYx+ctgpKSAMCjdNssDs1ogIJFErNt 46 | bnuquwKCAQBCenkqa3hjamoqtTOIFPo0DOscfqQ0s/+I3RuvkrxCOOfPODqWs1CT 47 | RxeJXAP8HFWJT3+hF5EMoyq3zLrmtCy+TmvbHI4UR7ckyl1s4VUbj8ngVlC+7lL4 48 | /bE1pf9HftbClEVmBoZJIotHeLR+th2dk8b2eCNriOreK3SQ7g59csrI0jW49PRu 49 | gY2D4sOwi0fq6kPvDNxvi/KwaRvXmpgBZ36AxHRVbKEJ0Sb4Atd4SfhuUr+iNwEk 50 | HyhtBimIJbhED6vz4aayJus66pCxxRRkY+7+fXhNIKQebTku9dCFQrHOD/XniX5i 51 | VEFNqpMj7nrZYbt1xluD9OTTLrdfKVjX 52 | -----END PRIVATE KEY----- 53 | -------------------------------------------------------------------------------- /server/configs/certs/server/server-req.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIIE4DCCAsgCAQAwgZoxCzAJBgNVBAYTAkZSMRYwFAYDVQQIDA1JbGUgZGUgRnJh 3 | bmNlMQ4wDAYDVQQHDAVQYXJpczEQMA4GA1UECgwHUEMgQm9vazERMA8GA1UECwwI 4 | Q29tcHV0ZXIxGTAXBgNVBAMMECoubGlmdGJyaWRnZS5jb20xIzAhBgkqhkiG9w0B 5 | CQEWFGxpZnRicmlkZ2VAZ21haWwuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8A 6 | MIICCgKCAgEAySvqefdNZctwI5fGxRBMJVyC308NOOtqsVQwWYo4q8qPB/7KsAMn 7 | GbNBLOuY1HowbQOVo81SWqbJueI2k2ffIQbxV3RPT+9gwUgyT+6iKQMHS/phJ41j 8 | 1h2PVnAU99qO8VZEmJGqJ47nojJl6L3pwMPEDVp/OSXdO7+kDdiaSicgzbRmEUK0 9 | qQyS1c1tmyYUq2nmVm8AnOw6ipz5vC3uAxnwTyhPyjgP6cPh6oLDsEgmVK6Lg6Id 10 | wEOCRSTUyzkBQId/TxwM+QA9vhvDvl9wWaIApRdAFveCYc7I8X3bxZOwgtcNQz0z 11 | 9LODGPqZKgzgvkMrcoLpenJ9avQmDHUfG+EQQ37minpfZRfhT1YhfPMlWxUt/19t 12 | arjpSjtkAs74SJSJFG0UWh0LpafwVTdpokY2NXdJiaINkiqXFWJAOrc1vY7LHLmg 13 | 041iz+L+3QNJ+nSaZKc9YFemfeBhl6znN0Do3JIueaP5BsZcJjDc662kllxlj5fJ 14 | euLpuyWye2plU5yLk8QOSpPowtKRbthtgTt9Tpg7K2/tDGPXOKyS3anXZRQLi+5W 15 | wIGAIeOimunGVSW7m4UD8j2HJE1HxutZs8AvQ4BzkEJPUwQa03vEabuSzxkoWDk+ 16 | hdu55KL9CkepUGbwd1ygIFT9LFewqnLqjLVuPPGidm/UekwoKZMWCmcCAwEAAaAA 17 | MA0GCSqGSIb3DQEBCwUAA4ICAQBMzWOfKbrFFmtFTtGnkl2y1JsgAQ248Bi8UvKc 18 | 3Xsbtf2ZhwlIAvPXoN6p7snmshGw+V/Y54cc/Kirl9sl1ldK83pXXemzsCw6yzcp 19 | tuthpediTWC++4SQ4p3SRx9dDe72wW+Etcl13nwjD1i29BB4IWMsAmmgdhg/nNTv 20 | CVhWemypUgbmC6Lgtu7Bjd6laHLOnTlQWQGUO2xYXR4kQ/qIPyXNy5lI5Qu/jrkS 21 | L4QJRBzC63ZT8UKnDqHjIh0LpoiXr3+Att8lCvu+Vg8oTHk6AlKoJiQ9Uw6jBbJz 22 | E7OA20cgrMT37mOUIGClurp8+ezt0ZZozDNnWJ/AD3fVxWpMpuWuGJhGCy9CU0/b 23 | X3TNlMMdhJmliCaFjfAOQa4DR8zOsjnGyT6FtKOISJXJ4o1gUPnzpQpW1PubjtnX 24 | YaJWUy+Kag2abe5EhzjN5Kgexkc2eyn0iiiQJte1uMkYIgUioLDnYDD9VFA7gbU7 25 | KFCdmFyUKFOF7rjDY/eHIxNxHz3ST5FfoON1Hfk029tqOkt+Y3E6KXD/gvaiSdUb 26 | RNe/LKJB9w/fFMicqVqgJiPPBlb7E0F5F2mLtQZD5KMSnVfIUppaKBzWOrekGcEN 27 | /nguAl/VmDV1xruEi3NxK+Q3h0j6VIrzCS2ynyQMVvrV3Wkw3O0LOdRly9aL3d/x 28 | QjzHgA== 29 | -----END CERTIFICATE REQUEST----- 30 | -------------------------------------------------------------------------------- /server/configs/full.yaml: -------------------------------------------------------------------------------- 1 | listen: localhost:9293 2 | host: 0.0.0.0 3 | port: 5050 4 | data.dir: /foo 5 | metadata.cache.max.age: 1m 6 | 7 | batch.max: 8 | messages: 10 9 | time: 1s 10 | 11 | logging: 12 | level: debug 13 | recovery: true 14 | raft: true 15 | nats: true 16 | 17 | streams: 18 | retention.max: 19 | bytes: 1024 20 | messages: 100 21 | age: 1h 22 | cleaner.interval: 1m 23 | segment.max: 24 | bytes: 64 25 | age: 1m 26 | compact: 27 | enabled: true 28 | max.goroutines: 2 29 | 30 | clustering: 31 | server.id: foo 32 | namespace: bar 33 | raft: 34 | snapshot: 35 | retain: 10 36 | threshold: 100 37 | cache.size: 5 38 | bootstrap.peers: 39 | - a 40 | - b 41 | replica: 42 | max: 43 | lag.time: 1m 44 | leader.timeout: 30s 45 | idle.wait: 2s 46 | fetch.timeout: 3s 47 | min.insync.replicas: '1' 48 | replication.max.bytes: 1024 49 | 50 | activity.stream: 51 | enabled: true 52 | publish.timeout: 1m 53 | publish.ack.policy: leader 54 | 55 | nats: 56 | embedded: true 57 | embedded.config: nats.conf 58 | servers: 59 | - nats://localhost:4222 60 | user: user 61 | password: pass 62 | 63 | cursors: 64 | stream: 65 | partitions: 2 66 | replication.factor: 3 67 | auto.pause.time: 1m 68 | 69 | groups: 70 | consumer.timeout: 1m 71 | coordinator.timeout: 2m 72 | -------------------------------------------------------------------------------- /server/configs/host.yaml: -------------------------------------------------------------------------------- 1 | host: 192.168.0.1 2 | port: 4222 3 | -------------------------------------------------------------------------------- /server/configs/invalid-listen.yaml: -------------------------------------------------------------------------------- 1 | listen: '192.168.0.1:4222x' 2 | -------------------------------------------------------------------------------- /server/configs/listen-host.yaml: -------------------------------------------------------------------------------- 1 | listen: '192.168.0.1:4222' 2 | host: 'my-host' 3 | port: '4333' 4 | -------------------------------------------------------------------------------- /server/configs/listen.yaml: -------------------------------------------------------------------------------- 1 | listen: '192.168.0.1:4222' 2 | -------------------------------------------------------------------------------- /server/configs/nats-auth.yaml: -------------------------------------------------------------------------------- 1 | nats: 2 | user: "admin" 3 | password: "password" 4 | 5 | -------------------------------------------------------------------------------- /server/configs/simple.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | host: 0.0.0.0 3 | port: 5050 4 | logging: 5 | level: debug 6 | recovery: true 7 | streams: 8 | retention: 9 | max: 10 | bytes: 1024 11 | -------------------------------------------------------------------------------- /server/configs/tls-authz.yaml: -------------------------------------------------------------------------------- 1 | host: 0.0.0.0 2 | port: 5050 3 | tls: 4 | key: ./configs/certs/server/server-key.pem 5 | cert: ./configs/certs/server/server-cert.pem 6 | client.auth.enabled: true 7 | client.auth.ca: ./configs/certs/ca-cert.pem 8 | 9 | client.authz.enabled: true 10 | client.authz.model: ./configs/authz/model.conf 11 | client.authz.policy: ./configs/authz/policy.csv 12 | 13 | logging.level: error 14 | clustering: 15 | server.id: a 16 | namespace: a 17 | raft.bootstrap.seed: true 18 | nats.embedded: true 19 | -------------------------------------------------------------------------------- /server/configs/tls-nats.yaml: -------------------------------------------------------------------------------- 1 | nats: 2 | servers: 3 | - nats://localhost:4222 4 | user: user 5 | password: pass 6 | tls: 7 | cert: ./configs/certs/server/server-cert.pem 8 | key: ./configs/certs/server/server-key.pem 9 | ca: ./configs/certs/ca-cert.pem 10 | -------------------------------------------------------------------------------- /server/configs/tls.yaml: -------------------------------------------------------------------------------- 1 | port: 5050 2 | tls: 3 | key: ./configs/certs/server/server-key.pem 4 | cert: ./configs/certs/server/server-cert.pem 5 | client.auth.enabled: true 6 | client.auth.ca: ./configs/certs/ca-cert.pem 7 | logging.level: error 8 | clustering.raft.bootstrap.seed: true 9 | nats.embedded: true 10 | -------------------------------------------------------------------------------- /server/configs/unknown-setting.yaml: -------------------------------------------------------------------------------- 1 | host: localhost 2 | port: 5050 3 | foo.bar.baz: qux 4 | -------------------------------------------------------------------------------- /server/encryption/handler_interface.go: -------------------------------------------------------------------------------- 1 | package encryption 2 | 3 | // Codec provides the necessary method to safely retrieve 4 | // secret encryption key to encrypt/decrypt data on server side. 5 | type Codec interface { 6 | 7 | // Seal takes the message, performs the encryption and returns 8 | // the bytes to store according to the following layout: 9 | // The first byte contains the size of the wrapped key, followed by the wrapped key and 10 | // finally the encrypted message. 11 | // 12 | // | byte 0 | byte 1 | byte 2 | ... | byte (n +1) | byte (n+2) | ... | byte (n + m + 2) | 13 | // |----------|------------|------------|----------|--------------|----------------|------|------------------| 14 | // | key size | key byte 0 | key byte 1 | ... | key byte n | message byte 0 | ... | message byte m | 15 | 16 | Seal([]byte) ([]byte, error) 17 | 18 | // Read takes the cipher text, decrypts it and returns the decrypted data. 19 | // The incoming byte array has the following structure: 20 | // The first byte indicates the size of the wrapped key. 21 | // The n next bytes contain the wrapped key itself (n is the value of the first byte). 22 | // The remaining bytes are the message itself. 23 | // 24 | // | byte 0 | byte 1 | byte 2 | ... | byte (n +1) | byte (n+2) | ... | byte (n + m + 2) | 25 | // |----------|------------|------------|----------|--------------|----------------|------|------------------| 26 | // | key size | key byte 0 | key byte 1 | ... | key byte n | message byte 0 | ... | message byte m | 27 | 28 | Read([]byte) ([]byte, error) 29 | } 30 | -------------------------------------------------------------------------------- /server/fsm_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path/filepath" 7 | "strconv" 8 | "testing" 9 | "time" 10 | 11 | lift "github.com/liftbridge-io/go-liftbridge/v2" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | // Ensure Raft FSM properly snapshots and restores state. 16 | func TestFSMSnapshotRestore(t *testing.T) { 17 | defer cleanupStorage(t) 18 | 19 | // Configure the server as a seed. 20 | s1Config := getTestConfig("a", true, 5050) 21 | s1 := runServerWithConfig(t, s1Config) 22 | defer s1.Stop() 23 | 24 | // Wait to elect self as leader. 25 | getMetadataLeader(t, 10*time.Second, s1) 26 | 27 | client, err := lift.Connect([]string{"localhost:5050"}) 28 | require.NoError(t, err) 29 | defer client.Close() 30 | 31 | // Create some streams. 32 | require.NoError(t, client.CreateStream(context.Background(), "foo", "foo")) 33 | require.NoError(t, client.CreateStream(context.Background(), "bar", "bar", lift.Partitions(3))) 34 | 35 | // Force a snapshot. 36 | future := s1.getRaft().Snapshot() 37 | require.NoError(t, future.Error()) 38 | 39 | // Restart the server. 40 | s1.Stop() 41 | s1 = runServerWithConfig(t, s1.config) 42 | defer s1.Stop() 43 | 44 | // Ensure streams are recovered. 45 | waitForPartition(t, 10*time.Second, "foo", 0, s1) 46 | waitForPartition(t, 10*time.Second, "bar", 0, s1) 47 | waitForPartition(t, 10*time.Second, "bar", 1, s1) 48 | waitForPartition(t, 10*time.Second, "bar", 2, s1) 49 | require.Len(t, s1.metadata.GetStreams(), 2) 50 | } 51 | 52 | // Ensure streams that get deleted are deleted on restart but streams that get 53 | // deleted and then recreated do not get deleted on restart. 54 | func TestTombstoneStreamsOnRestart(t *testing.T) { 55 | defer cleanupStorage(t) 56 | 57 | // Configure the server as a seed. 58 | s1Config := getTestConfig("a", true, 5050) 59 | s1 := runServerWithConfig(t, s1Config) 60 | defer s1.Stop() 61 | 62 | // Wait to elect self as leader. 63 | getMetadataLeader(t, 10*time.Second, s1) 64 | 65 | client, err := lift.Connect([]string{"localhost:5050"}) 66 | require.NoError(t, err) 67 | defer client.Close() 68 | 69 | // Create some streams. 70 | require.NoError(t, client.CreateStream(context.Background(), "foo", "foo")) 71 | require.NoError(t, client.CreateStream(context.Background(), "bar", "bar")) 72 | 73 | // Delete stream foo. 74 | require.NoError(t, client.DeleteStream(context.Background(), "foo")) 75 | 76 | // Delete and then recreate stream bar. 77 | require.NoError(t, client.DeleteStream(context.Background(), "bar")) 78 | require.NoError(t, client.CreateStream(context.Background(), "bar", "bar")) 79 | 80 | // Publish some messages to bar. 81 | for i := 0; i < 2; i++ { 82 | _, err = client.Publish(context.Background(), "bar", []byte(strconv.Itoa(i))) 83 | require.NoError(t, err) 84 | } 85 | 86 | // Restart the server. 87 | s1.Stop() 88 | s1.config.Port = 5051 89 | s1 = runServerWithConfig(t, s1.config) 90 | defer s1.Stop() 91 | 92 | // Wait to elect self as leader. 93 | getMetadataLeader(t, 10*time.Second, s1) 94 | waitForPartition(t, 5*time.Second, "bar", 0, s1) 95 | 96 | client, err = lift.Connect([]string{"localhost:5051"}) 97 | require.NoError(t, err) 98 | defer client.Close() 99 | 100 | // Ensure bar stream exists and has the expected messages. 101 | ch := make(chan *lift.Message) 102 | ctx, cancel := context.WithCancel(context.Background()) 103 | err = client.Subscribe(ctx, "bar", func(msg *lift.Message, err error) { 104 | require.NoError(t, err) 105 | ch <- msg 106 | }, lift.StartAtEarliestReceived()) 107 | require.NoError(t, err) 108 | for i := 0; i < 2; i++ { 109 | select { 110 | case msg := <-ch: 111 | require.Equal(t, []byte(strconv.Itoa(i)), msg.Value()) 112 | case <-time.After(5 * time.Second): 113 | t.Fatal("Did not receive expected message") 114 | } 115 | } 116 | cancel() 117 | 118 | // Ensure foo stream doesn't exist. 119 | _, err = os.Stat(filepath.Join(s1Config.DataDir, "streams", "foo")) 120 | require.True(t, os.IsNotExist(err)) 121 | } 122 | -------------------------------------------------------------------------------- /server/health/health.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "google.golang.org/grpc" 5 | "google.golang.org/grpc/health" 6 | "google.golang.org/grpc/health/grpc_health_v1" 7 | ) 8 | 9 | const serviceName = "proto.API" // taken from compiled protobuf file api.go.pb (line 793) 10 | 11 | var server = health.NewServer() 12 | 13 | // Register the health service with a gRPC server. 14 | func Register(srv *grpc.Server) { 15 | grpc_health_v1.RegisterHealthServer(srv, server) 16 | } 17 | 18 | // SetServing marks the service as healthy. 19 | func SetServing() { 20 | server.SetServingStatus(serviceName, grpc_health_v1.HealthCheckResponse_SERVING) 21 | } 22 | 23 | // SetNotServing marks the service as unhealthy. 24 | func SetNotServing() { 25 | server.SetServingStatus(serviceName, grpc_health_v1.HealthCheckResponse_NOT_SERVING) 26 | } 27 | -------------------------------------------------------------------------------- /server/protocol/Makefile: -------------------------------------------------------------------------------- 1 | go: 2 | protoc --gofast_out=plugins=grpc:. *.proto -------------------------------------------------------------------------------- /server/signal.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | "syscall" 7 | ) 8 | 9 | // handleSignals sets up a handler for SIGINT to do a graceful shutdown. 10 | func (s *Server) handleSignals() { 11 | c := make(chan os.Signal, 1) 12 | signal.Notify(c, os.Interrupt, syscall.SIGHUP) 13 | // Use a naked goroutine instead of startGoroutine because this stops the 14 | // server which would cause a deadlock. 15 | go func() { 16 | for sig := range c { 17 | switch sig { 18 | case os.Interrupt: 19 | if err := s.Stop(); err != nil { 20 | s.logger.Errorf("Error occurred shutting down server while handling interrupt: %v", err) 21 | os.Exit(1) 22 | } 23 | os.Exit(0) 24 | 25 | case syscall.SIGHUP: 26 | // Reload authz permissions from storage 27 | s.authzEnforcer.authzLock.Lock() 28 | // LoadPolicy, if fails, will likely throw panic 29 | // Casbin raise a panic if it fails to load the policy, i.e: policy file is corrupted, 30 | // Refer to issue: https://github.com/casbin/casbin/issues/640 31 | if err := s.authzEnforcer.enforcer.LoadPolicy(); err != nil { 32 | s.logger.Errorf("Error occurred while reloading authorization permissions from storage: %v", err) 33 | s.authzEnforcer.authzLock.Unlock() 34 | continue 35 | } 36 | s.authzEnforcer.authzLock.Unlock() 37 | s.logger.Info("Reloaded authorization permissions successfully") 38 | 39 | } 40 | } 41 | }() 42 | } 43 | -------------------------------------------------------------------------------- /server/version.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | // Version of the Liftbridge server. 4 | const Version = "v1.9.0" 5 | -------------------------------------------------------------------------------- /skaffold.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: skaffold/v1beta17 3 | kind: Config 4 | build: 5 | artifacts: 6 | - image: liftbridge 7 | context: . 8 | docker: 9 | dockerfile: k8s/Dockerfile.k8s 10 | - image: natsboard 11 | context: docker/dev-cluster/ 12 | docker: 13 | dockerfile: Dockerfile.natsboard 14 | tagPolicy: 15 | sha256: {} 16 | local: 17 | push: false 18 | useBuildkit: true 19 | deploy: {} 20 | 21 | profiles: 22 | - name: dev 23 | build: 24 | local: 25 | push: false 26 | deploy: 27 | kubeContext: kubernetes-admin@kind 28 | kustomize: 29 | path: ./k8s/dev 30 | - name: deploy-k8s-image 31 | build: 32 | local: 33 | push: true 34 | artifacts: 35 | - image: liftbridge-io/liftbridge-k8s 36 | context: . 37 | docker: 38 | dockerfile: k8s/Dockerfile.k8s 39 | tagPolicy: 40 | gitCommit: {} 41 | deploy: 42 | kustomize: 43 | path: ./k8s/empty 44 | -------------------------------------------------------------------------------- /website/.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | # If you would like to upload your .git directory, .gitignore file or files 11 | # from your .gitignore file, remove the corresponding line 12 | # below: 13 | .git 14 | .gitignore 15 | 16 | # Python pycache: 17 | __pycache__/ 18 | # Ignored by the build system 19 | /setup.cfg 20 | 21 | # Website artifacts (only need build directory): 22 | node_modules/ 23 | core/ 24 | i18n/ 25 | pages/ 26 | static/ 27 | versioned_docs/ 28 | versioned_sidebars/ 29 | README.md 30 | package.json 31 | sidebars.json 32 | siteConfig.js 33 | versions.json 34 | yarn.lock 35 | -------------------------------------------------------------------------------- /website/app.yaml: -------------------------------------------------------------------------------- 1 | runtime: python37 2 | 3 | handlers: 4 | - url: / 5 | static_files: build/liftbridge/index.html 6 | upload: build/liftbridge/index.html 7 | secure: always 8 | 9 | # Redirect /docs to documentation. 10 | - url: /docs(\/?) 11 | static_files: build/liftbridge/docs/overview.html 12 | upload: build/liftbridge/docs/overview.html 13 | secure: always 14 | 15 | - url: /(.*) 16 | static_files: build/liftbridge/\1 17 | upload: build/liftbridge/(.*) 18 | secure: always 19 | -------------------------------------------------------------------------------- /website/core/Footer.js: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | 3 | class Footer extends React.Component { 4 | docUrl(doc, language) { 5 | const baseUrl = this.props.config.baseUrl; 6 | const docsUrl = this.props.config.docsUrl; 7 | const docsPart = `${docsUrl ? `${docsUrl}/` : ''}`; 8 | return `${baseUrl}${docsPart}${doc}`; 9 | } 10 | 11 | pageUrl(doc, language) { 12 | const baseUrl = this.props.config.baseUrl; 13 | return baseUrl + (language ? `${language}/` : '') + doc; 14 | } 15 | 16 | render() { 17 | return ( 18 | 75 | ); 76 | } 77 | } 78 | 79 | module.exports = Footer; 80 | -------------------------------------------------------------------------------- /website/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "license": "Apache-2.0", 3 | "scripts": { 4 | "examples": "docusaurus-examples", 5 | "start": "docusaurus-start", 6 | "build": "docusaurus-build", 7 | "publish-gh-pages": "docusaurus-publish", 8 | "write-translations": "docusaurus-write-translations", 9 | "version": "docusaurus-version", 10 | "rename-version": "docusaurus-rename-version" 11 | }, 12 | "devDependencies": { 13 | "docusaurus": "^1.14.7" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /website/pages/en/acknowledgements.js: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | 3 | const CompLibrary = require('../../core/CompLibrary.js'); 4 | 5 | const Container = CompLibrary.Container; 6 | const GridBlock = CompLibrary.GridBlock; 7 | 8 | function Acknowledgements(props) { 9 | const {config: siteConfig, language = ''} = props; 10 | const {baseUrl, docsUrl} = siteConfig; 11 | const docsPart = `${docsUrl ? `${docsUrl}/` : ''}`; 12 | const docUrl = doc => `${baseUrl}${docsPart}${doc}`; 13 | 14 | const supportLinks = [ 15 | { 16 | content: `Learn more using the [documentation on this site.](${docUrl( 17 | 'overview.html', 18 | )})`, 19 | title: 'Browse Docs', 20 | }, 21 | { 22 | content: 'You can follow and contact us on Twitter.', 23 | title: 'Twitter', 24 | }, 25 | { 26 | content: 'Visit the GitHub repo to browse ' + 27 | 'and submit issues. Create pull requests to contribute bug fixes and new features.', 28 | title: 'GitHub', 29 | }, 30 | ]; 31 | 32 | return ( 33 |
34 | 35 |
36 |
37 |

Acknowledgements

38 |
39 |
    40 |
  • 41 | Derek Collison and NATS team for building NATS and NATS Streaming and providing lots of inspiration. 42 |
    • 43 |
  • 44 | Travis Jeffery for building Jocko, a Go implementation 45 | of Kafka. The Liftbridge log implementation builds heavily upon the commit log 46 | from Jocko. 47 |
    • 48 |
  • 49 | Apache Kafka for inspiring large parts of the design, particularly around replication. 50 |
    • 51 |
52 |
53 | 54 |
55 | ); 56 | } 57 | 58 | module.exports = Acknowledgements; 59 | -------------------------------------------------------------------------------- /website/pages/en/help.js: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | 3 | const CompLibrary = require('../../core/CompLibrary.js'); 4 | 5 | const Container = CompLibrary.Container; 6 | const GridBlock = CompLibrary.GridBlock; 7 | 8 | function Help(props) { 9 | const {config: siteConfig, language = ''} = props; 10 | const {baseUrl, docsUrl} = siteConfig; 11 | const docsPart = `${docsUrl ? `${docsUrl}/` : ''}`; 12 | const docUrl = doc => `${baseUrl}${docsPart}${doc}`; 13 | 14 | const supportLinks = [ 15 | { 16 | content: `Learn more using the [documentation on this site.](${docUrl( 17 | 'overview.html', 18 | )})`, 19 | title: 'Browse Docs', 20 | }, 21 | { 22 | content: 'You can follow and contact us on Twitter.', 23 | title: 'Twitter', 24 | }, 25 | { 26 | content: 'Visit the GitHub repo to browse ' + 27 | 'and submit issues. Create pull requests to contribute bug fixes and new features.', 28 | title: 'GitHub', 29 | }, 30 | { 31 | content: '
' + 32 | '

Join the Slack Community

' + 33 | '

Want to talk with project maintainers, contributors, and the community? Join the Liftbridge Slack channel by requesting an invitation below.

' + 34 | '
' + 35 | '' + 36 | '' + 37 | '
'+ 38 | '
', 39 | title: '', 40 | }, 41 | ]; 42 | 43 | return ( 44 |
45 | 46 |
47 |
48 |

Need help?

49 |
50 |

If you need help with Liftbridge, try one of the support channels below.

51 | 52 |
53 | 54 |
55 | ); 56 | } 57 | 58 | module.exports = Help; 59 | -------------------------------------------------------------------------------- /website/sidebars.json: -------------------------------------------------------------------------------- 1 | { 2 | "docs": { 3 | "Overview": [ 4 | "overview", 5 | "faq", 6 | "concepts", 7 | "quick-start", 8 | "feature-comparison", 9 | "roadmap" 10 | ], 11 | "Configuration": [ 12 | "configuration", 13 | "ha-and-consistency-configuration", 14 | "scalability-configuration", 15 | "embedded-nats", 16 | "authentication-authorization" 17 | ], 18 | "Deployment": [ 19 | "deployment" 20 | ], 21 | "Developing With Liftbridge": [ 22 | "activity", 23 | "pausing-streams", 24 | "consumer-groups", 25 | "cursors" 26 | ], 27 | "Technical Deep Dive": [ 28 | "replication-protocol", 29 | "envelope-protocol" 30 | ], 31 | "Client Libraries": [ 32 | "clients", 33 | "client-implementation" 34 | ] 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /website/siteConfig.js: -------------------------------------------------------------------------------- 1 | // See https://docusaurus.io/docs/site-config for all the possible 2 | // site configuration options. 3 | 4 | // List of projects/orgs using your project for the users page. 5 | const users = [ 6 | { 7 | caption: 'User1', 8 | // You will need to prepend the image path with your baseUrl 9 | // if it is not '/', like: '/test-site/img/image.jpg'. 10 | image: '/img/undraw_open_source.svg', 11 | infoLink: 'https://www.facebook.com', 12 | pinned: true, 13 | }, 14 | ]; 15 | 16 | const siteConfig = { 17 | title: 'LIFTBRIDGE', // Title for your website. 18 | tagline: 'Lightweight, fault-tolerant message streams', 19 | url: 'https://liftbridge-io.github.io', // Your website URL 20 | baseUrl: '/', // Base URL for your project */ 21 | // For github.io type URLs, you would set the url and baseUrl like: 22 | // url: 'https://facebook.github.io', 23 | // baseUrl: '/test-site/', 24 | 25 | // Used for publishing and more 26 | projectName: 'liftbridge', 27 | organizationName: 'liftbridge-io', 28 | // For top-level user or org sites, the organization is still the same. 29 | // e.g., for the https://JoelMarcey.github.io site, it would be set like... 30 | // organizationName: 'JoelMarcey' 31 | 32 | // For no header links in the top nav bar -> headerLinks: [], 33 | headerLinks: [ 34 | {href: 'https://github.com/liftbridge-io/liftbridge', label: 'GitHub'}, 35 | {doc: 'overview', label: 'Docs'}, 36 | {href: 'https://github.com/liftbridge-io/liftbridge/releases', label: 'Download'}, 37 | {href: 'https://bravenewgeek.com/category/liftbridge/', label: 'Blog'}, 38 | {page: 'help', label: 'Help'}, 39 | ], 40 | 41 | // If you have users set above, you add it here: 42 | //users, 43 | 44 | /* path to images for header/footer */ 45 | headerIcon: 'img/liftbridge_icon.png', 46 | footerIcon: 'img/liftbridge_icon.png', 47 | favicon: 'img/favicon.png', 48 | 49 | /* Colors for website */ 50 | colors: { 51 | primaryColor: '#3B4763', 52 | secondaryColor: '#323859', 53 | }, 54 | 55 | /* Custom fonts for website */ 56 | /* 57 | fonts: { 58 | myFont: [ 59 | "Times New Roman", 60 | "Serif" 61 | ], 62 | myOtherFont: [ 63 | "-apple-system", 64 | "system-ui" 65 | ] 66 | }, 67 | */ 68 | 69 | // Add Algolia search. 70 | algolia: { 71 | apiKey: '81e2fdf38ab8cb38f8fbeb2a7c9939a7', 72 | indexName: 'liftbridge', 73 | }, 74 | 75 | // This copyright info is used in /core/Footer.js and blog RSS/Atom feeds. 76 | copyright: `Copyright © ${new Date().getFullYear()} Tyler Treat`, 77 | 78 | highlight: { 79 | // Highlight.js theme to use for syntax highlighting in code blocks. 80 | theme: 'default', 81 | }, 82 | 83 | // Add custom scripts here that would be placed in