├── ee ├── .gitignore ├── scripts │ ├── gqlgen-control.sh │ └── README.md ├── cloud │ ├── deployments │ │ ├── helm │ │ │ ├── web_values.yaml │ │ │ ├── web │ │ │ │ ├── Chart.yaml │ │ │ │ ├── values.yaml │ │ │ │ └── templates │ │ │ │ │ └── service.yaml │ │ │ ├── backend │ │ │ │ ├── Chart.yaml │ │ │ │ ├── templates │ │ │ │ │ ├── ctrl-server-service.yaml │ │ │ │ │ └── data-server-service.yaml │ │ │ │ └── values.yaml │ │ │ ├── README.md │ │ │ └── backend_values.yaml │ │ ├── kube │ │ │ ├── jupyterhub │ │ │ │ ├── Dockerfile │ │ │ │ └── deploy.sh │ │ │ ├── cert-manager │ │ │ │ ├── cluster-issuer-staging.yaml │ │ │ │ └── cluster-issuer-prod.yaml │ │ │ ├── billing │ │ │ │ ├── billing-pod.yaml │ │ │ │ └── billing-cronjob.yaml │ │ │ └── migrations │ │ │ │ └── migrate.yaml │ │ └── gitlab │ │ │ └── admin-service-account.yaml │ └── README.md ├── server │ └── control │ │ ├── schema │ │ ├── base.graphql │ │ ├── billing_method.graphql │ │ ├── billed_resources.graphql │ │ ├── billing_plans.graphql │ │ └── billing_info.graphql │ │ ├── README.md │ │ └── gql │ │ └── uuid.go ├── services │ ├── bi │ │ └── README.md │ └── billing │ │ ├── README.md │ │ └── run.go ├── build │ ├── web │ │ └── Dockerfile │ └── backend │ │ └── Dockerfile ├── config │ ├── payments.yaml │ └── README.md ├── cmd │ └── beneath │ │ ├── dependencies │ │ └── ee_control_server.go │ │ └── README.md ├── migrations │ ├── migrations.go │ ├── 004_nullable_billing_method.go │ ├── 006_rm_entity_name.go │ ├── 008_billing_plan_base_price.go │ ├── 007_billing_floats.go │ ├── 009_billing_plan_multiple_users.go │ ├── 011_billing_plan_remove_private_projects.go │ ├── 012_billing_and_invoice_times.go │ └── 003_tax_info.go └── pkg │ └── paymentsutil │ └── paymentsutil.go ├── CODEOWNERS ├── clients ├── python │ ├── runtime.txt │ ├── docs │ │ ├── .gitignore │ │ ├── _static │ │ │ └── .gitkeep │ │ ├── table.rst │ │ ├── client.rst │ │ ├── cursor.rst │ │ ├── consumer.rst │ │ ├── misc │ │ │ └── index.rst │ │ ├── checkpointer.rst │ │ ├── instance.rst │ │ ├── job.rst │ │ ├── errors.rst │ │ ├── easy.rst │ │ ├── pipeline.rst │ │ └── index.rst │ ├── beneath │ │ ├── admin │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ └── secrets.py │ │ ├── proto │ │ │ └── __init__.py │ │ ├── beam │ │ │ ├── __init__.py │ │ │ └── read_from_beneath.py │ │ └── pipeline │ │ │ └── __init__.py │ ├── poetry.toml │ ├── Makefile │ ├── .flake8 │ ├── netlify.toml │ ├── tools │ │ ├── pypi-publish.sh │ │ ├── dev-jupyter.sh │ │ └── proto-build.sh │ └── .vscode │ │ └── settings.json ├── js │ ├── .gitignore │ ├── src │ │ ├── version.ts │ │ ├── index.ts │ │ └── types.ts │ ├── .eslintignore │ ├── netlify.toml │ ├── typedoc.json │ ├── jest.config.js │ ├── .prettierrc.js │ ├── tsconfig.json │ └── .eslintrc.js ├── java │ ├── settings.gradle │ ├── .gitignore │ ├── gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ ├── src │ │ └── main │ │ │ ├── graphql │ │ │ └── dev │ │ │ │ └── beneath │ │ │ │ └── client │ │ │ │ ├── CompileSchema.graphql │ │ │ │ ├── CreateTableInstance.graphql │ │ │ │ ├── CreateProject.graphql │ │ │ │ └── ProjectByOrganizationAndName.graphql │ │ │ ├── java │ │ │ └── dev │ │ │ │ └── beneath │ │ │ │ └── client │ │ │ │ ├── admin │ │ │ │ ├── GraphQLException.java │ │ │ │ ├── BaseResource.java │ │ │ │ └── AdminClient.java │ │ │ │ ├── AuthenticationException.java │ │ │ │ ├── InstanceIdAndRecordPb.java │ │ │ │ ├── InstanceRecordAndSize.java │ │ │ │ ├── CheckpointedState.java │ │ │ │ └── utils │ │ │ │ ├── JsonUtils.java │ │ │ │ ├── TableIdentifier.java │ │ │ │ └── SubscriptionIdentifier.java │ │ │ └── resources │ │ │ └── log4j.properties │ ├── tools │ │ └── get-graphql-schema.sh │ ├── CONTRIBUTING.md │ └── README.md ├── js-react │ ├── .gitignore │ ├── .eslintignore │ ├── netlify.toml │ ├── src │ │ ├── index.test.ts │ │ ├── shared.ts │ │ └── index.ts │ ├── typedoc.json │ ├── jest.config.js │ ├── .prettierrc.js │ ├── tsconfig.json │ └── .eslintrc.js └── README.md ├── web ├── .gitignore ├── public │ ├── robots.txt │ ├── favicon.ico │ └── assets │ │ └── favicon │ │ ├── apple-icon.png │ │ ├── favicon-16x16.png │ │ ├── favicon-32x32.png │ │ ├── favicon-96x96.png │ │ ├── ms-icon-70x70.png │ │ ├── apple-icon-57x57.png │ │ ├── apple-icon-60x60.png │ │ ├── apple-icon-72x72.png │ │ ├── apple-icon-76x76.png │ │ ├── ms-icon-144x144.png │ │ ├── ms-icon-150x150.png │ │ ├── ms-icon-310x310.png │ │ ├── android-icon-36x36.png │ │ ├── android-icon-48x48.png │ │ ├── android-icon-72x72.png │ │ ├── android-icon-96x96.png │ │ ├── apple-icon-114x114.png │ │ ├── apple-icon-120x120.png │ │ ├── apple-icon-144x144.png │ │ ├── apple-icon-152x152.png │ │ ├── apple-icon-180x180.png │ │ ├── android-icon-144x144.png │ │ ├── android-icon-192x192.png │ │ ├── apple-icon-precomposed.png │ │ ├── browserconfig.xml │ │ └── manifest.json ├── .eslintignore ├── next-env.d.ts ├── nodemon.json ├── ee │ ├── apollo │ │ ├── possibleTypes.json │ │ ├── queries │ │ │ ├── billingMethod.ts │ │ │ └── billingPlan.ts │ │ ├── typePolicies.ts │ │ └── types │ │ │ └── BillingMethods.ts │ ├── apollo.config.js │ └── scripts │ │ └── generateApolloTypes.sh ├── lib │ ├── names.tsx │ └── connection.ts ├── components │ ├── table │ │ └── types.ts │ ├── formik │ │ └── index.ts │ ├── ErrorNote.tsx │ ├── Loading.tsx │ ├── organization │ │ ├── ViewSecurity.tsx │ │ └── ViewSecrets.tsx │ ├── VSpace.tsx │ ├── forms │ │ └── FieldError.tsx │ ├── icons │ │ └── Github.tsx │ ├── console │ │ └── tiles │ │ │ └── LoadingTile.tsx │ └── service │ │ └── ViewUsage.tsx ├── .prettierrc.json ├── apollo │ ├── queries │ │ ├── local │ │ │ ├── token.ts │ │ │ └── records.ts │ │ └── user.ts │ ├── types │ │ ├── global.d.ts │ │ ├── AID.ts │ │ ├── Token.ts │ │ ├── RevokeUserSecret.ts │ │ ├── RevokeServiceSecret.ts │ │ ├── DeleteTableInstance.ts │ │ ├── CreateRecords.ts │ │ ├── CompileSchema.ts │ │ ├── AuthTicketByID.ts │ │ ├── SecretsForService.ts │ │ ├── SecretsForUser.ts │ │ ├── UpdateAuthTicket.ts │ │ ├── ProjectMembers.ts │ │ ├── RegisterUserConsent.ts │ │ ├── GetUsage.ts │ │ └── GetTableUsage.ts │ └── local-schemas │ │ └── token.ts ├── pages │ ├── 404.tsx │ ├── - │ │ ├── sql.tsx │ │ ├── auth │ │ │ ├── index.tsx │ │ │ └── ticket │ │ │ │ └── success.tsx │ │ └── create │ │ │ └── project.tsx │ └── _error.tsx ├── apollo.config.js ├── tsconfig.server.json ├── hooks │ ├── useToken.tsx │ └── useMe.tsx ├── scripts │ └── generateApolloTypes.sh ├── .babelrc ├── next.config.js └── .eslintrc.js ├── examples ├── cdc │ ├── fanout │ │ ├── .dockerignore │ │ ├── .gitignore │ │ ├── poetry.toml │ │ ├── .env.example │ │ ├── README.md │ │ ├── Dockerfile │ │ ├── pyproject.toml │ │ └── test │ │ │ └── test_schemas.py │ └── generator │ │ ├── .gitignore │ │ ├── gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ │ ├── settings.gradle │ │ ├── .env.example │ │ ├── Dockerfile │ │ ├── README.md │ │ └── src │ │ └── main │ │ └── java │ │ └── dev │ │ └── beneath │ │ └── cdc │ │ └── postgres │ │ └── App.java ├── financial-reference-data │ ├── .gitignore │ └── pyproject.toml ├── beta │ ├── lending-club │ │ ├── loans │ │ │ ├── .dockerignore │ │ │ ├── Dockerfile │ │ │ ├── requirements.txt │ │ │ └── kube.yaml │ │ └── loans-enriched │ │ │ ├── .dockerignore │ │ │ ├── model.pkl │ │ │ ├── Dockerfile │ │ │ ├── requirements.txt │ │ │ └── kube.yaml │ ├── reddit-sentiment │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ ├── requirements.txt │ │ └── coronavirus-posts.graphql │ ├── earthquake-notifications │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ └── requirements.txt │ ├── covid19 │ │ └── pyproject.toml │ ├── mock │ │ └── derive.py │ └── ethereum-known-addresses │ │ └── known-addresses.graphql ├── wallstreetbets-analytics │ ├── explore │ │ ├── .gitignore │ │ └── pyproject.toml │ ├── stock-prices │ │ ├── .gitignore │ │ └── pyproject.toml │ ├── scores │ │ ├── schemas │ │ │ ├── post_score.graphql │ │ │ └── comment_score.graphql │ │ ├── Dockerfile │ │ └── pyproject.toml │ ├── aggregates │ │ ├── Dockerfile │ │ ├── pyproject.toml │ │ ├── r-wallstreetbets-stock-metrics-24h.yaml │ │ └── config.py │ ├── sentiment │ │ ├── Dockerfile │ │ └── pyproject.toml │ └── stock-mentions │ │ ├── Dockerfile │ │ ├── pyproject.toml │ │ └── blacklist.py ├── clock-react │ ├── .gitignore │ ├── src │ │ ├── index.js │ │ └── App.js │ ├── public │ │ └── index.html │ └── package.json ├── earthquakes │ ├── README.md │ ├── schemas │ │ └── earthquake.graphql │ ├── Dockerfile │ ├── pyproject.toml │ ├── main.py │ └── kube.yaml ├── README.md ├── reddit │ ├── reddit_app_form.png │ ├── reddit_credentials.png │ ├── Dockerfile │ ├── schemas │ │ ├── comment.graphql │ │ └── post.graphql │ ├── pyproject.toml │ └── deployments │ │ └── README.md ├── clock │ ├── Dockerfile │ ├── README.md │ ├── pyproject.toml │ └── kube.yaml └── ethereum-blocks │ ├── Dockerfile │ ├── pyproject.toml │ ├── README.md │ └── kube.yaml ├── CONTRIBUTING.md ├── scripts ├── gqlgen-control.sh ├── README.md └── proto-build.sh ├── assets ├── README.md └── logo │ ├── banner-text-background.png │ ├── square-logo-background.png │ ├── square-text-background.png │ ├── square-letter-background.png │ ├── banner-icon-text-background.png │ └── square-icon-text-background.png ├── yarn.lock ├── pkg ├── codec │ └── ext │ │ └── tuple │ │ └── testdata │ │ └── tuples.golden ├── README.md ├── queryparse │ └── parse.go ├── bytesutil │ ├── bytesutil_test.go │ └── bytesutil.go ├── ws │ └── ws.go ├── schemalang │ ├── transpilers │ │ └── avro_test.go │ ├── avro │ │ └── avro.go │ └── indexes_test.go ├── ctxutil │ └── ctxutil.go ├── grpcutil │ └── grpcutil.go ├── envutil │ └── config.go └── mathutil │ └── mathutil.go ├── services ├── project │ ├── README.md │ └── starter.go ├── table │ ├── util.go │ └── README.md ├── middleware │ ├── README.md │ └── middleware.go ├── data │ └── README.md ├── permissions │ └── README.md ├── user │ ├── README.md │ └── user.go ├── organization │ └── README.md ├── service │ └── README.md ├── usage │ └── README.md └── secret │ ├── README.md │ └── events.go ├── .gitlab-ci.yml ├── docs ├── misc │ ├── _index.md │ ├── reference.md │ └── contributing.md ├── concepts │ └── _index.md ├── usage │ └── _index.md ├── reading-writing-data │ └── _index.md └── README.md ├── server ├── README.md ├── control │ ├── schema │ │ └── base.graphql │ ├── resolver │ │ ├── stream_index.go │ │ └── stream_instance.go │ └── gql │ │ └── uuid.go └── data │ ├── grpc │ ├── read.go │ └── ping.go │ └── README.md ├── models └── validate.go ├── test ├── testdata │ └── foobar.graphql ├── integration │ └── util.go └── README.md ├── migrations ├── migrations.go ├── 020_tax_info.go ├── 019_billing_method.go ├── 028_rm_entity_name.go ├── 030_billing_floats.go ├── 006_secret.go ├── 027_billing_plan_quotas.go ├── 022_nullable_billing_method.go ├── 031_billing_plan_base_price.go ├── 033_billing_plan_multiple_users.go ├── 042_billing_plan_remove_private_projects.go ├── README.md ├── 045_streams_meta.go ├── 021_master_users.go ├── 041_featured_projects.go ├── 015_stream_retention.go ├── 044_private_project_default.go ├── 009_organization_unique.go ├── 038_instance_versions.go ├── 010_project_display_name.go ├── 024_user_org_perms_create.go ├── 025_organization_quotas.go ├── 029_prepaid_quotas.go ├── 034_user_consent.go ├── 008_service_dates.go ├── 012_stream_instances_counts.go ├── 048_auth_ticket.go └── 040_different_stream_schema_types.go ├── cmd ├── README.md └── beneath │ ├── dependencies │ ├── hub.go │ ├── redis.go │ ├── data_server.go │ └── mq.go │ └── cli │ └── cli.go ├── infra ├── engine │ ├── driver │ │ ├── bigquery │ │ │ └── proto │ │ │ │ └── bigquery.proto │ │ ├── bigtable │ │ │ ├── proto │ │ │ │ └── cursor.proto │ │ │ └── bigtable_test.go │ │ └── mock │ │ │ ├── util.go │ │ │ └── mock.go │ ├── proto │ │ ├── taskqueue.proto │ │ ├── quota.proto │ │ └── engine.proto │ └── README.md ├── db │ └── util.go ├── redis │ └── redis.go └── README.md ├── doc.go ├── licenses └── CC-BY-SA-4.0.txt ├── .gitignore ├── .github ├── workflows │ └── mirror.yml └── ISSUE_TEMPLATE │ ├── refactor.md │ └── question.md ├── config └── drivers.yaml └── LICENSE /ee/.gitignore: -------------------------------------------------------------------------------- 1 | config/.* -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @_bem @greenep12 2 | -------------------------------------------------------------------------------- /clients/python/runtime.txt: -------------------------------------------------------------------------------- 1 | 3.7 -------------------------------------------------------------------------------- /web/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /clients/python/docs/.gitignore: -------------------------------------------------------------------------------- 1 | _build -------------------------------------------------------------------------------- /clients/python/docs/_static/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /clients/python/beneath/admin/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /clients/python/beneath/proto/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/cdc/fanout/.dockerignore: -------------------------------------------------------------------------------- 1 | .venv 2 | -------------------------------------------------------------------------------- /examples/financial-reference-data/.gitignore: -------------------------------------------------------------------------------- 1 | data -------------------------------------------------------------------------------- /clients/js/.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | docs 3 | node_modules -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Please see `contributing/README.md` 2 | -------------------------------------------------------------------------------- /clients/java/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'beneath' -------------------------------------------------------------------------------- /examples/beta/lending-club/loans/.dockerignore: -------------------------------------------------------------------------------- 1 | .venv 2 | -------------------------------------------------------------------------------- /examples/beta/reddit-sentiment/.dockerignore: -------------------------------------------------------------------------------- 1 | .venv 2 | -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/explore/.gitignore: -------------------------------------------------------------------------------- 1 | data -------------------------------------------------------------------------------- /clients/js-react/.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | docs 3 | node_modules 4 | -------------------------------------------------------------------------------- /examples/beta/earthquake-notifications/.dockerignore: -------------------------------------------------------------------------------- 1 | .venv 2 | -------------------------------------------------------------------------------- /examples/cdc/fanout/.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | .env.dev 3 | .env.test -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/stock-prices/.gitignore: -------------------------------------------------------------------------------- 1 | data -------------------------------------------------------------------------------- /web/public/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: /-/auth 3 | -------------------------------------------------------------------------------- /examples/beta/lending-club/loans-enriched/.dockerignore: -------------------------------------------------------------------------------- 1 | .venv 2 | -------------------------------------------------------------------------------- /clients/js/src/version.ts: -------------------------------------------------------------------------------- 1 | export const PACKAGE_VERSION = "1.2.0"; 2 | -------------------------------------------------------------------------------- /examples/cdc/fanout/poetry.toml: -------------------------------------------------------------------------------- 1 | [virtualenvs] 2 | in-project = true 3 | -------------------------------------------------------------------------------- /examples/clock-react/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | build 3 | node_modules 4 | -------------------------------------------------------------------------------- /clients/js/.eslintignore: -------------------------------------------------------------------------------- 1 | **/*.js 2 | node_modules 3 | dist 4 | coverage 5 | -------------------------------------------------------------------------------- /web/.eslintignore: -------------------------------------------------------------------------------- 1 | **/*.js 2 | node_modules 3 | dist 4 | coverage 5 | .next 6 | -------------------------------------------------------------------------------- /clients/js-react/.eslintignore: -------------------------------------------------------------------------------- 1 | **/*.js 2 | node_modules 3 | dist 4 | coverage 5 | -------------------------------------------------------------------------------- /clients/js/netlify.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | command = "yarn docs" 3 | publish = "docs" 4 | -------------------------------------------------------------------------------- /clients/python/poetry.toml: -------------------------------------------------------------------------------- 1 | [virtualenvs] 2 | create = true 3 | in-project = true 4 | -------------------------------------------------------------------------------- /examples/cdc/fanout/.env.example: -------------------------------------------------------------------------------- 1 | BENEATH_ENV= 2 | BENEATH_SECRET= 3 | DATABASE_DBNAME= -------------------------------------------------------------------------------- /clients/js-react/netlify.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | command = "yarn docs" 3 | publish = "docs" 4 | -------------------------------------------------------------------------------- /examples/earthquakes/README.md: -------------------------------------------------------------------------------- 1 | Run with: 2 | 3 | ```bash 4 | poetry run python main.py 5 | ``` -------------------------------------------------------------------------------- /web/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/favicon.ico -------------------------------------------------------------------------------- /scripts/gqlgen-control.sh: -------------------------------------------------------------------------------- 1 | go run github.com/99designs/gqlgen --config server/control/gql/gqlgen.yml -------------------------------------------------------------------------------- /web/next-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | /// 3 | -------------------------------------------------------------------------------- /clients/js-react/src/index.test.ts: -------------------------------------------------------------------------------- 1 | test('empty', () => { 2 | // expect(sum(1, 2)).toBe(3); 3 | }); 4 | -------------------------------------------------------------------------------- /clients/python/docs/table.rst: -------------------------------------------------------------------------------- 1 | Table 2 | ====== 3 | 4 | .. autoclass:: beneath.Table() 5 | :members: 6 | -------------------------------------------------------------------------------- /ee/scripts/gqlgen-control.sh: -------------------------------------------------------------------------------- 1 | go run github.com/99designs/gqlgen --config ee/server/control/gql/gqlgen.yml -------------------------------------------------------------------------------- /assets/README.md: -------------------------------------------------------------------------------- 1 | # `assets/` 2 | 3 | This directory contains assorted assets such as logos, images, etc. 4 | -------------------------------------------------------------------------------- /clients/java/.gitignore: -------------------------------------------------------------------------------- 1 | .project 2 | .settings 3 | .classpath 4 | .gradle 5 | build 6 | bin 7 | gradle.properties -------------------------------------------------------------------------------- /clients/python/docs/client.rst: -------------------------------------------------------------------------------- 1 | Client 2 | ====== 3 | 4 | .. autoclass:: beneath.Client 5 | :members: 6 | -------------------------------------------------------------------------------- /clients/python/docs/cursor.rst: -------------------------------------------------------------------------------- 1 | Cursor 2 | ====== 3 | 4 | .. autoclass:: beneath.Cursor() 5 | :members: 6 | -------------------------------------------------------------------------------- /ee/cloud/deployments/helm/web_values.yaml: -------------------------------------------------------------------------------- 1 | ingress: 2 | hosts: 3 | - beneath.dev 4 | - www.beneath.dev 5 | -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | -------------------------------------------------------------------------------- /clients/js/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "entryPoints": ["./src/index.ts"], 3 | "out": "docs/", 4 | "theme": "minimal" 5 | } -------------------------------------------------------------------------------- /clients/python/Makefile: -------------------------------------------------------------------------------- 1 | docs: 2 | poetry install 3 | poetry run sphinx-build -W docs/ docs/_build 4 | .PHONY: docs 5 | -------------------------------------------------------------------------------- /clients/python/docs/consumer.rst: -------------------------------------------------------------------------------- 1 | Consumer 2 | ======== 3 | 4 | .. autoclass:: beneath.Consumer() 5 | :members: 6 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # `examples/` 2 | 3 | This folder contains example schemas, apps and tutorials for using Beneath. 4 | -------------------------------------------------------------------------------- /scripts/README.md: -------------------------------------------------------------------------------- 1 | # `scripts/` 2 | 3 | This folder contains scripts used during development (typically generators) 4 | -------------------------------------------------------------------------------- /clients/js-react/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "entryPoints": ["./src/index.ts"], 3 | "out": "docs/", 4 | "theme": "minimal" 5 | } -------------------------------------------------------------------------------- /clients/python/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 100 3 | extend-ignore = E203 4 | exclude = .git,__pycache__,docs,proto 5 | -------------------------------------------------------------------------------- /ee/cloud/deployments/kube/jupyterhub/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM jupyterhub/k8s-hub:v0.4 2 | # RUN pip3 install jupyterhub-hashauthenticator -------------------------------------------------------------------------------- /examples/cdc/generator/.gitignore: -------------------------------------------------------------------------------- 1 | .project 2 | .settings 3 | .classpath 4 | .gradle 5 | .env.dev 6 | .env.test 7 | build 8 | bin -------------------------------------------------------------------------------- /examples/reddit/reddit_app_form.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/examples/reddit/reddit_app_form.png -------------------------------------------------------------------------------- /assets/logo/banner-text-background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/assets/logo/banner-text-background.png -------------------------------------------------------------------------------- /assets/logo/square-logo-background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/assets/logo/square-logo-background.png -------------------------------------------------------------------------------- /assets/logo/square-text-background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/assets/logo/square-text-background.png -------------------------------------------------------------------------------- /clients/python/docs/misc/index.rst: -------------------------------------------------------------------------------- 1 | Miscellaneous 2 | ============= 3 | 4 | .. toctree:: 5 | :maxdepth: 3 6 | 7 | asyncio 8 | -------------------------------------------------------------------------------- /examples/reddit/reddit_credentials.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/examples/reddit/reddit_credentials.png -------------------------------------------------------------------------------- /assets/logo/square-letter-background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/assets/logo/square-letter-background.png -------------------------------------------------------------------------------- /clients/python/docs/checkpointer.rst: -------------------------------------------------------------------------------- 1 | Checkpointer 2 | ============ 3 | 4 | .. autoclass:: beneath.Checkpointer() 5 | :members: 6 | -------------------------------------------------------------------------------- /clients/python/docs/instance.rst: -------------------------------------------------------------------------------- 1 | TableInstance 2 | ============== 3 | 4 | .. autoclass:: beneath.TableInstance() 5 | :members: 6 | -------------------------------------------------------------------------------- /clients/python/netlify.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | publish = "docs/_build" 3 | command = "pip3 install -q poetry && poetry run make docs" 4 | -------------------------------------------------------------------------------- /ee/scripts/README.md: -------------------------------------------------------------------------------- 1 | # `scripts/` 2 | 3 | This folder contains scripts used during development of `ee` code (typically generators) 4 | -------------------------------------------------------------------------------- /pkg/codec/ext/tuple/testdata/tuples.golden: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/pkg/codec/ext/tuple/testdata/tuples.golden -------------------------------------------------------------------------------- /services/project/README.md: -------------------------------------------------------------------------------- 1 | # `services/project/` 2 | 3 | This service manages the `Project` model (also see `models/project.go`). 4 | -------------------------------------------------------------------------------- /web/public/assets/favicon/apple-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/apple-icon.png -------------------------------------------------------------------------------- /assets/logo/banner-icon-text-background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/assets/logo/banner-icon-text-background.png -------------------------------------------------------------------------------- /assets/logo/square-icon-text-background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/assets/logo/square-icon-text-background.png -------------------------------------------------------------------------------- /ee/cloud/README.md: -------------------------------------------------------------------------------- 1 | # `ee/cloud/` 2 | 3 | This folder contains files related only to the managed Beneath instance running at `beneath.dev`. 4 | -------------------------------------------------------------------------------- /web/nodemon.json: -------------------------------------------------------------------------------- 1 | { 2 | "watch": ["server"], 3 | "exec": "ts-node --project tsconfig.server.json server.ts", 4 | "ext": "js ts" 5 | } 6 | -------------------------------------------------------------------------------- /web/public/assets/favicon/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/favicon-16x16.png -------------------------------------------------------------------------------- /web/public/assets/favicon/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/favicon-32x32.png -------------------------------------------------------------------------------- /web/public/assets/favicon/favicon-96x96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/favicon-96x96.png -------------------------------------------------------------------------------- /web/public/assets/favicon/ms-icon-70x70.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/ms-icon-70x70.png -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - build 3 | - deploy 4 | 5 | include: 6 | - local: /ee/.gitlab-ci.yml 7 | - local: /ee/cloud/.gitlab-ci.yml 8 | -------------------------------------------------------------------------------- /clients/java/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/clients/java/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /docs/misc/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | menu: 3 | docs: 4 | identifier: misc 5 | name: Miscellaneous 6 | weight: 900 7 | weight: 900 8 | --- 9 | -------------------------------------------------------------------------------- /web/public/assets/favicon/apple-icon-57x57.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/apple-icon-57x57.png -------------------------------------------------------------------------------- /web/public/assets/favicon/apple-icon-60x60.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/apple-icon-60x60.png -------------------------------------------------------------------------------- /web/public/assets/favicon/apple-icon-72x72.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/apple-icon-72x72.png -------------------------------------------------------------------------------- /web/public/assets/favicon/apple-icon-76x76.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/apple-icon-76x76.png -------------------------------------------------------------------------------- /web/public/assets/favicon/ms-icon-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/ms-icon-144x144.png -------------------------------------------------------------------------------- /web/public/assets/favicon/ms-icon-150x150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/ms-icon-150x150.png -------------------------------------------------------------------------------- /web/public/assets/favicon/ms-icon-310x310.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/ms-icon-310x310.png -------------------------------------------------------------------------------- /docs/concepts/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | menu: 3 | docs: 4 | identifier: concepts 5 | name: Concepts 6 | weight: 300 7 | weight: 300 8 | --- 9 | -------------------------------------------------------------------------------- /docs/usage/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | menu: 3 | docs: 4 | identifier: usage 5 | name: Usage & billing 6 | weight: 800 7 | weight: 800 8 | --- 9 | -------------------------------------------------------------------------------- /web/public/assets/favicon/android-icon-36x36.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/android-icon-36x36.png -------------------------------------------------------------------------------- /web/public/assets/favicon/android-icon-48x48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/android-icon-48x48.png -------------------------------------------------------------------------------- /web/public/assets/favicon/android-icon-72x72.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/android-icon-72x72.png -------------------------------------------------------------------------------- /web/public/assets/favicon/android-icon-96x96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/android-icon-96x96.png -------------------------------------------------------------------------------- /web/public/assets/favicon/apple-icon-114x114.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/apple-icon-114x114.png -------------------------------------------------------------------------------- /web/public/assets/favicon/apple-icon-120x120.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/apple-icon-120x120.png -------------------------------------------------------------------------------- /web/public/assets/favicon/apple-icon-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/apple-icon-144x144.png -------------------------------------------------------------------------------- /web/public/assets/favicon/apple-icon-152x152.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/apple-icon-152x152.png -------------------------------------------------------------------------------- /web/public/assets/favicon/apple-icon-180x180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/apple-icon-180x180.png -------------------------------------------------------------------------------- /examples/beta/lending-club/loans-enriched/model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/examples/beta/lending-club/loans-enriched/model.pkl -------------------------------------------------------------------------------- /server/README.md: -------------------------------------------------------------------------------- 1 | # `server/` 2 | 3 | This folder contains implementations of the control-plane (GraphQL) server and data-plane (REST and GRPC) server. 4 | -------------------------------------------------------------------------------- /web/public/assets/favicon/android-icon-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/android-icon-144x144.png -------------------------------------------------------------------------------- /web/public/assets/favicon/android-icon-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/android-icon-192x192.png -------------------------------------------------------------------------------- /web/public/assets/favicon/apple-icon-precomposed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/web/public/assets/favicon/apple-icon-precomposed.png -------------------------------------------------------------------------------- /clients/python/docs/job.rst: -------------------------------------------------------------------------------- 1 | Job 2 | === 3 | 4 | .. autoclass:: beneath.Job() 5 | :members: 6 | 7 | .. autoclass:: beneath.JobStatus() 8 | :members: 9 | -------------------------------------------------------------------------------- /clients/js-react/src/shared.ts: -------------------------------------------------------------------------------- 1 | export type FetchMoreOptions = { pageSize?: number }; 2 | export type FetchMoreFunction = (opts?: FetchMoreOptions) => Promise; 3 | -------------------------------------------------------------------------------- /examples/cdc/generator/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/beneath-hq/beneath/HEAD/examples/cdc/generator/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/scores/schemas/post_score.graphql: -------------------------------------------------------------------------------- 1 | type Score @schema { 2 | id: String! @key 3 | score: Int! 4 | upvote_ratio: Float! 5 | } 6 | -------------------------------------------------------------------------------- /clients/js/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | testRegex: "(/__tests__/.*|(\\.|/)(test|spec))\\.tsx?$" 5 | }; 6 | -------------------------------------------------------------------------------- /ee/cloud/deployments/helm/web/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: "1.0" 3 | description: A Helm chart for the Beneath frontend 4 | name: web 5 | version: 0.1.0 6 | -------------------------------------------------------------------------------- /examples/cdc/generator/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'cdc-generator' 2 | 3 | // use this if developing the client library 4 | // includeBuild '../../../clients/java' -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/scores/schemas/comment_score.graphql: -------------------------------------------------------------------------------- 1 | type Score @schema { 2 | id: String! @key 3 | score: Int! 4 | upvote_ratio: Float! 5 | } 6 | -------------------------------------------------------------------------------- /services/table/util.go: -------------------------------------------------------------------------------- 1 | package table 2 | 3 | func derefBool(val *bool, fallback bool) bool { 4 | if val != nil { 5 | return *val 6 | } 7 | return fallback 8 | } 9 | -------------------------------------------------------------------------------- /clients/js-react/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | testRegex: "(/__tests__/.*|(\\.|/)(test|spec))\\.tsx?$" 5 | }; 6 | -------------------------------------------------------------------------------- /ee/server/control/schema/base.graphql: -------------------------------------------------------------------------------- 1 | scalar Time 2 | scalar UUID 3 | 4 | type Query { 5 | empty: String 6 | } 7 | 8 | type Mutation { 9 | empty: String 10 | } 11 | -------------------------------------------------------------------------------- /models/validate.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import "gopkg.in/go-playground/validator.v9" 4 | 5 | // Validator used for validation of models 6 | var Validator = validator.New() 7 | -------------------------------------------------------------------------------- /clients/python/docs/errors.rst: -------------------------------------------------------------------------------- 1 | Errors 2 | ====== 3 | 4 | .. autoclass:: beneath.AuthenticationError 5 | :members: 6 | 7 | .. autoclass:: beneath.GraphQLError 8 | :members: 9 | -------------------------------------------------------------------------------- /docs/reading-writing-data/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | menu: 3 | docs: 4 | identifier: reading-writing-data 5 | name: Reading & writing data 6 | weight: 400 7 | weight: 400 8 | --- 9 | -------------------------------------------------------------------------------- /ee/cloud/deployments/helm/backend/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: "1.0" 3 | description: A Helm chart for the Beneath backend services 4 | name: backend 5 | version: 0.1.0 6 | -------------------------------------------------------------------------------- /services/middleware/README.md: -------------------------------------------------------------------------------- 1 | # `services/middleware/` 2 | 3 | This service creates and implements HTTP and GRPC server middleware for authentication, logging, rate limiting and more. 4 | -------------------------------------------------------------------------------- /clients/js-react/src/index.ts: -------------------------------------------------------------------------------- 1 | export { Client, Job, Record, TableQualifier } from "beneath"; 2 | export * from "./shared"; 3 | export * from "./useRecords"; 4 | export * from "./useWarehouse"; 5 | -------------------------------------------------------------------------------- /web/ee/apollo/possibleTypes.json: -------------------------------------------------------------------------------- 1 | {"Int":[],"Boolean":[],"Time":[],"BillingInfo":[],"UUID":[],"Query":[],"BillingPlan":[],"BillingMethod":[],"Mutation":[],"Float":[],"String":[],"ID":[],"BilledResource":[]} -------------------------------------------------------------------------------- /ee/services/bi/README.md: -------------------------------------------------------------------------------- 1 | # `ee/services/bi/` 2 | 3 | Package `bi` forwards platform data for business intelligence (BI) purposes. It's a temporary hack that we should get rid of (possible with metatables). 4 | -------------------------------------------------------------------------------- /examples/cdc/fanout/README.md: -------------------------------------------------------------------------------- 1 | The `fanout` service consumes the `raw-changes` table produced by the `generator` service and writes each change to its own schemaful table. 2 | 3 | Schema evolution is not yet supported. -------------------------------------------------------------------------------- /test/testdata/foobar.graphql: -------------------------------------------------------------------------------- 1 | " Schema comment " 2 | type FooBar @table @key(fields: ["a", "b"]) { 3 | a: String! 4 | b: Int! 5 | c: Bytes4! 6 | d: Timestamp! 7 | e: Numeric 8 | f: Bytes 9 | } 10 | -------------------------------------------------------------------------------- /web/lib/names.tsx: -------------------------------------------------------------------------------- 1 | export const toURLName = (name: string) => { 2 | return name.replace(/_/g, "-"); 3 | }; 4 | 5 | export const toBackendName = (name: string) => { 6 | return name.replace(/-/g, "_"); 7 | }; 8 | -------------------------------------------------------------------------------- /clients/java/src/main/graphql/dev/beneath/client/CompileSchema.graphql: -------------------------------------------------------------------------------- 1 | query CompileSchema($input: CompileSchemaInput!) { 2 | compileSchema(input: $input) { 3 | canonicalAvroSchema 4 | canonicalIndexes 5 | } 6 | } -------------------------------------------------------------------------------- /ee/server/control/README.md: -------------------------------------------------------------------------------- 1 | # `ee/server/control/` 2 | 3 | See `server.go` for details on how this plugs into the main control server. 4 | 5 | To regenerate the EE GraphQL resolvers, run `./ee/scripts/gqlgen-control.sh`. 6 | -------------------------------------------------------------------------------- /migrations/migrations.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/beneath-hq/beneath/pkg/migrationsutil" 5 | ) 6 | 7 | // Migrator registers and runs migrations 8 | var Migrator = migrationsutil.New("gopg_migrations") 9 | -------------------------------------------------------------------------------- /web/components/table/types.ts: -------------------------------------------------------------------------------- 1 | export interface TableInstance { 2 | tableInstanceID: string; 3 | version: number; 4 | createdOn: ControlTime; 5 | madePrimaryOn: ControlTime | null; 6 | madeFinalOn: ControlTime | null; 7 | } 8 | -------------------------------------------------------------------------------- /clients/python/docs/easy.rst: -------------------------------------------------------------------------------- 1 | Easy helpers 2 | ============ 3 | 4 | Beneath includes a handful of "easy" helpers that wrap several common steps 5 | in one package-level operation. 6 | 7 | .. automodule:: beneath.easy 8 | :members: 9 | -------------------------------------------------------------------------------- /cmd/README.md: -------------------------------------------------------------------------------- 1 | # `cmd/` 2 | 3 | The Beneath *backend* is comprised of a handful different services that can all be started with the same executable, see `cmd/beneath/` for more. 4 | 5 | The Beneath *frontend* is found and started in `web/`. 6 | -------------------------------------------------------------------------------- /ee/build/web/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:12-alpine 2 | 3 | COPY ./package.json ./yarn.lock /source/ 4 | WORKDIR /source/ 5 | RUN yarn install 6 | 7 | COPY . /source 8 | RUN yarn build-ee 9 | 10 | EXPOSE 3000 11 | CMD ["yarn", "start-ee"] 12 | -------------------------------------------------------------------------------- /cmd/beneath/dependencies/hub.go: -------------------------------------------------------------------------------- 1 | package dependencies 2 | 3 | import ( 4 | "github.com/beneath-hq/beneath/bus" 5 | "github.com/beneath-hq/beneath/cmd/beneath/cli" 6 | ) 7 | 8 | func init() { 9 | cli.AddDependency(bus.NewBus) 10 | } 11 | -------------------------------------------------------------------------------- /examples/beta/reddit-sentiment/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7 2 | WORKDIR /app 3 | COPY requirements.txt ./ 4 | RUN pip install -r requirements.txt 5 | COPY stream_submissions.py coronavirus-posts.graphql ./ 6 | CMD ["python", "stream_submissions.py"] 7 | -------------------------------------------------------------------------------- /server/control/schema/base.graphql: -------------------------------------------------------------------------------- 1 | scalar Time 2 | scalar UUID 3 | 4 | type Query { 5 | empty: String 6 | ping: String! 7 | } 8 | 9 | type Mutation { 10 | empty: String 11 | } 12 | 13 | type Subscription { 14 | empty: String 15 | } 16 | -------------------------------------------------------------------------------- /clients/js/src/index.ts: -------------------------------------------------------------------------------- 1 | export { Client, ClientOptions } from "./Client"; 2 | export { Cursor } from "./Cursor"; 3 | export { Job } from "./Job"; 4 | export { QueryLogResult, QueryIndexResult, Table, WriteResult } from "./Table"; 5 | export * from "./types"; 6 | -------------------------------------------------------------------------------- /clients/js/src/types.ts: -------------------------------------------------------------------------------- 1 | export type Record = TRecord & { 2 | "@meta": { key: string, timestamp: number } 3 | }; 4 | 5 | export type TableQualifier = string | { instanceID: string } | { organization: string, project: string, table: string }; 6 | -------------------------------------------------------------------------------- /ee/config/payments.yaml: -------------------------------------------------------------------------------- 1 | # See README.md in this folder for details 2 | 3 | payments: 4 | drivers: 5 | - driver: "anarchism" 6 | - driver: "stripecard" 7 | stripe_secret: "" 8 | - driver: "stripewire" 9 | stripe_secret: "" 10 | -------------------------------------------------------------------------------- /examples/cdc/generator/.env.example: -------------------------------------------------------------------------------- 1 | BENEATH_ENV= 2 | BENEATH_SECRET= 3 | BENEATH_USERNAME= 4 | DATABASE_HOSTNAME= 5 | DATABASE_PORT= 6 | DATABASE_USER= 7 | DATABASE_PASSWORD= 8 | DATABASE_DBNAME= 9 | DATABASE_SERVER_NAME= 10 | TABLE_INCLUDE_LIST= 11 | -------------------------------------------------------------------------------- /web/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "es5", 3 | "tabWidth": 2, 4 | "semi": true, 5 | "singleQuote": false, 6 | "printWidth": 120, 7 | "bracketSpacing": true, 8 | "jsxBracketSameLine": false, 9 | "arrowParens": "always" 10 | } 11 | -------------------------------------------------------------------------------- /web/apollo/queries/local/token.ts: -------------------------------------------------------------------------------- 1 | import gql from "graphql-tag"; 2 | 3 | export const GET_AID = gql` 4 | query AID { 5 | aid @client 6 | } 7 | `; 8 | 9 | export const GET_TOKEN = gql` 10 | query Token { 11 | token @client 12 | } 13 | `; 14 | -------------------------------------------------------------------------------- /clients/java/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /clients/js/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | trailingComma: "es5", 3 | tabWidth: 2, 4 | semi: true, 5 | singleQuote: false, 6 | printWidth: 120, 7 | bracketSpacing: true, 8 | jsxBracketSameLine: false, 9 | arrowParens: "always", 10 | }; 11 | -------------------------------------------------------------------------------- /infra/engine/driver/bigquery/proto/bigquery.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | option go_package = "proto"; 3 | package bigquery; 4 | 5 | message Cursor { 6 | string dataset = 1; 7 | string table = 2; 8 | string token = 3; 9 | string avro_schema = 4; 10 | } 11 | -------------------------------------------------------------------------------- /pkg/README.md: -------------------------------------------------------------------------------- 1 | # `pkg/` 2 | 3 | This directory contains stand-alone Go packages (i.e. that do not depend on other parts of the codebase). A good indicator of whether a module fits in here is if it could potentially be published as a stand-alone open source project. 4 | -------------------------------------------------------------------------------- /clients/js-react/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | trailingComma: "es5", 3 | tabWidth: 2, 4 | semi: true, 5 | singleQuote: false, 6 | printWidth: 120, 7 | bracketSpacing: true, 8 | jsxBracketSameLine: false, 9 | arrowParens: "always", 10 | }; 11 | -------------------------------------------------------------------------------- /clients/python/docs/pipeline.rst: -------------------------------------------------------------------------------- 1 | Pipeline 2 | ======== 3 | 4 | .. autoclass:: beneath.Pipeline 5 | :inherited-members: 6 | :members: 7 | 8 | .. autoclass:: beneath.Action 9 | :members: 10 | 11 | .. autoclass:: beneath.Strategy 12 | :members: 13 | -------------------------------------------------------------------------------- /web/apollo/types/global.d.ts: -------------------------------------------------------------------------------- 1 | // GraphQL (control) custom scalars 2 | type ControlUUID = string; 3 | type ControlTime = string; 4 | type ControlJSON = any; 5 | 6 | declare namespace NodeJS { 7 | export interface Process { 8 | browser: boolean; 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package beneath is the top-level package for Beneath, a platform for building 3 | analytics applications. 4 | 5 | For more information, visit https://beneath.dev or browse the source code at 6 | https://github.com/beneath-hq/beneath. 7 | */ 8 | package beneath 9 | -------------------------------------------------------------------------------- /examples/clock-react/src/index.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import ReactDOM from "react-dom"; 3 | import App from "./App"; 4 | 5 | ReactDOM.render( 6 | 7 | 8 | , 9 | document.getElementById("root") 10 | ); 11 | -------------------------------------------------------------------------------- /services/data/README.md: -------------------------------------------------------------------------------- 1 | # `services/data/` 2 | 3 | This service handles data-plane functionality like reading data, queuing data writes, serving subscriptions and running the background data writing worker. 4 | 5 | It's used by the `data-server` and `data-worker` startables. 6 | -------------------------------------------------------------------------------- /examples/cdc/generator/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /examples/clock/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | RUN pip install poetry==1.1.4 3 | WORKDIR /app 4 | COPY poetry.lock pyproject.toml /app/ 5 | RUN poetry config virtualenvs.create false \ 6 | && poetry install --no-dev --no-interaction --no-ansi 7 | COPY . . 8 | ENTRYPOINT ["python", "clock.py"] 9 | -------------------------------------------------------------------------------- /examples/reddit/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | RUN pip install poetry==1.1.4 3 | WORKDIR /app 4 | COPY poetry.lock pyproject.toml /app/ 5 | RUN poetry config virtualenvs.create false \ 6 | && poetry install --no-dev --no-interaction --no-ansi 7 | COPY . . 8 | ENTRYPOINT ["python", "main.py"] 9 | -------------------------------------------------------------------------------- /infra/engine/proto/taskqueue.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | option go_package = "github.com/beneath-hq/beneath/infra/engine/proto"; 3 | package engine; 4 | 5 | message BusMsg { 6 | bytes id = 1; 7 | string name = 2; 8 | int64 timestamp = 3; 9 | bytes payload = 4; 10 | } 11 | -------------------------------------------------------------------------------- /web/pages/404.tsx: -------------------------------------------------------------------------------- 1 | import { NextPage } from "next"; 2 | import React from "react"; 3 | 4 | import ErrorPage from "../components/ErrorPage"; 5 | 6 | const FourOhFourPage: NextPage = () => { 7 | return ; 8 | }; 9 | 10 | export default FourOhFourPage; 11 | -------------------------------------------------------------------------------- /ee/cloud/deployments/helm/README.md: -------------------------------------------------------------------------------- 1 | # `ee/cloud/deployments/helm/` 2 | 3 | Before committing a Helm chart in production, please: 4 | 5 | - Run the Helm linter 6 | - Do a dry-run and inspect the compiled files 7 | 8 | There's more details here: https://helm.sh/docs/chart_template_guide/debugging/ 9 | -------------------------------------------------------------------------------- /ee/cmd/beneath/dependencies/ee_control_server.go: -------------------------------------------------------------------------------- 1 | package dependencies 2 | 3 | import ( 4 | "github.com/beneath-hq/beneath/cmd/beneath/cli" 5 | eecontrol "github.com/beneath-hq/beneath/ee/server/control" 6 | ) 7 | 8 | func init() { 9 | cli.AddDependency(eecontrol.NewServer) 10 | } 11 | -------------------------------------------------------------------------------- /examples/cdc/fanout/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | RUN pip install poetry==1.1.12 3 | WORKDIR /app 4 | COPY poetry.lock pyproject.toml . 5 | RUN poetry config virtualenvs.create false \ 6 | && poetry install --no-dev --no-interaction --no-ansi 7 | COPY . . 8 | ENTRYPOINT ["python", "main.py"] 9 | -------------------------------------------------------------------------------- /web/apollo/queries/local/records.ts: -------------------------------------------------------------------------------- 1 | import gql from "graphql-tag"; 2 | 3 | export const CREATE_RECORDS = gql` 4 | mutation CreateRecords($instanceID: UUID!, $json: JSON!) { 5 | createRecords(instanceID: $instanceID, json: $json) @client { 6 | error 7 | } 8 | } 9 | `; 10 | -------------------------------------------------------------------------------- /examples/ethereum-blocks/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | RUN pip install poetry==1.1.4 3 | WORKDIR /app 4 | COPY poetry.lock pyproject.toml /app/ 5 | RUN poetry config virtualenvs.create false \ 6 | && poetry install --no-dev --no-interaction --no-ansi 7 | COPY . . 8 | ENTRYPOINT ["python", "main.py"] 9 | -------------------------------------------------------------------------------- /clients/java/src/main/java/dev/beneath/client/admin/GraphQLException.java: -------------------------------------------------------------------------------- 1 | package dev.beneath.client.admin; 2 | 3 | /** 4 | * GraphQL error 5 | */ 6 | public class GraphQLException extends RuntimeException { 7 | public GraphQLException(String message) { 8 | super(message); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /services/permissions/README.md: -------------------------------------------------------------------------------- 1 | # `services/permissions/` 2 | 3 | This service manages access permissions between _owners_ (users and services) and _resources_ (organizations, projects and tables). 4 | 5 | Amongst other things, it offers a permissions cache that can be used to check permissions on every request. 6 | -------------------------------------------------------------------------------- /clients/java/src/main/java/dev/beneath/client/AuthenticationException.java: -------------------------------------------------------------------------------- 1 | package dev.beneath.client; 2 | 3 | /** 4 | * Error returned for failed authentication 5 | */ 6 | class AuthenticationException extends RuntimeException { 7 | AuthenticationException(String message) { 8 | super(message); 9 | } 10 | } -------------------------------------------------------------------------------- /web/apollo.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | client: { 3 | includes: ["apollo/**/*.ts"], 4 | excludes: ["**/node_modules", "**/__tests__", ".next", "kube", "apollo/types"], 5 | service: { 6 | name: "beneath-control", 7 | url: "http://localhost:4000/graphql", 8 | }, 9 | }, 10 | }; 11 | -------------------------------------------------------------------------------- /web/public/assets/favicon/browserconfig.xml: -------------------------------------------------------------------------------- 1 | 2 | #ffffff -------------------------------------------------------------------------------- /ee/migrations/migrations.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/beneath-hq/beneath/pkg/migrationsutil" 5 | ) 6 | 7 | // Migrator registers and runs migrations. 8 | // Tracks migrations in a different table than non-enterprise migrations. 9 | var Migrator = migrationsutil.New("gopg_migrations_ee") 10 | -------------------------------------------------------------------------------- /examples/beta/earthquake-notifications/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7 2 | WORKDIR /app 3 | COPY requirements.txt ./ 4 | RUN pip install -r requirements.txt 5 | COPY earthquake_notifications.py ./ 6 | CMD ["python", "earthquake_notifications.py", "run", "demos/earthquakes/notify", "--read-quota-mb", "1000", "--write-quota-mb", "0"] 7 | -------------------------------------------------------------------------------- /examples/beta/lending-club/loans/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7 2 | WORKDIR /app 3 | COPY requirements.txt ./ 4 | RUN pip install -r requirements.txt 5 | COPY fetch_new_loans.py loans.graphql ./ 6 | CMD ["python", "fetch_new_loans.py", "run", "epg/lending-club/fetch-new-loans", "--read-quota-mb", "1000", "--write-quota-mb", "3000"] 7 | -------------------------------------------------------------------------------- /examples/earthquakes/schemas/earthquake.graphql: -------------------------------------------------------------------------------- 1 | type Earthquake @schema { 2 | " Time of earthquake " 3 | time: Timestamp! @key 4 | 5 | " Richter scale magnitude " 6 | mag: Float32 7 | 8 | " Location of earthquake " 9 | place: String! 10 | 11 | " Link for more information " 12 | detail: String 13 | } 14 | -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/aggregates/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | RUN pip install poetry==1.1.4 3 | WORKDIR /app 4 | COPY poetry.lock pyproject.toml /app/ 5 | RUN poetry config virtualenvs.create false \ 6 | && poetry install --no-dev --no-interaction --no-ansi 7 | COPY . . 8 | ENTRYPOINT ["python", "aggregate.py"] 9 | -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/scores/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | RUN pip install poetry==1.1.4 3 | WORKDIR /app 4 | COPY poetry.lock pyproject.toml /app/ 5 | RUN poetry config virtualenvs.create false \ 6 | && poetry install --no-dev --no-interaction --no-ansi 7 | COPY . . 8 | ENTRYPOINT ["python", "get_scores.py"] 9 | -------------------------------------------------------------------------------- /migrations/020_tax_info.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // moved to ee 10 | return nil 11 | }, func(db migrations.DB) (err error) { 12 | return nil 13 | }) 14 | } 15 | -------------------------------------------------------------------------------- /services/user/README.md: -------------------------------------------------------------------------------- 1 | # `services/user/` 2 | 3 | This service manages the `User` model (also see `models/user.go`). 4 | 5 | Beware that `User` and `Organization` have a tight relationship: every `User` automatically has a "personal" `Organization` (so also check out `services/organization/` if you need to edit something). 6 | -------------------------------------------------------------------------------- /web/tsconfig.server.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "module": "commonjs", 5 | "outDir": "dist", 6 | "target": "es2017", 7 | "isolatedModules": false, 8 | "noEmit": false 9 | }, 10 | "include": [ 11 | "server/**/*.ts", 12 | "server.ts" 13 | ] 14 | } -------------------------------------------------------------------------------- /migrations/019_billing_method.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // moved to ee 10 | return nil 11 | }, func(db migrations.DB) (err error) { 12 | return nil 13 | }) 14 | } 15 | -------------------------------------------------------------------------------- /migrations/028_rm_entity_name.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // moved to er 10 | return nil 11 | }, func(db migrations.DB) (err error) { 12 | return nil 13 | }) 14 | } 15 | -------------------------------------------------------------------------------- /migrations/030_billing_floats.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // moved to ee 10 | return nil 11 | }, func(db migrations.DB) (err error) { 12 | return nil 13 | }) 14 | } 15 | -------------------------------------------------------------------------------- /web/ee/apollo.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | client: { 3 | includes: ["apollo/**/*.ts"], 4 | excludes: ["**/node_modules", "**/__tests__", ".next", "kube", "ee/apollo/types"], 5 | service: { 6 | name: "beneath-control-ee", 7 | url: "http://localhost:4000/ee/graphql", 8 | }, 9 | }, 10 | }; 11 | -------------------------------------------------------------------------------- /clients/python/tools/pypi-publish.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 1. Upgrade setuptools and wheel 4 | python3 -m pip install --upgrade setuptools wheel twine 5 | 6 | # 2. generate distribution archives 7 | python3 setup.py sdist bdist_wheel 8 | 9 | # 3. upload the distribution archives 10 | python3 -m twine upload dist/* 11 | -------------------------------------------------------------------------------- /migrations/006_secret.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // Removed 10 | return nil 11 | }, func(db migrations.DB) (err error) { 12 | // Removed 13 | return nil 14 | }) 15 | } 16 | -------------------------------------------------------------------------------- /migrations/027_billing_plan_quotas.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // moved to ee 10 | return nil 11 | }, func(db migrations.DB) (err error) { 12 | return nil 13 | }) 14 | } 15 | -------------------------------------------------------------------------------- /clients/python/tools/dev-jupyter.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # check we're in the client repo 4 | if [[ ! -f tools/$(basename $0) ]]; then 5 | echo "you must execute this script from the python client root folder (clients/python)" 6 | exit 1 7 | fi 8 | 9 | PYTHONPATH=$(pwd $0):$PYTHONPATH BENEATH_ENV=dev jupyter notebook 10 | -------------------------------------------------------------------------------- /ee/server/control/schema/billing_method.graphql: -------------------------------------------------------------------------------- 1 | extend type Query { 2 | billingMethods(organizationID: UUID!): [BillingMethod!]! 3 | } 4 | 5 | type BillingMethod { 6 | billingMethodID: ID! 7 | organizationID: UUID! 8 | paymentsDriver: String! 9 | driverPayload: String! 10 | createdOn: Time! 11 | updatedOn: Time! 12 | } 13 | -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/sentiment/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | RUN pip install poetry==1.1.4 3 | WORKDIR /app 4 | COPY poetry.lock pyproject.toml /app/ 5 | RUN poetry config virtualenvs.create false \ 6 | && poetry install --no-dev --no-interaction --no-ansi 7 | COPY . . 8 | ENTRYPOINT ["python", "predict_sentiment_pipeline.py"] 9 | -------------------------------------------------------------------------------- /migrations/022_nullable_billing_method.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // moved to ee 10 | return nil 11 | }, func(db migrations.DB) (err error) { 12 | return nil 13 | }) 14 | } 15 | -------------------------------------------------------------------------------- /migrations/031_billing_plan_base_price.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // moved to ee 10 | return nil 11 | }, func(db migrations.DB) (err error) { 12 | return nil 13 | }) 14 | } 15 | -------------------------------------------------------------------------------- /migrations/033_billing_plan_multiple_users.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // moved to ee 10 | return nil 11 | }, func(db migrations.DB) (err error) { 12 | return nil 13 | }) 14 | } 15 | -------------------------------------------------------------------------------- /clients/java/src/main/graphql/dev/beneath/client/CreateTableInstance.graphql: -------------------------------------------------------------------------------- 1 | mutation CreateTableInstance($input: CreateTableInstanceInput!) { 2 | createTableInstance(input: $input) { 3 | tableInstanceID 4 | tableID 5 | createdOn 6 | version 7 | madePrimaryOn 8 | madeFinalOn 9 | } 10 | } -------------------------------------------------------------------------------- /clients/python/beneath/beam/__init__.py: -------------------------------------------------------------------------------- 1 | from beneath.beam.write_to_beneath import WriteToBeneath 2 | from beneath.beam.read_from_beneath import ReadFromBeneath 3 | from beneath.beam.convert_types import ConvertFromJSONTypes 4 | 5 | __all__ = [ 6 | "WriteToBeneath", 7 | "ReadFromBeneath", 8 | "ConvertFromJSONTypes", 9 | ] 10 | -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/stock-mentions/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | RUN pip install poetry==1.1.4 3 | WORKDIR /app 4 | COPY poetry.lock pyproject.toml /app/ 5 | RUN poetry config virtualenvs.create false \ 6 | && poetry install --no-dev --no-interaction --no-ansi 7 | COPY . . 8 | ENTRYPOINT ["python", "find_mentions_pipeline.py"] 9 | -------------------------------------------------------------------------------- /ee/services/billing/README.md: -------------------------------------------------------------------------------- 1 | # `ee/services/billing/` 2 | 3 | The billing service handles billing info and consumed resources. It keeps tabs on *what to bill*, but the `payments` service handles the actual invoicing. 4 | 5 | Some of the billing logic is not super trivial. For sanity, we've tried to gather most of the complex logic in `commit.go`. 6 | -------------------------------------------------------------------------------- /migrations/042_billing_plan_remove_private_projects.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // moved to ee 10 | return nil 11 | }, func(db migrations.DB) (err error) { 12 | return nil 13 | }) 14 | } 15 | -------------------------------------------------------------------------------- /services/organization/README.md: -------------------------------------------------------------------------------- 1 | # `services/organization/` 2 | 3 | This service manages the `Organization` model (also see `models/organization.go`). 4 | 5 | Beware that `Organization` and `User` have a tight relationship: every `User` automatically has a "personal" `Organization` (so also check out `services/user/` if you need to edit something). 6 | -------------------------------------------------------------------------------- /licenses/CC-BY-SA-4.0.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019-present Beneath Systems ApS 2 | 3 | This work is licensed under the Creative Commons Attribution-ShareAlike 4.0 4 | International License. To view a copy of this license, visit 5 | http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to Creative 6 | Commons, PO Box 1866, Mountain View, CA 94042, USA. 7 | -------------------------------------------------------------------------------- /scripts/proto-build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | shopt -s expand_aliases 4 | alias compile="protoc --go_out=plugins=grpc,paths=source_relative:." 5 | 6 | compile infra/engine/proto/*.proto 7 | compile infra/engine/driver/bigtable/proto/*.proto 8 | compile infra/engine/driver/bigquery/proto/*.proto 9 | compile server/data/grpc/proto/*.proto 10 | -------------------------------------------------------------------------------- /examples/reddit/schemas/comment.graphql: -------------------------------------------------------------------------------- 1 | type Comment @schema { 2 | created_on: Timestamp! @key 3 | id: String! @key 4 | author: String! 5 | subreddit: String! 6 | post_id: String! 7 | parent_id: String! 8 | text: String 9 | permalink: String 10 | is_submitter: Boolean! 11 | is_distinguished: Boolean! 12 | is_stickied: Boolean! 13 | } -------------------------------------------------------------------------------- /web/ee/apollo/queries/billingMethod.ts: -------------------------------------------------------------------------------- 1 | import gql from "graphql-tag"; 2 | 3 | export const QUERY_BILLING_METHODS = gql` 4 | query BillingMethods($organizationID: UUID!){ 5 | billingMethods(organizationID: $organizationID) { 6 | billingMethodID 7 | organizationID 8 | paymentsDriver 9 | driverPayload 10 | } 11 | } 12 | `; 13 | -------------------------------------------------------------------------------- /examples/beta/lending-club/loans-enriched/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7 2 | WORKDIR /app 3 | COPY requirements.txt ./ 4 | RUN pip install -r requirements.txt 5 | COPY enrich_loans.py loans_enriched.graphql model.pkl ./ 6 | CMD ["python", "enrich_loans.py", "run", "epg/lending-club/enrich-loans", "--strategy", "delta", "--read-quota-mb", "1000", "--write-quota-mb", "1000"] 7 | -------------------------------------------------------------------------------- /pkg/queryparse/parse.go: -------------------------------------------------------------------------------- 1 | package queryparse 2 | 3 | import "strings" 4 | 5 | // StringToQuery attempts to parse str as either a JSON or Where query 6 | func StringToQuery(str string) (Query, error) { 7 | str = strings.TrimSpace(str) 8 | if len(str) != 0 && str[0] == '{' { 9 | return JSONStringToQuery(str) 10 | } 11 | return WhereStringToQuery(str) 12 | } 13 | -------------------------------------------------------------------------------- /clients/java/src/main/graphql/dev/beneath/client/CreateProject.graphql: -------------------------------------------------------------------------------- 1 | mutation CreateProject($input: CreateProjectInput!) { 2 | createProject(input: $input) { 3 | projectID 4 | name 5 | displayName 6 | public 7 | site 8 | description 9 | photoURL 10 | createdOn 11 | updatedOn 12 | } 13 | } -------------------------------------------------------------------------------- /services/service/README.md: -------------------------------------------------------------------------------- 1 | # `services/service/` 2 | 3 | This service contains functionality for finding and creating services. It's the services service... Yeah, that is CONFUSING! Explainer: In the codebase, a "service" wraps functionality for a specific domain of the app, while in Beneath, a "service" is a *non-user account* (also known as a "service account" in e.g. GCP). 4 | -------------------------------------------------------------------------------- /web/components/formik/index.ts: -------------------------------------------------------------------------------- 1 | export { default as Checkbox } from "./Checkbox"; 2 | export { default as Form } from "./Form"; 3 | export { default as handleSubmitMutation } from "./handleSubmitMutation"; 4 | export { default as RadioGroup } from "./RadioGroup"; 5 | export { default as SelectField } from "./SelectField"; 6 | export { default as TextField } from "./TextField"; 7 | -------------------------------------------------------------------------------- /services/table/README.md: -------------------------------------------------------------------------------- 1 | # `services/table/` 2 | 3 | This service manages the `Table` and `TableInstance` models (also see `models/table.go`). 4 | 5 | It's functionality includes: 6 | 7 | - Compiling table schemas 8 | - Managing instance versions 9 | - Managing storage of instances in the underlying data systems (`infra/engine/`) 10 | - Various table metadata caches 11 | -------------------------------------------------------------------------------- /test/integration/util.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | ) 7 | 8 | func panicIf(err error) { 9 | if err != nil { 10 | panic(err) 11 | } 12 | } 13 | 14 | func readTestdata(filename string) string { 15 | schema, err := ioutil.ReadFile(fmt.Sprintf("test/testdata/%s", filename)) 16 | panicIf(err) 17 | return string(schema) 18 | } 19 | -------------------------------------------------------------------------------- /clients/python/beneath/admin/base.py: -------------------------------------------------------------------------------- 1 | from beneath.connection import Connection 2 | 3 | 4 | class _ResourceBase: 5 | def __init__(self, conn: Connection, dry=False): 6 | self.conn = conn 7 | self.dry = dry 8 | 9 | def _before_mutation(self): 10 | if self.dry: 11 | raise Exception("Cannot run mutation on a client where dry=True") 12 | -------------------------------------------------------------------------------- /examples/cdc/generator/Dockerfile: -------------------------------------------------------------------------------- 1 | # Create builder image 2 | FROM gradle:7.3.3-jdk17-alpine as builder 3 | WORKDIR /app 4 | COPY build.gradle settings.gradle . 5 | COPY src src 6 | RUN gradle --no-daemon build 7 | 8 | # Create run image 9 | FROM openjdk:17-alpine 10 | COPY --from=builder /app/build/libs/cdc-generator-1.0.0-SNAPSHOT.jar app.jar 11 | ENTRYPOINT java -jar app.jar 12 | -------------------------------------------------------------------------------- /examples/clock/README.md: -------------------------------------------------------------------------------- 1 | To run locally: 2 | 3 | Set up poetry virtual environment 4 | 5 | ```bash 6 | poetry install 7 | poetry shell 8 | ``` 9 | 10 | Stage the tables and a service for the pipeline, then run it 11 | 12 | ```bash 13 | python clock.py stage USERNAME/PROJECT/clock --read-quota-mb 1000 --write-quota-mb 2000 14 | python clock.py run USERNAME/PROJECT/clock 15 | ``` 16 | -------------------------------------------------------------------------------- /examples/earthquakes/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8 2 | RUN pip install poetry==1.1.4 3 | WORKDIR /app 4 | COPY poetry.lock pyproject.toml /app/ 5 | RUN poetry config virtualenvs.create false \ 6 | && poetry install --no-dev --no-interaction --no-ansi 7 | COPY . . 8 | CMD ["python", "main.py", "run", "examples/earthquakes/earthquakes", "--read-quota-mb", "1000", "--write-quota-mb", "1000"] 9 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | # `test/` 2 | 3 | This directory contains end-to-end/integration tests 4 | 5 | ### Running these tests 6 | 7 | The tests must run at a package level to include the `TestMain` wrapper. The `BENEATH_ENV` environment variable should also be set to `test`. From the root directory, run tests with 8 | 9 | ``` 10 | BENEATH_ENV=test go test ./test/integration/ 11 | ``` 12 | -------------------------------------------------------------------------------- /clients/java/src/main/java/dev/beneath/client/InstanceIdAndRecordPb.java: -------------------------------------------------------------------------------- 1 | package dev.beneath.client; 2 | 3 | import java.util.UUID; 4 | 5 | public class InstanceIdAndRecordPb { 6 | public UUID instanceId; 7 | public Record record; 8 | 9 | InstanceIdAndRecordPb(UUID instanceId, Record record) { 10 | this.instanceId = instanceId; 11 | this.record = record; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /web/apollo/types/AID.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: AID 8 | // ==================================================== 9 | 10 | export interface AID { 11 | aid: string | null; 12 | } 13 | -------------------------------------------------------------------------------- /web/components/ErrorNote.tsx: -------------------------------------------------------------------------------- 1 | import { Grid, Typography } from "@material-ui/core"; 2 | import { FC } from "react"; 3 | 4 | const ErrorNote: FC<{ error?: Error }> = ({ error }) => { 5 | return ( 6 | 7 | {error ? error.message : "Unexpected error"} 8 | 9 | ); 10 | }; 11 | 12 | export default ErrorNote; 13 | -------------------------------------------------------------------------------- /ee/config/README.md: -------------------------------------------------------------------------------- 1 | # `ee/config/` 2 | 3 | This folder contains templates for extra EE-related config keys. 4 | 5 | Beneath doesn't support loading multiple config files, and by default only looks in the root `config/` directory for `.ENV.yaml` files (i.e. it will not pickup on a config file in `ee/config/`). So you'll want to copy the relevant keys from these templates into your main config file. 6 | -------------------------------------------------------------------------------- /web/apollo/types/Token.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: Token 8 | // ==================================================== 9 | 10 | export interface Token { 11 | token: string | null; 12 | } 13 | -------------------------------------------------------------------------------- /examples/clock-react/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | React App 7 | 8 | 9 | 10 |
11 | 12 | 13 | -------------------------------------------------------------------------------- /ee/cmd/beneath/README.md: -------------------------------------------------------------------------------- 1 | # `ee/cmd/beneath/` 2 | 3 | This folder contains the main entrypoint for the EE version of Beneath. 4 | 5 | It's `main.go` is largely a duplicate of `cmd/beneath/main.go`, but registers extra EE-related commands and dependencies (namely `ee/server/control/` and `ee/services/`). When making changes here, make sure to consider if they should be mirrored in `cmd/beneath` (or vice versa!). 6 | -------------------------------------------------------------------------------- /infra/engine/proto/quota.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | option go_package = "github.com/beneath-hq/beneath/infra/engine/proto"; 3 | package engine; 4 | 5 | message QuotaUsage { 6 | int64 read_ops = 1; 7 | int64 read_records = 2; 8 | int64 read_bytes = 3; 9 | int64 write_ops = 4; 10 | int64 write_records = 5; 11 | int64 write_bytes = 6; 12 | int64 scan_ops = 7; 13 | int64 scan_bytes = 8; 14 | } -------------------------------------------------------------------------------- /web/hooks/useToken.tsx: -------------------------------------------------------------------------------- 1 | import { useQuery } from "@apollo/client"; 2 | 3 | import { GET_TOKEN } from "../apollo/queries/local/token"; 4 | import { Token } from "../apollo/types/Token"; 5 | 6 | export const useToken = () => { 7 | const { loading, error, data } = useQuery(GET_TOKEN); 8 | if (data) { 9 | const { token } = data; 10 | return token; 11 | } 12 | 13 | return null; 14 | }; 15 | -------------------------------------------------------------------------------- /clients/java/tools/get-graphql-schema.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | # check we're in the client repo 4 | if [[ ! -f tools/$(basename $0) ]]; then 5 | echo "you must execute this script from the java client root folder (clients/java)" 6 | exit 1 7 | fi 8 | 9 | ./gradlew downloadApolloSchema \ 10 | --endpoint="http:host.docker.internal:4000/graphql" \ 11 | --schema="src/main/graphql/dev/beneath/schema.graphqls" -------------------------------------------------------------------------------- /ee/build/backend/Dockerfile: -------------------------------------------------------------------------------- 1 | # Create builder image 2 | FROM golang:alpine as builder 3 | WORKDIR /app 4 | COPY go.mod go.sum ./ 5 | RUN go mod download 6 | COPY . . 7 | RUN CGO_ENABLED=0 GOOS=linux go build -a -o main ee/cmd/beneath/main.go 8 | 9 | # Create run image 10 | FROM alpine:latest 11 | RUN apk --no-cache add ca-certificates 12 | WORKDIR /root 13 | COPY --from=builder /app/main . 14 | ENTRYPOINT ["./main"] 15 | -------------------------------------------------------------------------------- /ee/cloud/deployments/helm/web/values.yaml: -------------------------------------------------------------------------------- 1 | nameOverride: "" 2 | fullnameOverride: "" 3 | 4 | replicaCount: 1 5 | 6 | image: 7 | pullPolicy: IfNotPresent 8 | repository: gcr.io/beneath/ee-web 9 | tag: latest 10 | 11 | ingress: 12 | hosts: [] 13 | 14 | resources: {} 15 | # limits: 16 | # cpu: 100m 17 | # memory: 128Mi 18 | # requests: 19 | # cpu: 100m 20 | # memory: 128Mi 21 | 22 | extraEnv: [] 23 | -------------------------------------------------------------------------------- /web/pages/-/sql.tsx: -------------------------------------------------------------------------------- 1 | import { NextPage } from "next"; 2 | 3 | import { withApollo } from "../../apollo/withApollo"; 4 | import Page from "../../components/Page"; 5 | import Main from "../../components/sql/Main"; 6 | 7 | const SQLPage: NextPage = () => { 8 | return ( 9 | 10 |
11 | 12 | ); 13 | }; 14 | 15 | export default withApollo(SQLPage); 16 | -------------------------------------------------------------------------------- /web/pages/-/auth/index.tsx: -------------------------------------------------------------------------------- 1 | import { NextPage } from "next"; 2 | 3 | import { withApollo } from "apollo/withApollo"; 4 | import Page from "components/Page"; 5 | import Auth from "components/Auth"; 6 | 7 | const AuthPage: NextPage = () => { 8 | return ( 9 | 10 | 11 | 12 | ); 13 | }; 14 | 15 | export default withApollo(AuthPage); 16 | -------------------------------------------------------------------------------- /services/usage/README.md: -------------------------------------------------------------------------------- 1 | # `services/usage/` 2 | 3 | This service handles reading and tracking of quota usage (i.e. reads, writes and scans). 4 | 5 | Note that if you use the service to *track* usage (and not just read usage), it has a worker component that must `Run` concurrently to properly flush tracked usage in the background. 6 | 7 | It's used to *read and track* usage by the `data-server`, and to *read* usage metrics by the `control-server`. 8 | -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/aggregates/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "aggregates" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["greenep12 "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.8" 9 | beneath = "^1.4.1" 10 | asyncio = "^3.4.3" 11 | 12 | [tool.poetry.dev-dependencies] 13 | 14 | [build-system] 15 | requires = ["poetry-core>=1.0.0"] 16 | build-backend = "poetry.core.masonry.api" 17 | -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/sentiment/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "sentiment" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["greenep12 "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.8" 9 | beneath = "^1.4.1" 10 | textblob = "^0.15.3" 11 | 12 | [tool.poetry.dev-dependencies] 13 | 14 | [build-system] 15 | requires = ["poetry-core>=1.0.0"] 16 | build-backend = "poetry.core.masonry.api" 17 | -------------------------------------------------------------------------------- /clients/js/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "module": "commonjs", 5 | "lib": [ 6 | "es2017", 7 | "es7", 8 | "es6", 9 | "dom" 10 | ], 11 | "declaration": true, 12 | "outDir": "dist", 13 | "strict": true, 14 | "esModuleInterop": true 15 | }, 16 | "include": [ 17 | "src/**/*" 18 | ], 19 | "exclude": [ 20 | "dist", 21 | "node_modules" 22 | ] 23 | } -------------------------------------------------------------------------------- /examples/beta/covid19/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "covid19" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Beneath Systems "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.7" 9 | beneath = "^1.2.12" 10 | requests = "^2.25.1" 11 | DateTime = "^4.3" 12 | 13 | [tool.poetry.dev-dependencies] 14 | 15 | [build-system] 16 | requires = ["poetry-core>=1.0.0"] 17 | build-backend = "poetry.core.masonry.api" 18 | -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/stock-prices/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "stock-prices" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["greenep12 "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.8" 9 | beneath = "^1.4.1" 10 | jupyter = "^1.0.0" 11 | 12 | [tool.poetry.dev-dependencies] 13 | 14 | [build-system] 15 | requires = ["poetry-core>=1.0.0"] 16 | build-backend = "poetry.core.masonry.api" 17 | -------------------------------------------------------------------------------- /infra/db/util.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "github.com/go-pg/pg/v9" 5 | ) 6 | 7 | // AssertFoundOne uses the error from a QueryOne operation 8 | // (which includes Select for a single object) to determine 9 | // if a result was found or not. Panics on database errors 10 | func AssertFoundOne(err error) bool { 11 | if err != nil { 12 | if err == pg.ErrNoRows { 13 | return false 14 | } 15 | panic(err) 16 | } 17 | return true 18 | } 19 | -------------------------------------------------------------------------------- /pkg/bytesutil/bytesutil_test.go: -------------------------------------------------------------------------------- 1 | package bytesutil 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestIntToBytes(t *testing.T) { 10 | xs := []int64{0, 1, 2, -1, -2, 314, -314, 0xFFFFFFFF, -0xFFFFFFFF, 0x7FFFFFFFFFFFFFFF, -0x8000000000000000} 11 | for _, x := range xs { 12 | b := IntToBytes(x) 13 | y := BytesToInt(b) 14 | assert.Equal(t, x, y) 15 | assert.Equal(t, 8, len(b)) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /clients/js-react/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "module": "commonjs", 5 | "lib": [ 6 | "es2017", 7 | "es7", 8 | "es6", 9 | "dom" 10 | ], 11 | "declaration": true, 12 | "outDir": "dist", 13 | "strict": true, 14 | "esModuleInterop": true 15 | }, 16 | "include": [ 17 | "src/**/*" 18 | ], 19 | "exclude": [ 20 | "dist", 21 | "node_modules" 22 | ] 23 | } -------------------------------------------------------------------------------- /examples/ethereum-blocks/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "ethereum-blocks" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Beneath Systems "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.8" 9 | beneath = "^1.3.5" 10 | web3 = "^5.17.0" 11 | tenacity = "^7.0.0" 12 | 13 | [tool.poetry.dev-dependencies] 14 | 15 | [build-system] 16 | requires = ["poetry-core>=1.0.0"] 17 | build-backend = "poetry.core.masonry.api" 18 | -------------------------------------------------------------------------------- /examples/reddit/schemas/post.graphql: -------------------------------------------------------------------------------- 1 | type Post @schema { 2 | created_on: Timestamp! @key 3 | id: String! @key 4 | author: String! 5 | subreddit: String! 6 | title: String! 7 | text: String 8 | link: String 9 | permalink: String 10 | flair: String 11 | is_over_18: Boolean! 12 | is_original_content: Boolean! 13 | is_self_post: Boolean! 14 | is_distinguished: Boolean! 15 | is_spoiler: Boolean! 16 | is_stickied: Boolean! 17 | } 18 | -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/stock-mentions/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "stock-mentions" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["greenep12 "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.8" 9 | beneath = "^1.4.0" 10 | pandas = "^1.2.4" 11 | 12 | [tool.poetry.dev-dependencies] 13 | 14 | [build-system] 15 | requires = ["poetry-core>=1.0.0"] 16 | build-backend = "poetry.core.masonry.api" 17 | -------------------------------------------------------------------------------- /ee/cloud/deployments/kube/jupyterhub/deploy.sh: -------------------------------------------------------------------------------- 1 | # add helm repo 2 | helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/ 3 | helm repo update 4 | 5 | # run upgrade 6 | helm upgrade --install jupyterhub jupyterhub/jupyterhub \ 7 | --namespace jupyterhub \ 8 | --version 0.7.0 \ 9 | --values helm/config.yaml 10 | 11 | # 0.9-174bbd5 12 | 13 | # !pip install psycopg2-binary 14 | # !pip install google-cloud-bigquery 15 | # !pip install squarify 16 | -------------------------------------------------------------------------------- /examples/clock/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "clock" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Beneath Systems "] 6 | license = "MIT" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.8" 10 | beneath = "^1.4.0" 11 | asyncio = "^3.4.3" 12 | 13 | [tool.poetry.dev-dependencies] 14 | black = "^20.8b1" 15 | 16 | [build-system] 17 | requires = ["poetry-core>=1.0.0"] 18 | build-backend = "poetry.core.masonry.api" 19 | -------------------------------------------------------------------------------- /infra/engine/driver/bigtable/proto/cursor.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | option go_package = "proto"; 3 | package bigtable; 4 | 5 | message CursorSet { 6 | repeated Cursor cursors = 1; 7 | } 8 | 9 | message Cursor { 10 | enum Type { 11 | LOG = 0; 12 | INDEX = 1; 13 | PEEK = 2; 14 | } 15 | Type type = 1; 16 | int64 log_start = 10; 17 | int64 log_end = 11; 18 | int32 index_id = 20; 19 | bytes index_start = 21; 20 | bytes index_end = 22; 21 | } 22 | -------------------------------------------------------------------------------- /examples/cdc/generator/README.md: -------------------------------------------------------------------------------- 1 | The `generator` service uses change data capture to subscribe to a source database (e.g. Postgres) and write the changes to a Beneath table. 2 | 3 | The service uses the [Debezium Engine](https://debezium.io/documentation/reference/1.6/development/engine.html) and checkpoints its state to Beneath. Typically, Debezium is deployed via Kafka Connect and checkpoints to Kafka, but we eliminate the Kafka dependency by using Beneath instead. 4 | 5 | -------------------------------------------------------------------------------- /infra/redis/redis.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | import ( 4 | "github.com/go-redis/redis/v7" 5 | ) 6 | 7 | // Options for opening a new Redis connection 8 | type Options struct { 9 | URL string 10 | } 11 | 12 | // NewRedis opens a new Redis connection 13 | func NewRedis(opts *Options) *redis.Client { 14 | redisOpts, err := redis.ParseURL(opts.URL) 15 | if err != nil { 16 | panic(err) 17 | } 18 | 19 | client := redis.NewClient(redisOpts) 20 | return client 21 | } 22 | -------------------------------------------------------------------------------- /examples/cdc/generator/src/main/java/dev/beneath/cdc/postgres/App.java: -------------------------------------------------------------------------------- 1 | package dev.beneath.cdc.postgres; 2 | 3 | import dev.beneath.client.BeneathClient; 4 | 5 | public class App { 6 | public static void main(String[] args) { 7 | BeneathClient client = new BeneathClient(CdcConfig.BENEATH_SECRET, false, CdcConfig.DEFAULT_WRITE_DELAY_MS); 8 | BeneathDebeziumEngine engine = new BeneathDebeziumEngine(client); 9 | engine.start(); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /infra/engine/README.md: -------------------------------------------------------------------------------- 1 | # `engine/` 2 | 3 | This directory contains the engine, which encapsulates logic for interfacing with the underlying data systems. 4 | 5 | The files in this repository are wrappers for functionality provided by the underlying drivers. Most of the interesting stuff happens in `infra/engine/driver/`. 6 | 7 | ## Updating protocol buffer definitions 8 | 9 | - Run `scripts/proto-build.sh` to (re)generate all protocol buffers classes in the repository 10 | -------------------------------------------------------------------------------- /web/scripts/generateApolloTypes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Generate apollo/possibleTypes.ts 4 | yarn run ts-node ./scripts/apolloGeneratePossibleTypes.ts "/graphql" "apollo/possibleTypes.json" 5 | 6 | # Generate Typescript types for our Apollo results in apollo/types/ 7 | yarn run apollo codegen:generate \ 8 | --config ./apollo.config.js \ 9 | --outputFlat apollo/types \ 10 | --target typescript \ 11 | --passthroughCustomScalars \ 12 | --customScalarsPrefix Control 13 | -------------------------------------------------------------------------------- /ee/cloud/deployments/kube/cert-manager/cluster-issuer-staging.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1 2 | kind: ClusterIssuer 3 | metadata: 4 | name: letsencrypt-staging 5 | namespace: cert-manager 6 | spec: 7 | acme: 8 | email: benjamin@beneath.dev 9 | server: https://acme-staging-v02.api.letsencrypt.org/directory 10 | privateKeySecretRef: 11 | name: letsencrypt-staging 12 | solvers: 13 | - http01: 14 | ingress: 15 | class: nginx 16 | -------------------------------------------------------------------------------- /web/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "development": { 4 | "presets": [ 5 | "next/babel" 6 | ] 7 | }, 8 | "production": { 9 | "presets": [ 10 | "next/babel" 11 | ] 12 | }, 13 | "test": { 14 | "presets": [ 15 | [ 16 | "next/babel", 17 | { 18 | "preset-env": { 19 | "modules": "commonjs" 20 | } 21 | } 22 | ] 23 | ] 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/explore/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "explore" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["greenep12 "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.8" 9 | beneath = "^1.4.1" 10 | plotly = "^5.0.0" 11 | chart-studio = "1.1.0" 12 | 13 | [tool.poetry.dev-dependencies] 14 | jupyter = "^1.0.0" 15 | 16 | [build-system] 17 | requires = ["poetry-core>=1.0.0"] 18 | build-backend = "poetry.core.masonry.api" 19 | -------------------------------------------------------------------------------- /infra/engine/proto/engine.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | option go_package = "github.com/beneath-hq/beneath/infra/engine/proto"; 3 | package engine; 4 | 5 | import "server/data/grpc/proto/gateway.proto"; 6 | 7 | message WriteRequest { 8 | bytes write_id = 1; 9 | repeated gateway.v1.InstanceRecords instance_records = 2; 10 | } 11 | 12 | message WriteReport { 13 | bytes write_id = 1; 14 | bytes instance_id = 2; 15 | int32 records_count = 3; 16 | int32 bytes_total = 4; 17 | } 18 | -------------------------------------------------------------------------------- /clients/python/beneath/pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | from beneath.pipeline.base_pipeline import Action, BasePipeline, Strategy 2 | from beneath.pipeline.parse_args import parse_pipeline_args 3 | from beneath.pipeline.pipeline import AsyncApplyFn, AsyncGenerateFn, Pipeline, PIPELINE_IDLE 4 | 5 | __all__ = [ 6 | "Action", 7 | "AsyncApplyFn", 8 | "AsyncGenerateFn", 9 | "BasePipeline", 10 | "parse_pipeline_args", 11 | "Pipeline", 12 | "PIPELINE_IDLE", 13 | "Strategy", 14 | ] 15 | -------------------------------------------------------------------------------- /ee/cloud/deployments/gitlab/admin-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: gitlab-admin 5 | namespace: kube-system 6 | --- 7 | apiVersion: rbac.authorization.k8s.io/v1beta1 8 | kind: ClusterRoleBinding 9 | metadata: 10 | name: gitlab-admin 11 | roleRef: 12 | apiGroup: rbac.authorization.k8s.io 13 | kind: ClusterRole 14 | name: cluster-admin 15 | subjects: 16 | - kind: ServiceAccount 17 | name: gitlab-admin 18 | namespace: kube-system 19 | -------------------------------------------------------------------------------- /examples/cdc/fanout/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "cdc-postgres" 3 | version = "1.0.0" 4 | description = "" 5 | authors = ["greenep12 "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.8" 9 | beneath = "^1.4.2" 10 | asyncio = "^3.4.3" 11 | DateTime = "^4.3" 12 | 13 | [tool.poetry.dev-dependencies] 14 | PyYAML = "^5.4.1" 15 | psycopg2 = "^2.9.1" 16 | 17 | [build-system] 18 | requires = ["poetry-core>=1.0.0"] 19 | build-backend = "poetry.core.masonry.api" 20 | -------------------------------------------------------------------------------- /examples/earthquakes/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "earthquakes" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Beneath Systems "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.7" 9 | aiohttp = "^3.7.3" 10 | asyncio = "^3.4.3" 11 | beneath = "^1.4.1" 12 | 13 | [tool.poetry.dev-dependencies] 14 | pytest = "^5.2" 15 | black = "^20.8b1" 16 | 17 | [build-system] 18 | requires = ["poetry-core>=1.0.0"] 19 | build-backend = "poetry.core.masonry.api" 20 | -------------------------------------------------------------------------------- /examples/financial-reference-data/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "financial-reference-data" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["greenep12 "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.8" 9 | jupyter = "^1.0.0" 10 | beneath = "^1.4.0" 11 | pandas = "^1.3.0" 12 | numpy = "^1.21.0" 13 | 14 | [tool.poetry.dev-dependencies] 15 | 16 | [build-system] 17 | requires = ["poetry-core>=1.0.0"] 18 | build-backend = "poetry.core.masonry.api" 19 | -------------------------------------------------------------------------------- /examples/reddit/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "reddit" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Beneath Systems "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.8" 9 | beneath = "^1.4.1" 10 | asyncpraw = "^7.3.1" 11 | asyncprawcore = "^2.2.1" 12 | tenacity = "^7.0.0" 13 | 14 | [tool.poetry.dev-dependencies] 15 | pytest = "^5.2" 16 | 17 | [build-system] 18 | requires = ["poetry-core>=1.0.0"] 19 | build-backend = "poetry.core.masonry.api" 20 | -------------------------------------------------------------------------------- /examples/beta/mock/derive.py: -------------------------------------------------------------------------------- 1 | """ 2 | To run this example: 3 | 1. Stage the pipeline 4 | python derive.py stage USERNAME/PROJECT/SERVICE 5 | 6 | 2. Run the pipeline 7 | python derive.py run USERNAME/PROJECT/SERVICE 8 | """ 9 | 10 | TABLE_PATH = "epg/sandbox/mock" 11 | 12 | import beneath 13 | 14 | p = beneath.Pipeline(parse_args=True) 15 | 16 | async def fn(data): 17 | print(data) 18 | 19 | data = p.read_table(TABLE_PATH) 20 | 21 | p.apply(data, fn) 22 | 23 | p.main() 24 | 25 | -------------------------------------------------------------------------------- /infra/README.md: -------------------------------------------------------------------------------- 1 | # `infra/` 2 | 3 | This folder contains packages for connecting to the various *external* infrastructure components that Beneath relies upon, namely: 4 | 5 | - `infra/db/` for connecting to Postgres 6 | - `infra/engine/` for connecting to log, index and warehouse data services (including drivers for specific systems) 7 | - `infra/mq/` for connecting to an at-least-once-delivery message queue (including drivers for specific systems) 8 | - `infra/redis/` for connecting to Redis 9 | 10 | -------------------------------------------------------------------------------- /clients/java/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # CONTRIBUTING 2 | 3 | ### Publishing to Maven Central 4 | 5 | - Increment the version number in `build.gradle` and `Config.java` 6 | - Set all the `gradle.properties` variables referenced in `build.gradle`. Follow this guide to get help: https://madhead.me/posts/no-bullshit-maven-publish/ 7 | - Run `./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository` 8 | - Update the config of recommended and deprecated versions in `services/data/clientversion/clientversion.go` 9 | -------------------------------------------------------------------------------- /clients/java/src/main/java/dev/beneath/client/InstanceRecordAndSize.java: -------------------------------------------------------------------------------- 1 | package dev.beneath.client; 2 | 3 | import org.apache.avro.generic.GenericRecord; 4 | 5 | public class InstanceRecordAndSize { 6 | public TableInstance instance; 7 | public GenericRecord record; 8 | public Integer size; 9 | 10 | InstanceRecordAndSize(TableInstance instance, GenericRecord record, Integer size) { 11 | this.instance = instance; 12 | this.record = record; 13 | this.size = size; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /clients/java/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # from pulsar example 2 | 3 | # Root logger option 4 | log4j.rootLogger=INFO, stdout 5 | 6 | # Redirect log messages to console 7 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 8 | log4j.appender.stdout.Target=System.out 9 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 10 | log4j.appender.stdout.layout.ConversionPattern=[%d{yyyy-MM-dd}T%d{HH:mm:ss.SSS}Z] %p %t %c{1}:%L %m%n 11 | log4j.logger.io.debezium.examples.apache.pulsar=DEBUG -------------------------------------------------------------------------------- /ee/cloud/deployments/kube/cert-manager/cluster-issuer-prod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1 2 | kind: ClusterIssuer 3 | metadata: 4 | name: letsencrypt-prod 5 | namespace: cert-manager 6 | spec: 7 | acme: 8 | email: benjamin@beneath.dev 9 | server: https://acme-v02.api.letsencrypt.org/directory 10 | preferredChain: "ISRG Root X1" 11 | privateKeySecretRef: 12 | name: letsencrypt-prod 13 | solvers: 14 | - http01: 15 | ingress: 16 | class: nginx 17 | -------------------------------------------------------------------------------- /pkg/ws/ws.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package ws provides a modified implementation of the Websockets-based 3 | RPC protocol used in [apollographql/subscriptions-transport-ws](https://github.com/apollographql/subscriptions-transport-ws/blob/master/PROTOCOL.md). 4 | 5 | The main difference is that this implementation accepts any query payload, not just GraphQL queries. 6 | 7 | If we ever need to increase performance, this [blog post](https://github.com/eranyanay/1m-go-websockets) was very inspiring. 8 | */ 9 | package ws 10 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | This directory contains all the documentation files display on [https://about.beneath.dev/docs/](https://about.beneath.dev/docs/). 2 | 3 | ### Read before editing 4 | 5 | - Each folder becomes a section of the documentation 6 | - The front-matter in the `_index.md` file gives the title and weight (order) of the section 7 | - The body in `_index.md` becomes the content for the section overview page 8 | - The `menu.docs.parent` key in the front matter of every non-index file must match the parent folder 9 | -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/scores/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "scores" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["greenep12 "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.8" 9 | beneath = "^1.4.2" 10 | asyncio = "^3.4.3" 11 | asyncpraw = "^7.3.0" 12 | asyncprawcore = "^2.2.0" 13 | tenacity = "^7.0.0" 14 | 15 | [tool.poetry.dev-dependencies] 16 | 17 | [build-system] 18 | requires = ["poetry-core>=1.0.0"] 19 | build-backend = "poetry.core.masonry.api" 20 | -------------------------------------------------------------------------------- /pkg/bytesutil/bytesutil.go: -------------------------------------------------------------------------------- 1 | package bytesutil 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | ) 7 | 8 | // IntToBytes encodes an int to bytes representation 9 | func IntToBytes(x int64) []byte { 10 | buf := new(bytes.Buffer) 11 | err := binary.Write(buf, binary.BigEndian, x) 12 | if err != nil { 13 | panic(err) 14 | } 15 | return buf.Bytes() 16 | } 17 | 18 | // BytesToInt decodes an int encoded with IntToBytes 19 | func BytesToInt(b []byte) int64 { 20 | return int64(binary.BigEndian.Uint64(b)) 21 | } 22 | -------------------------------------------------------------------------------- /web/apollo/types/RevokeUserSecret.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: RevokeUserSecret 8 | // ==================================================== 9 | 10 | export interface RevokeUserSecret { 11 | revokeUserSecret: boolean; 12 | } 13 | 14 | export interface RevokeUserSecretVariables { 15 | secretID: ControlUUID; 16 | } 17 | -------------------------------------------------------------------------------- /clients/js/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | node: true, 4 | browser: true, 5 | }, 6 | parser: "@typescript-eslint/parser", 7 | parserOptions: { 8 | ecmaVersion: 10, 9 | sourceType: "module", 10 | }, 11 | plugins: ["@typescript-eslint"], 12 | extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"], 13 | rules: { 14 | "no-unused-vars": "off", 15 | "@typescript-eslint/no-unused-vars": "off", 16 | "@typescript-eslint/no-explicit-any": "off", 17 | }, 18 | }; 19 | -------------------------------------------------------------------------------- /clients/js-react/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | node: true, 4 | browser: true, 5 | }, 6 | parser: "@typescript-eslint/parser", 7 | parserOptions: { 8 | ecmaVersion: 10, 9 | sourceType: "module", 10 | }, 11 | plugins: ["@typescript-eslint"], 12 | extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"], 13 | rules: { 14 | "no-unused-vars": "off", 15 | "@typescript-eslint/no-unused-vars": "off", 16 | "@typescript-eslint/no-explicit-any": "off", 17 | }, 18 | }; 19 | -------------------------------------------------------------------------------- /ee/server/control/schema/billed_resources.graphql: -------------------------------------------------------------------------------- 1 | extend type Query { 2 | billedResources(organizationID: UUID!, fromBillingTime: Time!, toBillingTime: Time!): [BilledResource!]! 3 | } 4 | 5 | type BilledResource { 6 | billedResourceID: ID! 7 | organizationID: UUID! 8 | billingTime: Time! 9 | entityID: UUID! 10 | entityKind: String! 11 | startTime: Time! 12 | endTime: Time! 13 | product: String! 14 | quantity: Float! 15 | totalPriceCents: Int! 16 | currency: String! 17 | createdOn: Time! 18 | updatedOn: Time! 19 | } 20 | -------------------------------------------------------------------------------- /web/apollo/types/RevokeServiceSecret.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: RevokeServiceSecret 8 | // ==================================================== 9 | 10 | export interface RevokeServiceSecret { 11 | revokeServiceSecret: boolean; 12 | } 13 | 14 | export interface RevokeServiceSecretVariables { 15 | secretID: ControlUUID; 16 | } 17 | -------------------------------------------------------------------------------- /web/apollo/types/DeleteTableInstance.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: DeleteTableInstance 8 | // ==================================================== 9 | 10 | export interface DeleteTableInstance { 11 | deleteTableInstance: boolean; 12 | } 13 | 14 | export interface DeleteTableInstanceVariables { 15 | instanceID: ControlUUID; 16 | } 17 | -------------------------------------------------------------------------------- /ee/migrations/004_nullable_billing_method.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // BillingInfo.BillingMethodID 10 | _, err = db.Exec(` 11 | ALTER TABLE billing_infos 12 | ALTER COLUMN billing_method_id DROP NOT NULL; 13 | `) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | // Done 19 | return nil 20 | }, func(db migrations.DB) (err error) { 21 | // Done 22 | return nil 23 | }) 24 | } 25 | -------------------------------------------------------------------------------- /examples/beta/lending-club/loans-enriched/requirements.txt: -------------------------------------------------------------------------------- 1 | aiogrpc==1.7 2 | aiohttp==3.6.2 3 | async-timeout==3.0.1 4 | attrs==19.3.0 5 | beneath==1.2.2 6 | chardet==3.0.4 7 | Cython==0.29.15 8 | fastavro==0.21.24 9 | grpcio==1.27.2 10 | idna==2.10 11 | joblib==0.16.0 12 | msgpack==1.0.0 13 | multidict==4.7.6 14 | numpy==1.19.1 15 | pandas==1.0.1 16 | protobuf==3.11.3 17 | python-dateutil==2.8.1 18 | pytz==2020.1 19 | scikit-learn==0.23.1 20 | scipy==1.5.2 21 | six==1.14.0 22 | sklearn==0.0 23 | threadpoolctl==2.1.0 24 | typing-extensions==3.7.4.2 25 | yarl==1.5.0 -------------------------------------------------------------------------------- /examples/beta/lending-club/loans/requirements.txt: -------------------------------------------------------------------------------- 1 | aiogrpc==1.7 2 | aiohttp==3.6.2 3 | async-timeout==3.0.1 4 | attrs==19.3.0 5 | beneath==1.2.1 6 | certifi==2020.6.20 7 | chardet==3.0.4 8 | Cython==0.29.15 9 | DateTime==4.3 10 | fastavro==0.21.24 11 | grpcio==1.27.2 12 | idna==2.10 13 | msgpack==1.0.0 14 | multidict==4.7.6 15 | numpy==1.19.1 16 | pandas==1.0.1 17 | protobuf==3.11.3 18 | python-dateutil==2.8.1 19 | pytz==2020.1 20 | requests==2.24.0 21 | six==1.14.0 22 | typing-extensions==3.7.4.2 23 | urllib3==1.25.10 24 | yarl==1.5.0 25 | zope.interface==5.1.0 -------------------------------------------------------------------------------- /pkg/schemalang/transpilers/avro_test.go: -------------------------------------------------------------------------------- 1 | package transpilers 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/beneath-hq/beneath/pkg/schemalang" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestAvro(t *testing.T) { 11 | s, err := FromAvro(UserSchemaAvroWithDoc) 12 | assert.Nil(t, err) 13 | 14 | err = schemalang.Check(s) 15 | assert.Nil(t, err) 16 | 17 | avro := ToAvro(s, false) 18 | assert.Equal(t, UserSchemaAvro, avro) 19 | 20 | avroWithDoc := ToAvro(s, true) 21 | assert.Equal(t, UserSchemaAvroWithDoc, avroWithDoc) 22 | } 23 | -------------------------------------------------------------------------------- /services/secret/README.md: -------------------------------------------------------------------------------- 1 | # `services/secret/` 2 | 3 | This service manages secrets (in other apps also known as access tokens or API keys). Secrets either belong to a user (`models.UserSecret`) or a service (`models.ServiceSecret`). Secrets are used to authenticate requests (including normal frontend requests). 4 | 5 | The package includes a cache for checking secrets that can be used to authenticate every request. It returns a `models.Secret` interface, which handily includes several owner-related fields joined from other tables (eg. the secret owner's quotas). 6 | -------------------------------------------------------------------------------- /web/components/Loading.tsx: -------------------------------------------------------------------------------- 1 | import CircularProgress from "@material-ui/core/CircularProgress"; 2 | import Grid, { GridJustification } from "@material-ui/core/Grid"; 3 | import { FC } from "react"; 4 | 5 | export interface LoadingProps { 6 | justify?: GridJustification; 7 | size?: number | string; 8 | } 9 | 10 | export const Loading: FC = (props) => ( 11 | 12 | 13 | 14 | 15 | 16 | ); 17 | 18 | export default Loading; 19 | -------------------------------------------------------------------------------- /web/components/organization/ViewSecurity.tsx: -------------------------------------------------------------------------------- 1 | import { Container } from "@material-ui/core"; 2 | import React, { FC } from "react"; 3 | 4 | import ListSessions from "components/organization/secrets/ListSessions"; 5 | 6 | export interface ViewBrowserSessionsProps { 7 | userID: string; 8 | } 9 | 10 | export const ViewBrowserSessions: FC = ({ userID }) => { 11 | return ( 12 | 13 | 14 | 15 | ); 16 | }; 17 | 18 | export default ViewBrowserSessions; 19 | -------------------------------------------------------------------------------- /infra/engine/driver/mock/util.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "github.com/beneath-hq/beneath/infra/engine/driver" 5 | ) 6 | 7 | type mockRecordsIterator struct{} 8 | 9 | // Next implements driver.RecordsIterator 10 | func (m mockRecordsIterator) Next() bool { 11 | return false 12 | } 13 | 14 | // Record implements driver.RecordsIterator 15 | func (m mockRecordsIterator) Record() driver.Record { 16 | return nil 17 | } 18 | 19 | // NextCursor implements driver.RecordsIterator 20 | func (m mockRecordsIterator) NextCursor() []byte { 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /web/components/VSpace.tsx: -------------------------------------------------------------------------------- 1 | import { makeStyles, Theme } from "@material-ui/core/styles"; 2 | import { FC } from "react"; 3 | 4 | type VSpaceProps = { 5 | units?: number; 6 | }; 7 | 8 | const useStyles = makeStyles((theme) => ({ 9 | space: ({ units }) => ({ 10 | display: "block", 11 | paddingTop: theme.spacing(units || 1), 12 | }), 13 | })); 14 | 15 | const VSpace: FC = ({ units }) => { 16 | const classes = useStyles({ units }); 17 | return
; 18 | }; 19 | 20 | export default VSpace; 21 | -------------------------------------------------------------------------------- /clients/python/tools/proto-build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # check we're in the client repo 4 | if [[ ! -f tools/$(basename $0) ]]; then 5 | echo "you must execute this script from the python client root folder (clients/python)" 6 | exit 1 7 | fi 8 | 9 | # copy canonical grpc proto file into client 10 | PROTO_SRC=../../gateway/grpc/proto/gateway.proto 11 | PROTO_DST=beneath/proto/gateway.proto 12 | cp $PROTO_SRC $PROTO_DST 13 | 14 | # generate python bindings 15 | python -m grpc_tools.protoc -I . --python_out=. --grpc_python_out=. ./beneath/proto/gateway.proto 16 | -------------------------------------------------------------------------------- /server/control/resolver/stream_index.go: -------------------------------------------------------------------------------- 1 | package resolver 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/beneath-hq/beneath/models" 7 | "github.com/beneath-hq/beneath/server/control/gql" 8 | ) 9 | 10 | // TableIndex returns the gql.TableIndexResolver 11 | func (r *Resolver) TableIndex() gql.TableIndexResolver { 12 | return &tableIndexResolver{r} 13 | } 14 | 15 | type tableIndexResolver struct{ *Resolver } 16 | 17 | func (r *tableIndexResolver) IndexID(ctx context.Context, obj *models.TableIndex) (string, error) { 18 | return obj.TableIndexID.String(), nil 19 | } 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | .cache 3 | .DS_Store 4 | config/.* 5 | .history 6 | .idea 7 | .ipynb_checkpoints 8 | .next 9 | .Python 10 | .tox 11 | .venv 12 | *.code-workspace 13 | *.egg-info 14 | *.pyc 15 | *.pyd 16 | *.pyo 17 | dist 18 | node_modules 19 | npm-debug.log 20 | pip-delete-this-directory.txt 21 | pip-log.txt 22 | temp 23 | tmp 24 | TODO 25 | venv 26 | yarn-error.log 27 | .vscode/launch.json 28 | .vscode/.ropeproject 29 | */.vscode/launch.json 30 | */.vscode/.ropeproject 31 | .pytest_cache 32 | beneath-models/.venv 33 | clients/**/build 34 | /public 35 | examples/**/yarn.lock -------------------------------------------------------------------------------- /clients/java/src/main/java/dev/beneath/client/admin/BaseResource.java: -------------------------------------------------------------------------------- 1 | package dev.beneath.client.admin; 2 | 3 | import dev.beneath.client.Connection; 4 | 5 | public abstract class BaseResource { 6 | protected final Connection conn; 7 | protected final Boolean dry; 8 | 9 | protected BaseResource(Connection conn, Boolean dry) { 10 | this.conn = conn; 11 | this.dry = dry; 12 | } 13 | 14 | protected void beforeMutation() { 15 | if (this.dry) { 16 | throw new RuntimeException("Cannot run mutation on a client where dry=True"); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /examples/reddit/deployments/README.md: -------------------------------------------------------------------------------- 1 | To set all the environment variables as one Kubernetes secret: 2 | 3 | ```bash 4 | kubectl create secret generic SUBREDDIT-scraper -n models \ 5 | --from-literal=beneath-secret=BENEATH_SECRET \ 6 | --from-literal=reddit-user-agent=REDDIT_USER_AGENT \ 7 | --from-literal=reddit-client-id=REDDIT_CLIENT_ID \ 8 | --from-literal=reddit-client-secret=REDDIT_CLIENT_SECRET \ 9 | --from-literal=reddit-username=REDDIT_USERNAME \ 10 | --from-literal=reddit-password=REDDIT_PASSWORD \ 11 | --from-literal=reddit-subreddit=REDDIT_SUBREDDIT 12 | ``` 13 | -------------------------------------------------------------------------------- /ee/cloud/deployments/helm/web/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "web.fullname" . }} 5 | labels: 6 | app.kubernetes.io/name: {{ include "web.name" . }} 7 | app.kubernetes.io/instance: {{ .Release.Name }} 8 | app.kubernetes.io/managed-by: {{ .Release.Service }} 9 | helm.sh/chart: {{ include "web.chart" . }} 10 | spec: 11 | ports: 12 | - port: 8080 13 | targetPort: web-port 14 | protocol: TCP 15 | selector: 16 | app.kubernetes.io/name: {{ include "web.name" . }} 17 | app.kubernetes.io/instance: {{ .Release.Name }} 18 | -------------------------------------------------------------------------------- /examples/beta/earthquake-notifications/requirements.txt: -------------------------------------------------------------------------------- 1 | aiogrpc==1.8 2 | aiohttp==3.7.2 3 | async-timeout==3.0.1 4 | attrs==20.2.0 5 | beneath==1.2.7 6 | certifi==2020.6.20 7 | chardet==3.0.4 8 | Cython==0.29.21 9 | DateTime==4.3 10 | fastavro==1.0.0.post1 11 | grpcio==1.27.2 12 | idna==2.10 13 | msgpack==1.0.0 14 | multidict==5.0.0 15 | numpy==1.19.2 16 | pandas==1.1.3 17 | protobuf==3.13.0 18 | PyJWT==1.7.1 19 | python-dateutil==2.8.1 20 | pytz==2020.1 21 | requests==2.24.0 22 | six==1.15.0 23 | twilio==6.46.0 24 | typing-extensions==3.7.4.3 25 | urllib3==1.25.11 26 | yarl==1.6.2 27 | zope.interface==5.1.2 28 | -------------------------------------------------------------------------------- /server/control/resolver/stream_instance.go: -------------------------------------------------------------------------------- 1 | package resolver 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/beneath-hq/beneath/models" 7 | "github.com/beneath-hq/beneath/server/control/gql" 8 | ) 9 | 10 | // TableInstance returns the gql.TableInstanceResolver 11 | func (r *Resolver) TableInstance() gql.TableInstanceResolver { 12 | return &tableInstanceResolver{r} 13 | } 14 | 15 | type tableInstanceResolver struct{ *Resolver } 16 | 17 | func (r *tableInstanceResolver) TableInstanceID(ctx context.Context, obj *models.TableInstance) (string, error) { 18 | return obj.TableInstanceID.String(), nil 19 | } 20 | -------------------------------------------------------------------------------- /clients/python/beneath/beam/read_from_beneath.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module implements reading data from Beneath into a Beam pipeline 3 | """ 4 | 5 | import apache_beam as beam 6 | 7 | 8 | class ReadFromBeneath(beam.PTransform): 9 | def __init__(self, table): 10 | self.table = table 11 | 12 | def expand(self, pvalue): 13 | table = self.table 14 | query = "select * from `{}`".format(table.bigquery_table) 15 | source = beam.io.BigQuerySource(query=query, use_standard_sql=True) 16 | # can probably get query schema with source.schema 17 | return pvalue | beam.io.Read(source) 18 | -------------------------------------------------------------------------------- /migrations/README.md: -------------------------------------------------------------------------------- 1 | # `migrations/` 2 | 3 | This folder contains control-level migrations for Postgres. 4 | 5 | All migrations should be SQL-based, and not use `go-pg`s features for automatically creating tables based on structs (since that makes future changes harder). 6 | 7 | Migrations can be run and manipulated using the backend CLI, run `go run cmd/beneath/main.go migrate -h` for details. Migrations currently also auto-run when a control server is started. Migrations are tracked in the table `gopg_migrations`. 8 | 9 | We use the [`go-pg` migrations library](https://github.com/go-pg/migrations/v7) to run migrations. 10 | -------------------------------------------------------------------------------- /pkg/ctxutil/ctxutil.go: -------------------------------------------------------------------------------- 1 | package ctxutil 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | ) 9 | 10 | // WithCancelOnTerminate derives a context that is cancelled on SIGINT and SIGTERM signals 11 | func WithCancelOnTerminate(ctx context.Context) context.Context { 12 | ctx, cancel := context.WithCancel(ctx) 13 | 14 | // handles SIGINT and SIGTERM gracefully 15 | go func() { 16 | defer cancel() 17 | c := make(chan os.Signal, 1) 18 | signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) 19 | select { 20 | case <-c: 21 | case <-ctx.Done(): 22 | } 23 | }() 24 | 25 | return ctx 26 | } 27 | -------------------------------------------------------------------------------- /server/data/grpc/read.go: -------------------------------------------------------------------------------- 1 | package grpc 2 | 3 | import ( 4 | "context" 5 | 6 | pb "github.com/beneath-hq/beneath/server/data/grpc/proto" 7 | "github.com/beneath-hq/beneath/services/data" 8 | ) 9 | 10 | func (s *gRPCServer) Read(ctx context.Context, req *pb.ReadRequest) (*pb.ReadResponse, error) { 11 | res, err := s.Service.HandleRead(ctx, &data.ReadRequest{ 12 | Cursor: req.Cursor, 13 | Limit: req.Limit, 14 | ReturnPB: true, 15 | }) 16 | if err != nil { 17 | return nil, err.GRPC() 18 | } 19 | 20 | return &pb.ReadResponse{ 21 | Records: res.PB, 22 | NextCursor: res.NextCursor, 23 | }, nil 24 | } 25 | -------------------------------------------------------------------------------- /clients/python/beneath/admin/secrets.py: -------------------------------------------------------------------------------- 1 | from beneath.admin.base import _ResourceBase 2 | 3 | 4 | class Secrets(_ResourceBase): 5 | async def revoke_service_secret(self, secret_id): 6 | self._before_mutation() 7 | result = await self.conn.query_control( 8 | variables={ 9 | "secretID": secret_id, 10 | }, 11 | query=""" 12 | mutation RevokeServiceSecret($secretID: UUID!) { 13 | revokeServiceSecret(secretID: $secretID) 14 | } 15 | """, 16 | ) 17 | return result["revokeServiceSecret"] 18 | -------------------------------------------------------------------------------- /examples/earthquakes/main.py: -------------------------------------------------------------------------------- 1 | import beneath 2 | 3 | from generators import earthquakes 4 | 5 | with open("schemas/earthquake.graphql", "r") as file: 6 | EARTHQUAKES_SCHEMA = file.read() 7 | 8 | if __name__ == "__main__": 9 | p = beneath.Pipeline(parse_args=True) 10 | p.description = "Continually pings the USGS earthquake API" 11 | earthquakes = p.generate(earthquakes.generate_earthquakes) 12 | p.write_table( 13 | earthquakes, 14 | "earthquakes", 15 | schema=EARTHQUAKES_SCHEMA, 16 | description="Earthquakes fetched from https://earthquake.usgs.gov/", 17 | ) 18 | p.main() 19 | -------------------------------------------------------------------------------- /migrations/045_streams_meta.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | _, err = db.Exec(` 10 | ALTER TABLE streams 11 | ADD meta boolean not null default false; 12 | `) 13 | if err != nil { 14 | return err 15 | } 16 | 17 | // Done 18 | return nil 19 | }, func(db migrations.DB) (err error) { 20 | _, err = db.Exec(` 21 | ALTER TABLE streams 22 | DROP meta; 23 | `) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | // Done 29 | return nil 30 | }) 31 | } 32 | -------------------------------------------------------------------------------- /web/ee/scripts/generateApolloTypes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Generate ee/apollo/possibleTypes.ts 4 | yarn run ts-node ./scripts/apolloGeneratePossibleTypes.ts "/ee/graphql" "ee/apollo/possibleTypes.json" 5 | 6 | # Generate Typescript types for our Apollo results in apollo/types/ 7 | yarn run apollo codegen:generate \ 8 | --config ./ee/apollo.config.js \ 9 | --outputFlat ee/apollo/types \ 10 | --target typescript \ 11 | --passthroughCustomScalars \ 12 | --customScalarsPrefix Control 13 | 14 | # The generated globalTypes.ts is currently empty, causing an isolatedModules error; delete it 15 | rm ./ee/apollo/types/globalTypes.ts 16 | -------------------------------------------------------------------------------- /.github/workflows/mirror.yml: -------------------------------------------------------------------------------- 1 | name: Gitlab Mirror 2 | 3 | on: 4 | push: 5 | branches: 6 | - stable 7 | 8 | jobs: 9 | mirror: 10 | name: Mirror 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | with: 15 | fetch-depth: 0 16 | - run: | 17 | git remote add target https://$USERNAME:$TOKEN@$URL 18 | git push -f --all target 19 | git push -f --tags target 20 | env: 21 | URL: ${{ secrets.GITLAB_TARGET_URL }} 22 | USERNAME: ${{ secrets.GITLAB_TARGET_USERNAME }} 23 | TOKEN: ${{ secrets.GITLAB_TARGET_TOKEN }} 24 | -------------------------------------------------------------------------------- /clients/java/src/main/graphql/dev/beneath/client/ProjectByOrganizationAndName.graphql: -------------------------------------------------------------------------------- 1 | query ProjectByOrganizationAndName( 2 | $organizationName: String! 3 | $projectName: String! 4 | ) { 5 | projectByOrganizationAndName( 6 | organizationName: $organizationName 7 | projectName: $projectName 8 | ) { 9 | projectID 10 | name 11 | displayName 12 | site 13 | description 14 | photoURL 15 | public 16 | createdOn 17 | updatedOn 18 | tables { 19 | name 20 | } 21 | services { 22 | name 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /web/components/organization/ViewSecrets.tsx: -------------------------------------------------------------------------------- 1 | import { Box, Container, Paper, Typography, useTheme } from "@material-ui/core"; 2 | import React, { FC } from "react"; 3 | 4 | import IssueSecret from "./secrets/IssueSecret"; 5 | import ListSecrets from "./secrets/ListSecrets"; 6 | 7 | export interface ViewSecretsProps { 8 | userID: string; 9 | } 10 | 11 | const ViewSecrets: FC = ({ userID }) => { 12 | return ( 13 | 14 | 15 | 16 | 17 | 18 | ); 19 | }; 20 | 21 | export default ViewSecrets; 22 | -------------------------------------------------------------------------------- /cmd/beneath/dependencies/redis.go: -------------------------------------------------------------------------------- 1 | package dependencies 2 | 3 | import ( 4 | "github.com/spf13/viper" 5 | 6 | "github.com/beneath-hq/beneath/cmd/beneath/cli" 7 | "github.com/beneath-hq/beneath/infra/redis" 8 | ) 9 | 10 | func init() { 11 | cli.AddDependency(redis.NewRedis) 12 | cli.AddDependency(func(v *viper.Viper) (*redis.Options, error) { 13 | var opts redis.Options 14 | return &opts, v.UnmarshalKey("control.redis", &opts) 15 | }) 16 | cli.AddConfigKey(&cli.ConfigKey{ 17 | Key: "control.redis.url", 18 | Default: "redis://localhost/", 19 | Description: "Redis connection URL for control-plane caching", 20 | }) 21 | 22 | } 23 | -------------------------------------------------------------------------------- /web/ee/apollo/typePolicies.ts: -------------------------------------------------------------------------------- 1 | import { TypePolicies } from "@apollo/client"; 2 | 3 | // Data normalization config. See https://www.apollographql.com/docs/react/caching/cache-configuration/#data-normalization 4 | // NOTE: setting keyFields to false causes objects to be embedded in the entry of their parent object, see: https://www.apollographql.com/docs/react/caching/cache-configuration/#disabling-normalization 5 | 6 | const typePolicies: TypePolicies = { 7 | BillingInfo: { keyFields: ["organizationID"] }, 8 | BillingMethod: { keyFields: ["billingMethodID"] }, 9 | BillingPlan: { keyFields: ["billingPlanID"] }, 10 | }; 11 | 12 | export default typePolicies; 13 | -------------------------------------------------------------------------------- /ee/server/control/schema/billing_plans.graphql: -------------------------------------------------------------------------------- 1 | extend type Query { 2 | billingPlans: [BillingPlan!]! 3 | } 4 | 5 | type BillingPlan { 6 | billingPlanID: ID! 7 | default: Boolean! 8 | name: String! 9 | description: String 10 | currency: String! 11 | period: String! 12 | basePriceCents: Int! 13 | seatPriceCents: Int! 14 | baseReadQuota: Int! 15 | baseWriteQuota: Int! 16 | baseScanQuota: Int! 17 | seatReadQuota: Int! 18 | seatWriteQuota: Int! 19 | seatScanQuota: Int! 20 | readQuota: Int! 21 | writeQuota: Int! 22 | scanQuota: Int! 23 | readOveragePriceCents: Int! 24 | writeOveragePriceCents: Int! 25 | scanOveragePriceCents: Int! 26 | UIRank: Int 27 | } 28 | -------------------------------------------------------------------------------- /migrations/021_master_users.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // User.Master 10 | _, err = db.Exec(` 11 | ALTER TABLE users 12 | ADD master boolean NOT NULL DEFAULT false; 13 | `) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | // Done 19 | return nil 20 | }, func(db migrations.DB) (err error) { 21 | // User.Master 22 | _, err = db.Exec(` 23 | ALTER TABLE users 24 | DROP master; 25 | `) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | // Done 31 | return nil 32 | }) 33 | } 34 | -------------------------------------------------------------------------------- /ee/cloud/deployments/helm/backend/templates/ctrl-server-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "backend.ctrlserver.fullname" . }} 5 | labels: 6 | app.kubernetes.io/name: {{ include "backend.ctrlserver.fullname" . }} 7 | app.kubernetes.io/instance: {{ .Release.Name }} 8 | app.kubernetes.io/managed-by: {{ .Release.Service }} 9 | helm.sh/chart: {{ include "backend.chart" . }} 10 | spec: 11 | ports: 12 | - port: 8080 13 | targetPort: ctrl-port 14 | protocol: TCP 15 | selector: 16 | app.kubernetes.io/name: {{ include "backend.ctrlserver.fullname" . }} 17 | app.kubernetes.io/instance: {{ .Release.Name }} 18 | -------------------------------------------------------------------------------- /ee/services/billing/run.go: -------------------------------------------------------------------------------- 1 | package billing 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | ) 7 | 8 | // RunBilling runs billing for every customer with next_billing_time < now 9 | func (s *Service) RunBilling(ctx context.Context) error { 10 | s.Logger.Infof("running billing") 11 | 12 | bis := s.FindBillingInfosToRun(ctx) 13 | for idx, bi := range bis { 14 | s.Logger.Infow(fmt.Sprintf("running billing %d/%d", idx, len(bis)), "organization_id", bi.OrganizationID.String()) 15 | err := s.CommitBilledResources(ctx, bi) 16 | if err != nil { 17 | return err 18 | } 19 | } 20 | 21 | s.Logger.Infof("completed billing for %d customers", len(bis)) 22 | return nil 23 | } 24 | -------------------------------------------------------------------------------- /examples/beta/reddit-sentiment/requirements.txt: -------------------------------------------------------------------------------- 1 | aiogrpc>=1.7 2 | aiohttp>=3.6.2 3 | async-timeout>=3.0.1 4 | attrs>=19.3.0 5 | beneath>=1.1.5 6 | certifi>=2020.4.5.1 7 | chardet>=3.0.4 8 | click>=7.1.2 9 | Cython==0.29.15 10 | fastavro==0.21.24 11 | grpcio==1.27.2 12 | idna>=2.9 13 | joblib>=0.15.1 14 | multidict>=4.7.6 15 | nltk>=3.5 16 | numpy>=1.18.5 17 | pandas==1.0.1 18 | praw>=7.0.0 19 | prawcore>=1.4.0 20 | protobuf==3.11.3 21 | python-dateutil>=2.8.1 22 | pytz>=2020.1 23 | regex>=2020.5.14 24 | requests>=2.23.0 25 | six==1.14.0 26 | structlog>=20.1.0 27 | textblob>=0.15.3 28 | tqdm>=4.46.1 29 | update-checker>=0.17 30 | urllib3>=1.25.9 31 | websocket-client>=0.57.0 32 | yarl>=1.4.2 -------------------------------------------------------------------------------- /web/apollo/types/CreateRecords.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: CreateRecords 8 | // ==================================================== 9 | 10 | export interface CreateRecords_createRecords { 11 | __typename: "CreateRecordsResponse"; 12 | error: string | null; 13 | } 14 | 15 | export interface CreateRecords { 16 | createRecords: CreateRecords_createRecords; 17 | } 18 | 19 | export interface CreateRecordsVariables { 20 | instanceID: ControlUUID; 21 | json: ControlJSON; 22 | } 23 | -------------------------------------------------------------------------------- /web/ee/apollo/queries/billingPlan.ts: -------------------------------------------------------------------------------- 1 | import gql from "graphql-tag"; 2 | 3 | export const QUERY_BILLING_PLANS = gql` 4 | query BillingPlans{ 5 | billingPlans { 6 | billingPlanID 7 | default 8 | name 9 | description 10 | currency 11 | period 12 | basePriceCents 13 | seatPriceCents 14 | baseReadQuota 15 | baseWriteQuota 16 | baseScanQuota 17 | seatReadQuota 18 | seatWriteQuota 19 | seatScanQuota 20 | readQuota 21 | writeQuota 22 | scanQuota 23 | readOveragePriceCents 24 | writeOveragePriceCents 25 | scanOveragePriceCents 26 | UIRank 27 | } 28 | } 29 | `; 30 | -------------------------------------------------------------------------------- /examples/beta/lending-club/loans-enriched/kube.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1beta1 2 | kind: CronJob 3 | metadata: 4 | name: lending-club-loans-enriched 5 | spec: 6 | schedule: "1 9,13,17,21 * * *" 7 | jobTemplate: 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | - name: lending-club-loans-enriched 13 | image: gcr.io/beneath/lending-club-loans-enriched:latest 14 | env: 15 | - name: BENEATH_SECRET 16 | valueFrom: 17 | secretKeyRef: 18 | name: lending-club-loans-enriched-service-secret 19 | key: secret 20 | restartPolicy: OnFailure 21 | -------------------------------------------------------------------------------- /server/data/grpc/ping.go: -------------------------------------------------------------------------------- 1 | package grpc 2 | 3 | import ( 4 | "context" 5 | 6 | pb "github.com/beneath-hq/beneath/server/data/grpc/proto" 7 | "github.com/beneath-hq/beneath/services/data" 8 | ) 9 | 10 | func (s *gRPCServer) Ping(ctx context.Context, req *pb.PingRequest) (*pb.PingResponse, error) { 11 | res, err := s.Service.HandlePing(ctx, &data.PingRequest{ 12 | ClientID: req.ClientId, 13 | ClientVersion: req.ClientVersion, 14 | }) 15 | if err != nil { 16 | return nil, err.GRPC() 17 | } 18 | return &pb.PingResponse{ 19 | Authenticated: res.Authenticated, 20 | VersionStatus: res.VersionStatus, 21 | RecommendedVersion: res.RecommendedVersion, 22 | }, nil 23 | } 24 | -------------------------------------------------------------------------------- /ee/migrations/006_rm_entity_name.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // BilledResource.EntityName 10 | _, err = db.Exec(` 11 | ALTER TABLE billed_resources 12 | DROP entity_name; 13 | `) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | return nil 19 | }, func(db migrations.DB) (err error) { 20 | // BilledResource.EntityName 21 | _, err = db.Exec(` 22 | ALTER TABLE billed_resources 23 | ADD entity_name text NOT NULL default 'TODO'; 24 | `) 25 | if err != nil { 26 | return err 27 | } 28 | 29 | return nil 30 | }) 31 | } 32 | -------------------------------------------------------------------------------- /examples/beta/ethereum-known-addresses/known-addresses.graphql: -------------------------------------------------------------------------------- 1 | " Manually curated table of known Ethereum addresses (includes both external and smart contract addresses) " 2 | type KnownAddress @table @key(fields: ["address"]) { 3 | " Address on the Ethereum mainnet " 4 | address: Bytes20! 5 | 6 | " The category that most suitably applies to the address " 7 | category: String! 8 | 9 | " The name of the project or company that the address belongs to " 10 | entity: String! 11 | 12 | " A brief description of the purpose of the address within the context of its name " 13 | description: String 14 | 15 | " A note describing possible evidence of the identity " 16 | note: String 17 | } 18 | -------------------------------------------------------------------------------- /examples/cdc/fanout/test/test_schemas.py: -------------------------------------------------------------------------------- 1 | import os, inspect, sys 2 | 3 | # add parent directory to path, so I can import from the main.py module 4 | currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) 5 | parentdir = os.path.dirname(currentdir) 6 | sys.path.insert(0, parentdir) 7 | 8 | from main import connect_to_source_db 9 | from operate_on_source_db import create_table_compound_pk, drop_table 10 | from schemas import get_schema 11 | 12 | if __name__ == "__main__": 13 | cursor = connect_to_source_db() 14 | drop_table(cursor, "table1") 15 | create_table_compound_pk(cursor, "table1") 16 | 17 | schema = get_schema(cursor, "table1") 18 | print(schema) -------------------------------------------------------------------------------- /ee/cloud/deployments/helm/backend_values.yaml: -------------------------------------------------------------------------------- 1 | image: 2 | pullPolicy: IfNotPresent 3 | repository: gcr.io/beneath/ee-backend 4 | tag: latest 5 | 6 | configSecretName: backend-config 7 | configFileName: production.yaml 8 | 9 | ctrl: 10 | server: 11 | host: control.beneath.dev 12 | resources: {} 13 | worker: 14 | resources: {} 15 | 16 | data: 17 | server: 18 | httpHost: data.beneath.dev 19 | grpcHost: grpc.data.beneath.dev 20 | 21 | extraEnv: 22 | - name: GOOGLE_APPLICATION_CREDENTIALS 23 | value: /var/secrets/google/key.json 24 | - name: IS_GKE 25 | value: "1" 26 | 27 | extraSecretMounts: 28 | - secretName: beneath-sa-key 29 | mountPath: /var/secrets/google 30 | -------------------------------------------------------------------------------- /examples/clock-react/src/App.js: -------------------------------------------------------------------------------- 1 | import { useRecords } from "beneath-react"; 2 | 3 | const App = () => { 4 | const { records, loading, error } = useRecords({ 5 | table: "examples/clock/clock-1m", 6 | query: { type: "log", peek: true }, 7 | subscribe: true, 8 | }); 9 | 10 | if (loading) { 11 | return

Loading...

; 12 | } else if (error) { 13 | return

Error: {error}

; 14 | } 15 | 16 | return ( 17 |
18 |

Clock

19 |
    20 | {records.map((record) => ( 21 |
  • {new Date(record.time).toLocaleString()}
    • 22 | ))} 23 |
24 |
25 | ); 26 | }; 27 | 28 | export default App; 29 | -------------------------------------------------------------------------------- /migrations/041_featured_projects.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | _, err = db.Exec(` 10 | ALTER TABLE projects 11 | ADD explore_rank integer; 12 | CREATE INDEX ON projects (explore_rank) WHERE explore_rank IS NOT NULL; 13 | `) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | // Done 19 | return nil 20 | }, func(db migrations.DB) (err error) { 21 | _, err = db.Exec(` 22 | ALTER TABLE projects 23 | DROP explore_rank; 24 | `) 25 | if err != nil { 26 | return err 27 | } 28 | 29 | // Done 30 | return nil 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /config/drivers.yaml: -------------------------------------------------------------------------------- 1 | # NOTE: The config.yaml template does not contain driver-specific keys. 2 | # Instead they are listed in this file. 3 | 4 | 5 | # # MQ driver for Google Cloud Pubsub 6 | # mq: 7 | # driver: "pubsub" 8 | # project_id: "" 9 | # subscriber_id: "" 10 | # emulator_host: "" 11 | # topic_prefix: "" 12 | # subscription_prefix: "" 13 | 14 | # # Index driver for Google Bigtable 15 | # data: 16 | # index: 17 | # driver: "bigtable" 18 | # project_id: "" 19 | # instance_id: "" 20 | # emulator_host: "" 21 | 22 | # # Warehouse driver Google BigQuery 23 | # data: 24 | # warehouse: 25 | # driver: "bigquery" 26 | # project_id: "" 27 | # instances_dataset_id: "" 28 | -------------------------------------------------------------------------------- /migrations/015_stream_retention.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // Stream.RetentionSeconds 10 | _, err = db.Exec(` 11 | ALTER TABLE streams 12 | ADD retention_seconds integer NOT NULL DEFAULT 0; 13 | `) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | // Done 19 | return nil 20 | }, func(db migrations.DB) (err error) { 21 | // Stream.RetentionSeconds 22 | _, err = db.Exec(` 23 | ALTER TABLE streams DROP retention_seconds; 24 | `) 25 | if err != nil { 26 | return err 27 | } 28 | 29 | // Done 30 | return nil 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /web/pages/_error.tsx: -------------------------------------------------------------------------------- 1 | import { Container, Typography, withStyles } from "@material-ui/core"; 2 | import { NextPage } from "next"; 3 | import React from "react"; 4 | 5 | import { withApollo } from "../apollo/withApollo"; 6 | import ErrorPage from "../components/ErrorPage"; 7 | 8 | export interface ErrorPageProps { 9 | statusCode?: number; 10 | } 11 | 12 | const errorPage: NextPage = ({ statusCode }) => { 13 | return ; 14 | }; 15 | 16 | errorPage.getInitialProps = async ({ res, err }) => { 17 | const statusCode = res ? res.statusCode : err ? err.statusCode : undefined; 18 | return { statusCode }; 19 | }; 20 | 21 | export default withApollo(errorPage); 22 | -------------------------------------------------------------------------------- /migrations/044_private_project_default.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // Project.Public 10 | _, err = db.Exec(` 11 | ALTER TABLE projects 12 | ALTER COLUMN public SET DEFAULT FALSE; 13 | `) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | // Done 19 | return nil 20 | }, func(db migrations.DB) (err error) { 21 | // Project.Public 22 | _, err = db.Exec(` 23 | ALTER TABLE projects 24 | ALTER COLUMN public SET DEFAULT TRUE; 25 | `) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | // Done 31 | return nil 32 | }) 33 | } 34 | -------------------------------------------------------------------------------- /web/apollo/local-schemas/token.ts: -------------------------------------------------------------------------------- 1 | import gql from "graphql-tag"; 2 | 3 | import { GET_AID, GET_TOKEN } from "../queries/local/token"; 4 | 5 | export const typeDefs = gql` 6 | extend type Query { 7 | aid: String 8 | token: String 9 | } 10 | `; 11 | 12 | export const resolvers = { 13 | Query: { 14 | aid: async (_: any, args: any, { cache }: any) => { 15 | const { aid } = cache.readQuery({ query: GET_AID }); 16 | return aid; 17 | }, 18 | token: async (_: any, args: any, { cache }: any) => { 19 | const { token } = cache.readQuery({ query: GET_TOKEN }); 20 | return token; 21 | }, 22 | }, 23 | }; 24 | 25 | export default { 26 | typeDefs, 27 | resolvers, 28 | }; 29 | -------------------------------------------------------------------------------- /services/middleware/middleware.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "github.com/beneath-hq/beneath/services/secret" 5 | "github.com/go-redis/redis/v7" 6 | "github.com/go-redis/redis_rate/v8" 7 | ) 8 | 9 | // Service provides various useful HTTP and GRPC middlewares, eg. for authentication and rate limiting 10 | type Service struct { 11 | Redis *redis.Client 12 | SecretService *secret.Service 13 | 14 | limiter *redis_rate.Limiter 15 | } 16 | 17 | // New creates a new middleware service 18 | func New(redis *redis.Client, secretService *secret.Service) *Service { 19 | s := &Service{ 20 | Redis: redis, 21 | SecretService: secretService, 22 | } 23 | s.initRateLimiter() 24 | return s 25 | } 26 | -------------------------------------------------------------------------------- /web/next.config.js: -------------------------------------------------------------------------------- 1 | const { PHASE_DEVELOPMENT_SERVER } = require("next/constants"); 2 | 3 | module.exports = (phase) => { 4 | // setup monaco editor 5 | const config = {}; 6 | 7 | // add BENEATH_ENV variable 8 | if (phase === PHASE_DEVELOPMENT_SERVER) { 9 | config.env = Object.assign({}, config.env, { 10 | BENEATH_EE: process.env.BENEATH_EE, 11 | BENEATH_ENV: "development", 12 | }); 13 | } else { 14 | config.env = Object.assign({}, config.env, { 15 | BENEATH_EE: process.env.BENEATH_EE, 16 | BENEATH_ENV: "production", 17 | }); 18 | } 19 | 20 | // use webpack 5 21 | config.future = { 22 | webpack5: true, 23 | }; 24 | 25 | // return 26 | return config; 27 | }; 28 | -------------------------------------------------------------------------------- /web/pages/-/create/project.tsx: -------------------------------------------------------------------------------- 1 | import { NextPage } from "next"; 2 | 3 | import { withApollo } from "apollo/withApollo"; 4 | import AuthToContinue from "components/AuthToContinue"; 5 | import Page from "components/Page"; 6 | import CreateProject from "components/project/CreateProject"; 7 | import { useToken } from "hooks/useToken"; 8 | 9 | const CreatePage: NextPage = () => { 10 | const token = useToken(); 11 | return ( 12 | 13 | {!token && } 14 | {token && } 15 | 16 | ); 17 | }; 18 | 19 | export default withApollo(CreatePage); 20 | -------------------------------------------------------------------------------- /migrations/009_organization_unique.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // Organization.Name UNIQUE 10 | _, err = db.Exec(` 11 | CREATE UNIQUE INDEX organizations_name_key ON public.organizations USING btree (lower(name)); 12 | `) 13 | if err != nil { 14 | return err 15 | } 16 | 17 | // Done 18 | return nil 19 | }, func(db migrations.DB) (err error) { 20 | // Organization.Name UNIQUE 21 | _, err = db.Exec(` 22 | DROP INDEX organizations_name_key; 23 | `) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | // Done 29 | return nil 30 | }) 31 | } 32 | -------------------------------------------------------------------------------- /web/apollo/types/CompileSchema.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | import { CompileSchemaInput } from "./globalTypes"; 7 | 8 | // ==================================================== 9 | // GraphQL query operation: CompileSchema 10 | // ==================================================== 11 | 12 | export interface CompileSchema_compileSchema { 13 | __typename: "CompileSchemaOutput"; 14 | canonicalIndexes: string; 15 | } 16 | 17 | export interface CompileSchema { 18 | compileSchema: CompileSchema_compileSchema; 19 | } 20 | 21 | export interface CompileSchemaVariables { 22 | input: CompileSchemaInput; 23 | } 24 | -------------------------------------------------------------------------------- /clients/java/src/main/java/dev/beneath/client/CheckpointedState.java: -------------------------------------------------------------------------------- 1 | package dev.beneath.client; 2 | 3 | import com.google.protobuf.ByteString; 4 | 5 | public class CheckpointedState { 6 | private ByteString replayCursor; 7 | private ByteString changesCursor; 8 | 9 | public CheckpointedState() { 10 | } 11 | 12 | public ByteString getReplayCursor() { 13 | return replayCursor; 14 | } 15 | 16 | public void setReplayCursor(ByteString replayCursor) { 17 | this.replayCursor = replayCursor; 18 | } 19 | 20 | public ByteString getChangesCursor() { 21 | return changesCursor; 22 | } 23 | 24 | public void setChangesCursor(ByteString changesCursor) { 25 | this.changesCursor = changesCursor; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /docs/misc/reference.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: API Reference 3 | description: Links to client-specific API reference documentation 4 | menu: 5 | docs: 6 | parent: misc 7 | weight: 100 8 | weight: 100 9 | --- 10 | 11 | Every Beneath client library has a separate API reference documentation page. We think they're the best way to get detailed information about the available classes and functions. 12 | 13 | - [Python client reference](https://python.docs.beneath.dev/) 14 | - [JavaScript (TypeScript) client reference](https://js.docs.beneath.dev/) 15 | - [React client reference](https://react.docs.beneath.dev/) 16 | 17 | You can find the source code for the client libraries [here](https://github.com/beneath-hq/beneath/tree/master/clients). 18 | -------------------------------------------------------------------------------- /pkg/grpcutil/grpcutil.go: -------------------------------------------------------------------------------- 1 | package grpcutil 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net" 7 | 8 | "google.golang.org/grpc" 9 | ) 10 | 11 | // ListenAndServeContext serves a GRPC server and performs a graceful shutdown 12 | // if/when ctx is cancelled. 13 | func ListenAndServeContext(ctx context.Context, server *grpc.Server, port int) error { 14 | lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | cctx, cancel := context.WithCancel(ctx) 20 | var serveErr error 21 | go func() { 22 | serveErr = server.Serve(lis) 23 | cancel() 24 | }() 25 | 26 | <-cctx.Done() 27 | if serveErr == nil { 28 | server.GracefulStop() 29 | } 30 | 31 | return serveErr 32 | } 33 | -------------------------------------------------------------------------------- /docs/misc/contributing.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Contributing 3 | description: Learn more about contributing to the development of Beneath 4 | menu: 5 | docs: 6 | parent: misc 7 | weight: 300 8 | weight: 300 9 | --- 10 | 11 | The source code for Beneath is public and available on [Github](https://github.com/beneath-hq/beneath). 12 | 13 | To report a bug, request a feature, or ask a question, visit/search the [Issues page](https://github.com/beneath-hq/beneath/issues). We love getting your input, so don't hesitate to engage! 14 | 15 | If you want to contribute to Beneath -- or just want to learn more about how Beneath works -- check out the [`contributing` folder](https://github.com/beneath-hq/beneath/tree/master/contributing) for details. 16 | -------------------------------------------------------------------------------- /ee/migrations/008_billing_plan_base_price.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // BillingPlan.BasePriceCents 10 | _, err = db.Exec(` 11 | ALTER TABLE billing_plans 12 | ADD base_price_cents integer NOT NULL default 0; 13 | `) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | // Done 19 | return nil 20 | }, func(db migrations.DB) (err error) { 21 | // BillingPlan.BasePriceCents 22 | _, err = db.Exec(` 23 | ALTER TABLE billing_plans 24 | DROP base_price_cents; 25 | `) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | // Done 31 | return nil 32 | }) 33 | } 34 | -------------------------------------------------------------------------------- /migrations/038_instance_versions.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | _, err = db.Exec(` 10 | ALTER TABLE stream_instances 11 | ADD version bigint NOT NULL default 0; 12 | 13 | CREATE UNIQUE INDEX ON public.stream_instances USING btree (stream_id, version); 14 | `) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | // Done 20 | return nil 21 | }, func(db migrations.DB) (err error) { 22 | _, err = db.Exec(` 23 | ALTER TABLE stream_instances 24 | DROP version; 25 | `) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | // Done 31 | return nil 32 | }) 33 | } 34 | -------------------------------------------------------------------------------- /ee/migrations/007_billing_floats.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // BilledResource.Quantity 10 | _, err = db.Exec(` 11 | ALTER TABLE billed_resources 12 | ALTER COLUMN quantity SET DATA TYPE real; 13 | `) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | // Done 19 | return nil 20 | }, func(db migrations.DB) (err error) { 21 | // BilledResource.Quantity 22 | _, err = db.Exec(` 23 | ALTER TABLE billed_resources 24 | ALTER COLUMN quantity SET DATA TYPE bigint; 25 | `) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | // Done 31 | return nil 32 | }) 33 | } 34 | -------------------------------------------------------------------------------- /ee/server/control/gql/uuid.go: -------------------------------------------------------------------------------- 1 | package gql 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/99designs/gqlgen/graphql" 8 | uuid "github.com/satori/go.uuid" 9 | ) 10 | 11 | // MarshalUUID marshals the UUID custom scalar 12 | func MarshalUUID(id uuid.UUID) graphql.Marshaler { 13 | return graphql.WriterFunc(func(w io.Writer) { 14 | w.Write([]byte("\"")) 15 | w.Write([]byte(id.String())) 16 | w.Write([]byte("\"")) 17 | }) 18 | } 19 | 20 | // UnmarshalUUID unmarshals the UUID custom scalar 21 | func UnmarshalUUID(v interface{}) (uuid.UUID, error) { 22 | switch v := v.(type) { 23 | case string: 24 | return uuid.FromString(v) 25 | default: 26 | return uuid.Nil, fmt.Errorf("Cannot unmarshal %T to UUID (expected string)", v) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /server/control/gql/uuid.go: -------------------------------------------------------------------------------- 1 | package gql 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/99designs/gqlgen/graphql" 8 | uuid "github.com/satori/go.uuid" 9 | ) 10 | 11 | // MarshalUUID marshals the UUID custom scalar 12 | func MarshalUUID(id uuid.UUID) graphql.Marshaler { 13 | return graphql.WriterFunc(func(w io.Writer) { 14 | w.Write([]byte("\"")) 15 | w.Write([]byte(id.String())) 16 | w.Write([]byte("\"")) 17 | }) 18 | } 19 | 20 | // UnmarshalUUID unmarshals the UUID custom scalar 21 | func UnmarshalUUID(v interface{}) (uuid.UUID, error) { 22 | switch v := v.(type) { 23 | case string: 24 | return uuid.FromString(v) 25 | default: 26 | return uuid.Nil, fmt.Errorf("Cannot unmarshal %T to UUID (expected string)", v) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /web/apollo/types/AuthTicketByID.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: AuthTicketByID 8 | // ==================================================== 9 | 10 | export interface AuthTicketByID_authTicketByID { 11 | __typename: "AuthTicket"; 12 | authTicketID: ControlUUID; 13 | requesterName: string; 14 | createdOn: ControlTime; 15 | updatedOn: ControlTime; 16 | } 17 | 18 | export interface AuthTicketByID { 19 | authTicketByID: AuthTicketByID_authTicketByID; 20 | } 21 | 22 | export interface AuthTicketByIDVariables { 23 | authTicketID: ControlUUID; 24 | } 25 | -------------------------------------------------------------------------------- /ee/migrations/009_billing_plan_multiple_users.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // BillingPlan.MultipleUsers 10 | _, err = db.Exec(` 11 | ALTER TABLE billing_plans 12 | RENAME COLUMN personal TO multiple_users; 13 | `) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | // Done 19 | return nil 20 | }, func(db migrations.DB) (err error) { 21 | // BillingPlan.MultipleUsers 22 | _, err = db.Exec(` 23 | ALTER TABLE billing_plans 24 | RENAME COLUMN multiple_users TO personal; 25 | `) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | // Done 31 | return nil 32 | }) 33 | } 34 | -------------------------------------------------------------------------------- /cmd/beneath/cli/cli.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/spf13/cobra" 7 | "github.com/spf13/viper" 8 | ) 9 | 10 | // CLI builds and serves the CLI for *launching* the Beneath backend (*not* the Python-based CLI 11 | // for local authentication, creating tables, etc.) 12 | type CLI struct { 13 | Root *cobra.Command 14 | v *viper.Viper 15 | } 16 | 17 | // NewCLI creates a new CLI instance 18 | func NewCLI() *CLI { 19 | c := &CLI{} 20 | c.v = viper.New() 21 | c.Root = c.newRootCmd() 22 | c.Root.AddCommand(c.newStartCmd()) 23 | c.Root.AddCommand(c.newConfigCmd()) 24 | return c 25 | } 26 | 27 | // Run parses and runs commands 28 | func (c *CLI) Run() { 29 | if err := c.Root.Execute(); err != nil { 30 | os.Exit(1) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /ee/migrations/011_billing_plan_remove_private_projects.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // BillingPlan.PrivateProjects 10 | _, err = db.Exec(` 11 | ALTER TABLE billing_plans 12 | DROP private_projects; 13 | `) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | // Done 19 | return nil 20 | }, func(db migrations.DB) (err error) { 21 | // BillingPlan.PrivateProjects 22 | _, err = db.Exec(` 23 | ALTER TABLE billing_plans 24 | ADD private_projects boolean NOT NULL default true; 25 | `) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | // Done 31 | return nil 32 | }) 33 | } 34 | -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/aggregates/r-wallstreetbets-stock-metrics-24h.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1beta1 2 | kind: CronJob 3 | metadata: 4 | name: r-wallstreetbets-stock-metrics-24h 5 | spec: 6 | schedule: "5 0 * * *" # GCP cronjobs use UTC 7 | jobTemplate: 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | - name: r-wallstreetbets-stock-metrics-24h 13 | image: gcr.io/beneath/examples-wallstreetbets-stock-metrics-24h:latest 14 | env: 15 | - name: BENEATH_SECRET 16 | valueFrom: 17 | secretKeyRef: 18 | name: r-wallstreetbets-stock-metrics-24h 19 | key: beneath-secret 20 | restartPolicy: OnFailure 21 | -------------------------------------------------------------------------------- /migrations/010_project_display_name.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // Project.DisplayName NOT NULL 10 | _, err = db.Exec(` 11 | ALTER TABLE projects ALTER COLUMN display_name DROP NOT NULL; 12 | `) 13 | if err != nil { 14 | return err 15 | } 16 | 17 | // Done 18 | return nil 19 | }, func(db migrations.DB) (err error) { 20 | // Irreversible 21 | // Project.DisplayName NOT NULL 22 | // _, err = db.Exec(` 23 | // ALTER TABLE projects ALTER COLUMN display_name SET NOT NULL; 24 | // `) 25 | // if err != nil { 26 | // return err 27 | // } 28 | 29 | // Done 30 | return nil 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /web/apollo/types/SecretsForService.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: SecretsForService 8 | // ==================================================== 9 | 10 | export interface SecretsForService_secretsForService { 11 | __typename: "ServiceSecret"; 12 | serviceSecretID: string; 13 | description: string; 14 | prefix: string; 15 | createdOn: ControlTime; 16 | } 17 | 18 | export interface SecretsForService { 19 | secretsForService: SecretsForService_secretsForService[]; 20 | } 21 | 22 | export interface SecretsForServiceVariables { 23 | serviceID: ControlUUID; 24 | } 25 | -------------------------------------------------------------------------------- /web/ee/apollo/types/BillingMethods.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: BillingMethods 8 | // ==================================================== 9 | 10 | export interface BillingMethods_billingMethods { 11 | __typename: "BillingMethod"; 12 | billingMethodID: string; 13 | organizationID: ControlUUID; 14 | paymentsDriver: string; 15 | driverPayload: string; 16 | } 17 | 18 | export interface BillingMethods { 19 | billingMethods: BillingMethods_billingMethods[]; 20 | } 21 | 22 | export interface BillingMethodsVariables { 23 | organizationID: ControlUUID; 24 | } 25 | -------------------------------------------------------------------------------- /clients/java/README.md: -------------------------------------------------------------------------------- 1 | # Beneath Java Client Library 2 | 3 | [![License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://github.com/beneath-hq/beneath/blob/master/clients/LICENSE) 4 | 5 | This folder contains the source code for the [Beneath](https://beneath.dev) Java library. 6 | 7 | Thus far, the Beneath Java library has only been used to develop the [Beneath CDC service](https://github.com/beneath-hq/beneath/tree/master/examples/cdc). The primary Beneath SDK is the [Python client](https://github.com/beneath-hq/beneath/tree/master/clients/python). 8 | 9 | ### Installation 10 | 11 | With Gradle: 12 | 13 | ```groovy 14 | repositories { 15 | mavenCentral() 16 | } 17 | 18 | dependencies { 19 | implementation "dev.beneath:beneath:1.0.0" 20 | } 21 | ``` 22 | -------------------------------------------------------------------------------- /ee/pkg/paymentsutil/paymentsutil.go: -------------------------------------------------------------------------------- 1 | package paymentsutil 2 | 3 | import "time" 4 | 5 | // IsBlacklisted checks to see if a country is sanctioned by the EU 6 | func IsBlacklisted(country string) bool { 7 | blacklist := []string{"North Korea"} // +Iran, +Syria, +Russia, +Ukraine? see: https://sanctionsmap.eu/#/main, need to read details 8 | for _, sanctioned := range blacklist { 9 | if country == sanctioned { 10 | return true 11 | } 12 | } 13 | return false 14 | } 15 | 16 | // ComputeTaxPercentage computes the tax percent out of 100 to be added to each invoice item 17 | func ComputeTaxPercentage(country string, region string, isCompany bool, date time.Time) (int, string) { 18 | if country == "Denmark" { 19 | return 25, "VAT Denmark" 20 | } 21 | return 0, "N/A" 22 | } 23 | -------------------------------------------------------------------------------- /infra/engine/driver/mock/mock.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "github.com/beneath-hq/beneath/infra/engine/driver" 5 | ) 6 | 7 | // Mock implements WarehouseService 8 | type Mock struct { 9 | } 10 | 11 | func init() { 12 | driver.AddDriver("mock", newMock) 13 | } 14 | 15 | func newMock(optsMap map[string]interface{}) (driver.Service, error) { 16 | return &Mock{}, nil 17 | } 18 | 19 | // AsLookupService implements Service 20 | func (p *Mock) AsLookupService() driver.LookupService { 21 | return nil 22 | } 23 | 24 | // AsWarehouseService implements Service 25 | func (p *Mock) AsWarehouseService() driver.WarehouseService { 26 | return p 27 | } 28 | 29 | // AsUsageService implements Service 30 | func (p *Mock) AsUsageService() driver.UsageService { 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /clients/python/docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to beneath's documentation! 2 | =================================== 3 | 4 | This is the API documentation for the `Beneath `_ Python client. 5 | 6 | These pages describe the classes and functions of the ``beneath`` module. For concepts, quick starts, and more, read the `main Beneath documentation `_. 7 | 8 | You can find the source code for this library in the `Beneath monorepo on Github `_. 9 | 10 | Contents: 11 | --------- 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | 16 | client 17 | table 18 | instance 19 | job 20 | cursor 21 | errors 22 | pipeline 23 | checkpointer 24 | consumer 25 | easy 26 | misc/index 27 | -------------------------------------------------------------------------------- /migrations/024_user_org_perms_create.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // PermissionsUsersOrganizations.Create 10 | _, err = db.Exec(` 11 | ALTER TABLE permissions_users_organizations 12 | ADD "create" boolean NOT NULL DEFAULT false; 13 | `) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | // Done 19 | return nil 20 | }, func(db migrations.DB) (err error) { 21 | // PermissionsUsersOrganizations.Create 22 | _, err = db.Exec(` 23 | ALTER TABLE permissions_users_organizations 24 | DROP "create"; 25 | `) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | // Done 31 | return nil 32 | }) 33 | } 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019-present Beneath Systems ApS 2 | 3 | The contents of this repository are licensed as follows: 4 | 5 | - All content in the "clients/" directory (which contains libraries for interfacing with Beneath) and "examples/" directory (which contains examples for using Beneath) is licensed under the "MIT/Expat" license (see "licenses/MIT.txt") 6 | - All content in the "docs/" directory (which contains developer documentation) is licensed under the "Creative Commons Attribution-ShareAlike 4.0 International" license (see "licenses/CC-BY-SA-4.0.txt") 7 | - All third party components copied or linked into the repository are licensed under the original license provided by their owners 8 | - All other content is licensed under the Business Source License 1.1 (BSL) (see "licenses/BSL.txt") 9 | -------------------------------------------------------------------------------- /cmd/beneath/dependencies/data_server.go: -------------------------------------------------------------------------------- 1 | package dependencies 2 | 3 | import ( 4 | "github.com/beneath-hq/beneath/cmd/beneath/cli" 5 | "github.com/beneath-hq/beneath/server/data" 6 | "github.com/spf13/viper" 7 | ) 8 | 9 | func init() { 10 | cli.AddDependency(data.NewServer) 11 | 12 | cli.AddDependency(func(v *viper.Viper) (*data.ServerOptions, error) { 13 | var opts data.ServerOptions 14 | return &opts, v.UnmarshalKey("data", &opts) 15 | }) 16 | 17 | cli.AddConfigKey(&cli.ConfigKey{ 18 | Key: "data.http_port", 19 | Default: "5000", 20 | Description: "data server port for HTTP", 21 | }) 22 | 23 | cli.AddConfigKey(&cli.ConfigKey{ 24 | Key: "data.grpc_port", 25 | Default: "50051", 26 | Description: "data server port for GRPC", 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /ee/cloud/deployments/helm/backend/values.yaml: -------------------------------------------------------------------------------- 1 | nameOverride: "" 2 | fullnameOverride: "" 3 | 4 | image: 5 | pullPolicy: IfNotPresent 6 | repository: gcr.io/beneath/ee-backend 7 | tag: latest 8 | 9 | configSecretName: "beneath-config" 10 | configFileName: config.yaml 11 | 12 | ctrl: 13 | server: 14 | host: "" 15 | replicaCount: 1 16 | resources: {} 17 | worker: 18 | replicaCount: 1 19 | resources: {} 20 | 21 | data: 22 | server: 23 | httpHost: "" 24 | grpcHost: "" 25 | replicaCount: 1 26 | resources: {} 27 | worker: 28 | replicaCount: 1 29 | resources: {} 30 | 31 | extraEnv: [] 32 | # - name: ENV_VAR 33 | # value: env-var-value 34 | 35 | extraSecretMounts: [] 36 | # - secretName: secret-files 37 | # mountPath: /etc/secrets 38 | 39 | -------------------------------------------------------------------------------- /examples/clock-react/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "clock-react", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "beneath-react": "^1.2.0", 7 | "react": "^17.0.1", 8 | "react-dom": "^17.0.1", 9 | "react-scripts": "4.0.2" 10 | }, 11 | "scripts": { 12 | "start": "react-scripts start", 13 | "build": "react-scripts build" 14 | }, 15 | "eslintConfig": { 16 | "extends": [ 17 | "react-app", 18 | "react-app/jest" 19 | ] 20 | }, 21 | "browserslist": { 22 | "production": [ 23 | ">0.2%", 24 | "not dead", 25 | "not op_mini all" 26 | ], 27 | "development": [ 28 | "last 1 chrome version", 29 | "last 1 firefox version", 30 | "last 1 safari version" 31 | ] 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /pkg/schemalang/avro/avro.go: -------------------------------------------------------------------------------- 1 | package avro 2 | 3 | // Type is an Avro type 4 | type Type string 5 | 6 | // LogicalType is an Avro logical type 7 | type LogicalType string 8 | 9 | // Constants for Type 10 | const ( 11 | StringType Type = "string" 12 | BytesType Type = "bytes" 13 | IntType Type = "int" 14 | LongType Type = "long" 15 | FloatType Type = "float" 16 | DoubleType Type = "double" 17 | BooleanType Type = "boolean" 18 | NullType Type = "null" 19 | 20 | FixedType Type = "fixed" 21 | ArrayType Type = "array" 22 | RecordType Type = "record" 23 | EnumType Type = "enum" 24 | 25 | DecimalLogicalType LogicalType = "decimal" 26 | TimestampMillisLogicalType LogicalType = "timestamp-millis" 27 | UUIDLogicalType LogicalType = "uuid" 28 | ) 29 | -------------------------------------------------------------------------------- /web/apollo/types/SecretsForUser.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: SecretsForUser 8 | // ==================================================== 9 | 10 | export interface SecretsForUser_secretsForUser { 11 | __typename: "UserSecret"; 12 | userSecretID: string; 13 | description: string; 14 | prefix: string; 15 | createdOn: ControlTime; 16 | readOnly: boolean; 17 | publicOnly: boolean; 18 | } 19 | 20 | export interface SecretsForUser { 21 | secretsForUser: SecretsForUser_secretsForUser[]; 22 | } 23 | 24 | export interface SecretsForUserVariables { 25 | userID: ControlUUID; 26 | } 27 | -------------------------------------------------------------------------------- /web/apollo/types/UpdateAuthTicket.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | import { UpdateAuthTicketInput } from "./globalTypes"; 7 | 8 | // ==================================================== 9 | // GraphQL mutation operation: UpdateAuthTicket 10 | // ==================================================== 11 | 12 | export interface UpdateAuthTicket_updateAuthTicket { 13 | __typename: "AuthTicket"; 14 | authTicketID: ControlUUID; 15 | updatedOn: ControlTime; 16 | } 17 | 18 | export interface UpdateAuthTicket { 19 | updateAuthTicket: UpdateAuthTicket_updateAuthTicket | null; 20 | } 21 | 22 | export interface UpdateAuthTicketVariables { 23 | input: UpdateAuthTicketInput; 24 | } 25 | -------------------------------------------------------------------------------- /ee/server/control/schema/billing_info.graphql: -------------------------------------------------------------------------------- 1 | extend type Query { 2 | billingInfo(organizationID: UUID!): BillingInfo! 3 | } 4 | 5 | extend type Mutation { 6 | updateBillingDetails(organizationID: UUID!, country: String, region: String, companyName: String, taxNumber: String): BillingInfo! 7 | updateBillingMethod(organizationID: UUID!, billingMethodID: UUID): BillingInfo! 8 | updateBillingPlan(organizationID: UUID!, billingPlanID: UUID!): BillingInfo! 9 | } 10 | 11 | type BillingInfo { 12 | organizationID: ID! 13 | billingMethod: BillingMethod 14 | billingPlan: BillingPlan! 15 | country: String! 16 | region: String 17 | companyName: String 18 | taxNumber: String 19 | nextBillingTime: Time! 20 | lastInvoiceTime: Time! 21 | createdOn: Time! 22 | updatedOn: Time! 23 | } 24 | -------------------------------------------------------------------------------- /clients/README.md: -------------------------------------------------------------------------------- 1 | # `clients/` 2 | 3 | This directory contains all the language-specific client libraries for interfacing with Beneath from external applications. 4 | 5 | Rules for clients: 6 | 7 | - Each client should be named after the language and possibly framework, e.g., `js-react` 8 | - Each client should contain a `README.md` file for external users and a `CONTRIBUTING.md` file for contributors 9 | - For documentation, we're inspired by [how Google does it](https://cloud.google.com/pubsub/docs/reference/libraries), which means: 10 | - Each client should generate API reference docs at a standalone site 11 | - Tutorials and example projects should go in the root `examples/` folder 12 | - Overview and explainers should go in the main Beneath documentation (i.e., the root `docs/` folder) 13 | -------------------------------------------------------------------------------- /ee/migrations/012_billing_and_invoice_times.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | _, err = db.Exec(` 10 | ALTER TABLE billing_infos 11 | ADD next_billing_time timestamptz NOT NULL DEFAULT now(), 12 | ADD last_invoice_time timestamptz NOT NULL DEFAULT now() 13 | ; 14 | `) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | // Done 20 | return nil 21 | }, func(db migrations.DB) (err error) { 22 | _, err = db.Exec(` 23 | ALTER TABLE billing_infos 24 | DROP next_billing_time, 25 | DROP last_invoice_time 26 | ; 27 | `) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | // Done 33 | return nil 34 | }) 35 | } 36 | -------------------------------------------------------------------------------- /migrations/025_organization_quotas.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // Organizations.ReadQuota, Organizations.WriteQuota 10 | _, err = db.Exec(` 11 | ALTER TABLE organizations 12 | ADD read_quota bigint, 13 | ADD write_quota bigint; 14 | `) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | // Done 20 | return nil 21 | }, func(db migrations.DB) (err error) { 22 | // Organizations.ReadQuota, Organizations.WriteQuota 23 | _, err = db.Exec(` 24 | ALTER TABLE organizations 25 | DROP read_quota, 26 | DROP write_quota; 27 | `) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | // Done 33 | return nil 34 | }) 35 | } 36 | -------------------------------------------------------------------------------- /migrations/029_prepaid_quotas.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // BillingPlan.ReadQuotaCap, BillingPlan.WriteQuotaCap 10 | _, err = db.Exec(` 11 | ALTER TABLE organizations 12 | ADD prepaid_read_quota bigint, 13 | ADD prepaid_write_quota bigint; 14 | `) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | return nil 20 | }, func(db migrations.DB) (err error) { 21 | // BillingPlan.ReadQuotaCap, BillingPlan.WriteQuotaCap 22 | _, err = db.Exec(` 23 | ALTER TABLE organizations 24 | DROP prepaid_read_quota, 25 | DROP prepaid_write_quota; 26 | `) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | return nil 32 | }) 33 | } 34 | -------------------------------------------------------------------------------- /ee/cloud/deployments/kube/billing/billing-pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: backend-billing 5 | spec: 6 | restartPolicy: Never 7 | volumes: 8 | - name: backend-config 9 | secret: 10 | secretName: backend-config 11 | - name: beneath-sa-key 12 | secret: 13 | secretName: beneath-sa-key 14 | containers: 15 | - name: backend 16 | image: gcr.io/beneath/ee-backend:latest 17 | args: ["billing", "run", "--config", "/etc/config/production.yaml"] 18 | volumeMounts: 19 | - name: backend-config 20 | mountPath: /etc/config 21 | - name: beneath-sa-key 22 | mountPath: /var/secrets/google 23 | env: 24 | - name: GOOGLE_APPLICATION_CREDENTIALS 25 | value: /var/secrets/google/key.json 26 | -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/aggregates/config.py: -------------------------------------------------------------------------------- 1 | BLACKLIST = { 2 | "FOMO", 3 | "DD", 4 | "EOD", 5 | "TA", 6 | "PT", 7 | "RSI", 8 | "HUGE", 9 | "ATH", 10 | "USA", 11 | "AI", 12 | "IMO", 13 | "AM", 14 | "UK", 15 | "BIG", 16 | "SO", 17 | "OR", 18 | "FOR", 19 | "ALL", 20 | "IT", 21 | "BE", 22 | "ARE", 23 | "NOW", 24 | "ON", 25 | "ME", 26 | "CAN", 27 | "VERY", 28 | "SI", 29 | "TV", 30 | "BY", 31 | "NEW", 32 | "OUT", 33 | "LOVE", 34 | "GO", 35 | "PM", 36 | "NEXT", 37 | "ANY", 38 | "ET", 39 | "HAS", 40 | "ONE", 41 | "PLAY", 42 | "LOW", 43 | "III", 44 | "CASH", 45 | "RNG", 46 | "GOOD", 47 | "REAL", 48 | "SEE", 49 | "RE", 50 | } 51 | -------------------------------------------------------------------------------- /examples/wallstreetbets-analytics/stock-mentions/blacklist.py: -------------------------------------------------------------------------------- 1 | BLACKLIST = { 2 | "FOMO", 3 | "DD", 4 | "EOD", 5 | "TA", 6 | "PT", 7 | "RSI", 8 | "HUGE", 9 | "ATH", 10 | "USA", 11 | "AI", 12 | "IMO", 13 | "AM", 14 | "UK", 15 | "BIG", 16 | "SO", 17 | "OR", 18 | "FOR", 19 | "ALL", 20 | "IT", 21 | "BE", 22 | "ARE", 23 | "NOW", 24 | "ON", 25 | "ME", 26 | "CAN", 27 | "VERY", 28 | "SI", 29 | "TV", 30 | "BY", 31 | "NEW", 32 | "OUT", 33 | "LOVE", 34 | "GO", 35 | "PM", 36 | "NEXT", 37 | "ANY", 38 | "ET", 39 | "HAS", 40 | "ONE", 41 | "PLAY", 42 | "LOW", 43 | "III", 44 | "CASH", 45 | "RNG", 46 | "GOOD", 47 | "REAL", 48 | "SEE", 49 | "RE", 50 | } -------------------------------------------------------------------------------- /web/lib/connection.ts: -------------------------------------------------------------------------------- 1 | export const IS_EE = !!process.env.BENEATH_EE; 2 | export const IS_PRODUCTION = process.env.NODE_ENV === "production"; 3 | 4 | export const HTTP_PROTOCOL = IS_PRODUCTION ? "https" : "http"; 5 | export const WEBSOCKET_PROTOCOL = IS_PRODUCTION ? "wss" : "ws"; 6 | 7 | export const CLIENT_HOST = IS_PRODUCTION ? "beneath.dev" : "localhost:3000"; 8 | export const CLIENT_URL = `${HTTP_PROTOCOL}://${CLIENT_HOST}`; 9 | 10 | export const API_HOST = IS_PRODUCTION ? "control.beneath.dev" : "localhost:4000"; 11 | export const API_URL = `${HTTP_PROTOCOL}://${API_HOST}`; 12 | 13 | export const GATEWAY_HOST = IS_PRODUCTION ? "data.beneath.dev" : "localhost:5000"; 14 | export const GATEWAY_URL = `${HTTP_PROTOCOL}://${GATEWAY_HOST}`; 15 | export const GATEWAY_URL_WS = `${WEBSOCKET_PROTOCOL}://${GATEWAY_HOST}`; 16 | -------------------------------------------------------------------------------- /ee/cloud/deployments/helm/backend/templates/data-server-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "backend.dataserver.fullname" . }} 5 | labels: 6 | app.kubernetes.io/name: {{ include "backend.dataserver.fullname" . }} 7 | app.kubernetes.io/instance: {{ .Release.Name }} 8 | app.kubernetes.io/managed-by: {{ .Release.Service }} 9 | helm.sh/chart: {{ include "backend.chart" . }} 10 | spec: 11 | ports: 12 | - port: 8080 13 | name: data-http-port 14 | targetPort: data-http-port 15 | protocol: TCP 16 | - port: 9090 17 | name: data-grpc-port 18 | targetPort: data-grpc-port 19 | protocol: TCP 20 | selector: 21 | app.kubernetes.io/name: {{ include "backend.dataserver.fullname" . }} 22 | app.kubernetes.io/instance: {{ .Release.Name }} 23 | -------------------------------------------------------------------------------- /ee/cloud/deployments/kube/billing/billing-cronjob.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1beta1 2 | kind: CronJob 3 | metadata: 4 | name: backend-billing 5 | spec: 6 | schedule: "0 9 * * *" 7 | jobTemplate: 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | - name: backend 13 | image: gcr.io/beneath/ee-backend:latest 14 | args: ["billing", "run", "--config", "/etc/config/production.yaml"] 15 | volumeMounts: 16 | - name: backend-config 17 | mountPath: /etc/config 18 | - name: beneath-sa-key 19 | mountPath: /var/secrets/google 20 | env: 21 | - name: GOOGLE_APPLICATION_CREDENTIALS 22 | value: /var/secrets/google/key.json 23 | restartPolicy: Never 24 | -------------------------------------------------------------------------------- /ee/cloud/deployments/kube/migrations/migrate.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: backend-migrate 5 | spec: 6 | restartPolicy: Never 7 | volumes: 8 | - name: backend-config 9 | secret: 10 | secretName: backend-config 11 | - name: beneath-sa-key 12 | secret: 13 | secretName: beneath-sa-key 14 | containers: 15 | - name: backend 16 | image: gcr.io/beneath/ee-backend:latest 17 | imagePullPolicy: IfNotPresent 18 | args: ["migrate-ee", "set_version", "11", "--config", "/etc/config/production.yaml"] 19 | volumeMounts: 20 | - name: backend-config 21 | mountPath: /etc/config 22 | - name: beneath-sa-key 23 | mountPath: /var/secrets/google 24 | env: 25 | - name: GOOGLE_APPLICATION_CREDENTIALS 26 | value: /var/secrets/google/key.json 27 | -------------------------------------------------------------------------------- /migrations/034_user_consent.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // User.ConsentTerms, User.ConsentNewsletter 10 | _, err = db.Exec(` 11 | ALTER TABLE users 12 | ADD consent_terms boolean NOT NULL DEFAULT false, 13 | ADD consent_newsletter boolean NOT NULL DEFAULT false; 14 | `) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | // Done 20 | return nil 21 | }, func(db migrations.DB) (err error) { 22 | // BillingPlan.MultipleUsers 23 | _, err = db.Exec(` 24 | ALTER TABLE users 25 | DROP consent_terms, 26 | DROP consent_newsletter; 27 | `) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | // Done 33 | return nil 34 | }) 35 | } 36 | -------------------------------------------------------------------------------- /web/apollo/types/ProjectMembers.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: ProjectMembers 8 | // ==================================================== 9 | 10 | export interface ProjectMembers_projectMembers { 11 | __typename: "ProjectMember"; 12 | projectID: string; 13 | userID: string; 14 | name: string; 15 | displayName: string; 16 | photoURL: string | null; 17 | view: boolean; 18 | create: boolean; 19 | admin: boolean; 20 | } 21 | 22 | export interface ProjectMembers { 23 | projectMembers: ProjectMembers_projectMembers[]; 24 | } 25 | 26 | export interface ProjectMembersVariables { 27 | projectID: ControlUUID; 28 | } 29 | -------------------------------------------------------------------------------- /ee/migrations/003_tax_info.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // BillingInfo tax info 10 | _, err = db.Exec(` 11 | ALTER TABLE billing_infos 12 | ADD country text, 13 | ADD region text, 14 | ADD company_name text, 15 | ADD tax_number text; 16 | `) 17 | if err != nil { 18 | return err 19 | } 20 | 21 | // Done 22 | return nil 23 | }, func(db migrations.DB) (err error) { 24 | // BillingInfo tax info 25 | _, err = db.Exec(` 26 | ALTER TABLE billing_infos 27 | DROP country, 28 | DROP region, 29 | DROP company_name, 30 | DROP tax_number; 31 | `) 32 | if err != nil { 33 | return err 34 | } 35 | 36 | // Done 37 | return nil 38 | }) 39 | } 40 | -------------------------------------------------------------------------------- /web/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | // root: true, 3 | parserOptions: { 4 | ecmaVersion: 10, 5 | sourceType: "module", 6 | ecmaFeatures: { 7 | jsx: true, 8 | }, 9 | }, 10 | overrides: [ 11 | { 12 | files: ["*.ts", "*.tsx"], 13 | parser: "@typescript-eslint/parser", 14 | plugins: ["@typescript-eslint"], 15 | extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"], 16 | parserOptions: { 17 | ecmaFeatures: { 18 | jsx: true, 19 | }, 20 | }, 21 | rules: { 22 | "@typescript-eslint/no-explicit-any": "off", 23 | "no-unused-vars": "off", 24 | "@typescript-eslint/no-unused-vars": "off", 25 | "@typescript-eslint/explicit-module-boundary-types": "off", 26 | }, 27 | }, 28 | ], 29 | }; 30 | -------------------------------------------------------------------------------- /web/components/forms/FieldError.tsx: -------------------------------------------------------------------------------- 1 | import { FormHelperText, makeStyles, Theme } from "@material-ui/core"; 2 | import { FC } from "react"; 3 | 4 | const useStyles = makeStyles((theme: Theme) => ({ 5 | helper: { 6 | ...theme.typography.body2, 7 | }, 8 | })); 9 | 10 | export type FieldErrorProps = { 11 | id?: string; 12 | error?: boolean; 13 | errorText?: string; 14 | }; 15 | 16 | const FieldError: FC = ({ id, error, errorText }) => { 17 | const classes = useStyles(); 18 | const errorTextId = errorText && id ? `${id}-error-text` : undefined; 19 | if (error && errorText) { 20 | return ( 21 | 22 | {errorText} 23 | 24 | ); 25 | } 26 | return <>; 27 | }; 28 | 29 | export default FieldError; 30 | -------------------------------------------------------------------------------- /web/components/icons/Github.tsx: -------------------------------------------------------------------------------- 1 | export const Github = () => { 2 | return ( 3 | 4 | 8 | 9 | ); 10 | }; 11 | 12 | export default Github; 13 | -------------------------------------------------------------------------------- /clients/java/src/main/java/dev/beneath/client/admin/AdminClient.java: -------------------------------------------------------------------------------- 1 | package dev.beneath.client.admin; 2 | 3 | import dev.beneath.client.Connection; 4 | 5 | /** 6 | * AdminClient isolates control-plane features. 7 | * 8 | * Args: connection (Connection): An authenticated connection to Beneath. 9 | */ 10 | public class AdminClient { 11 | public Connection connection; 12 | public Organizations organizations; 13 | public Projects projects; 14 | public Tables tables; 15 | 16 | public AdminClient(Connection connection, Boolean dry) { 17 | this.connection = connection; 18 | this.organizations = new Organizations(this.connection, dry); 19 | this.projects = new Projects(this.connection, dry); 20 | this.tables = new Tables(this.connection, dry); 21 | 22 | this.connection.createGraphQlConnection(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /clients/java/src/main/java/dev/beneath/client/utils/JsonUtils.java: -------------------------------------------------------------------------------- 1 | package dev.beneath.client.utils; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | 6 | public class JsonUtils { 7 | private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); 8 | 9 | public static String serialize(Object object) { 10 | try { 11 | return OBJECT_MAPPER.writeValueAsString(object); 12 | } catch (JsonProcessingException e) { 13 | throw new RuntimeException(e); 14 | } 15 | } 16 | 17 | public static T deserialize(String jsonString, Class objectType) { 18 | try { 19 | return OBJECT_MAPPER.readValue(jsonString, objectType); 20 | } catch (JsonProcessingException e) { 21 | throw new RuntimeException(e); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /examples/beta/lending-club/loans/kube.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1beta1 2 | kind: CronJob 3 | metadata: 4 | name: lending-club-loans 5 | spec: 6 | schedule: "0 9,13,17,21 * * *" 7 | jobTemplate: 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | - name: lending-club-loans 13 | image: gcr.io/beneath/lending-club-loans:latest 14 | env: 15 | - name: BENEATH_SECRET 16 | valueFrom: 17 | secretKeyRef: 18 | name: lending-club-loans-service-secret 19 | key: secret 20 | - name: LENDING_CLUB_API_KEY 21 | valueFrom: 22 | secretKeyRef: 23 | name: lending-club-api-key 24 | key: secret 25 | restartPolicy: OnFailure 26 | -------------------------------------------------------------------------------- /web/pages/-/auth/ticket/success.tsx: -------------------------------------------------------------------------------- 1 | import { Typography } from "@material-ui/core"; 2 | import { NextPage } from "next"; 3 | import React from "react"; 4 | 5 | import { withApollo } from "apollo/withApollo"; 6 | import Page from "components/Page"; 7 | import { Paper } from "components/Paper"; 8 | 9 | const TicketSuccessPage: NextPage = () => { 10 | return ( 11 | 12 | 13 | 14 | Authorize client 15 | 16 | 17 | Success! You can now close this window and continue. 18 | 19 | 20 | 21 | ); 22 | }; 23 | 24 | export default withApollo(TicketSuccessPage); 25 | -------------------------------------------------------------------------------- /cmd/beneath/dependencies/mq.go: -------------------------------------------------------------------------------- 1 | package dependencies 2 | 3 | import ( 4 | "github.com/beneath-hq/beneath/cmd/beneath/cli" 5 | "github.com/beneath-hq/beneath/infra/mq" 6 | "github.com/spf13/viper" 7 | 8 | // registers all mq drivers 9 | _ "github.com/beneath-hq/beneath/infra/mq/driver/pubsub" 10 | ) 11 | 12 | func init() { 13 | cli.AddDependency(mq.NewMessageQueue) 14 | cli.AddDependency(func(v *viper.Viper) (*mq.Options, error) { 15 | var opts mq.Options 16 | return &opts, v.UnmarshalKey("mq", &opts) 17 | }) 18 | 19 | cli.AddConfigKey(&cli.ConfigKey{ 20 | Key: "mq.driver", 21 | Default: "", 22 | Description: "driver to use for message queue", 23 | }) 24 | cli.AddConfigKey(&cli.ConfigKey{ 25 | Key: "mq.subscriber_id", 26 | Default: "", 27 | Description: "unique identifier for the subscriber", 28 | }) 29 | } 30 | -------------------------------------------------------------------------------- /infra/engine/driver/bigtable/bigtable_test.go: -------------------------------------------------------------------------------- 1 | package bigtable 2 | 3 | import "testing" 4 | 5 | func TestBigtable(t *testing.T) { 6 | // TODO: Write tests 7 | // Compacted reads 8 | // Non-compacted reads 9 | // Filtered reads 10 | // Secondary index reads 11 | // Writing where secondary indexes update works (doesn't create garbage on sequential writes) 12 | // Cleans up garbage secondary index data on reads 13 | 14 | // read uncompacted scenarios: 15 | // - 1st cursor covers missing segment, 2nd cursor covers present segment 16 | // - 1st cursor covers partially missing segment, 2nd cursor covers a totally missing segment, no third cursor 17 | // - an open-ended cursor should stay open, both if it finds data and if it doesn't 18 | // - 1st and 2nd cursors cover missing segments, third cursor covers present segment (invariant violation?) 19 | } 20 | -------------------------------------------------------------------------------- /web/apollo/types/RegisterUserConsent.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: RegisterUserConsent 8 | // ==================================================== 9 | 10 | export interface RegisterUserConsent_registerUserConsent { 11 | __typename: "PrivateUser"; 12 | userID: string; 13 | updatedOn: ControlTime; 14 | consentTerms: boolean; 15 | consentNewsletter: boolean; 16 | } 17 | 18 | export interface RegisterUserConsent { 19 | registerUserConsent: RegisterUserConsent_registerUserConsent; 20 | } 21 | 22 | export interface RegisterUserConsentVariables { 23 | userID: ControlUUID; 24 | terms?: boolean | null; 25 | newsletter?: boolean | null; 26 | } 27 | -------------------------------------------------------------------------------- /web/components/console/tiles/LoadingTile.tsx: -------------------------------------------------------------------------------- 1 | import React, { FC } from "react"; 2 | 3 | import { CircularProgress, Grid, makeStyles, Theme, Typography } from "@material-ui/core"; 4 | 5 | import { Tile, TileProps } from "./Tile"; 6 | 7 | const useStyles = makeStyles((theme: Theme) => ({ 8 | container: { 9 | height: "inherit", 10 | minHeight: "inherit", 11 | padding: theme.spacing(1), 12 | }, 13 | })); 14 | 15 | export const LoadingTile: FC = ({ ...tileProps }) => { 16 | const classes = useStyles(); 17 | return ( 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | ); 26 | }; 27 | 28 | export default LoadingTile; 29 | -------------------------------------------------------------------------------- /web/hooks/useMe.tsx: -------------------------------------------------------------------------------- 1 | import { useQuery, getApolloContext } from "@apollo/client"; 2 | import { useContext } from "react"; 3 | 4 | import { QUERY_ME } from "../apollo/queries/organization"; 5 | import { Me } from "../apollo/types/Me"; 6 | import { useToken } from "./useToken"; 7 | 8 | export const useMe = () => { 9 | // if apollo isn't available, return null (e.g. on 404 page) 10 | const apolloContext = useContext(getApolloContext()); 11 | if (!apolloContext.client) { 12 | return null; 13 | } 14 | 15 | const token = useToken(); 16 | if (!token) { 17 | return null; 18 | } 19 | 20 | const { loading, error, data } = useQuery(QUERY_ME); 21 | if (data) { 22 | return data.me; 23 | } 24 | 25 | if (error) { 26 | console.error("useMe error: ", error); 27 | } 28 | 29 | return null; 30 | }; 31 | 32 | export default useMe; 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/refactor.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Refactor 3 | about: Propose a code restructuring 4 | title: '' 5 | labels: refactor 6 | assignees: '' 7 | 8 | --- 9 | 10 | ### Summary 11 | 12 | 16 | 17 | ### Risks and challenges 18 | 19 | 23 | 24 | ### Involved components 25 | 26 | 29 | 30 | ### Proposed changes 31 | 32 | 36 | -------------------------------------------------------------------------------- /examples/beta/reddit-sentiment/coronavirus-posts.graphql: -------------------------------------------------------------------------------- 1 | " Reddit posts to the r/Coronavirus subreddit. Using a pre-trained NLP model to compute each submission's polarity and subjectivity. " 2 | type Post @schema @key(fields: ["subreddit", "author", "timestamp"]) { 3 | " Name of the subreddit " 4 | subreddit: String! 5 | 6 | " Author of the post " 7 | author: String! 8 | 9 | " Post timestamp " 10 | timestamp: Timestamp! 11 | 12 | " Submission title " 13 | title: String! 14 | 15 | " Submission url " 16 | url: String 17 | 18 | " Submission polarity on a scale from -1 (negative) to 0 (neutral) to 1 (positive). Computed with the textblob python package. " 19 | polarity: Float! 20 | 21 | " Submission subjectivity on a scale from 0 (objective) to 1 (subjective). Computed with the textblob python package. " 22 | subjectivity: Float! 23 | } 24 | -------------------------------------------------------------------------------- /clients/java/src/main/java/dev/beneath/client/utils/TableIdentifier.java: -------------------------------------------------------------------------------- 1 | package dev.beneath.client.utils; 2 | 3 | public class TableIdentifier { 4 | public String organization; 5 | public String project; 6 | public String table; 7 | 8 | public TableIdentifier(String organization, String project, String table) { 9 | this.organization = Utils.prettyEntityName(organization); 10 | this.project = Utils.prettyEntityName(project); 11 | this.table = Utils.prettyEntityName(table); 12 | } 13 | 14 | public static TableIdentifier fromPath(String path) { 15 | String[] parts = Utils.splitResource("table", path); 16 | return new TableIdentifier(parts[0], parts[1], parts[2]); 17 | } 18 | 19 | @Override 20 | public String toString() { 21 | return String.format("%s/%s/table:%s", this.organization, this.project, this.table); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /clients/python/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnPaste": false, 3 | "editor.formatOnSave": true, 4 | "editor.tabSize": 4, 5 | "files.exclude": { 6 | "**/.venv": true, 7 | "**/__pycache__": true, 8 | "**/.pytest_cache": true, 9 | "**/.ipynb_checkpoints": true, 10 | "**/dist": true, 11 | "**/build": true, 12 | "**/*.egg-info": true, 13 | }, 14 | "python.formatting.provider": "black", 15 | "python.linting.enabled": true, 16 | "python.linting.pylintEnabled": false, 17 | "python.linting.flake8Enabled": true, 18 | "python.testing.unittestArgs": [ 19 | "-v", 20 | "-s", 21 | ".", 22 | "-p", 23 | "test*.py" 24 | ], 25 | "python.testing.nosetestsEnabled": false, 26 | "python.testing.pytestEnabled": true, 27 | "python.testing.unittestEnabled": false, 28 | "python.pythonPath": ".venv/bin/python" 29 | } -------------------------------------------------------------------------------- /migrations/008_service_dates.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // Service.CreatedOn and Service.UpdatedOn 10 | _, err = db.Exec(` 11 | ALTER TABLE services 12 | ADD created_on timestamp with time zone NOT NULL DEFAULT now(), 13 | ADD updated_on timestamp with time zone NOT NULL DEFAULT now(); 14 | `) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | // Done 20 | return nil 21 | }, func(db migrations.DB) (err error) { 22 | // Service.CreatedOn and Service.UpdatedOn 23 | _, err = db.Exec(` 24 | ALTER TABLE services DROP created_on; 25 | ALTER TABLE services DROP updated_on; 26 | `) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | // Done 32 | return nil 33 | }) 34 | } 35 | -------------------------------------------------------------------------------- /services/secret/events.go: -------------------------------------------------------------------------------- 1 | package secret 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/beneath-hq/beneath/models" 7 | ) 8 | 9 | func (s *Service) userUpdated(ctx context.Context, msg *models.UserUpdatedEvent) error { 10 | s.cache.ClearForUser(ctx, msg.User.UserID) 11 | return nil 12 | } 13 | 14 | func (s *Service) organizationUpdated(ctx context.Context, msg *models.OrganizationUpdatedEvent) error { 15 | s.cache.ClearForOrganization(ctx, msg.Organization.OrganizationID) 16 | return nil 17 | } 18 | 19 | func (s *Service) serviceUpdated(ctx context.Context, msg *models.ServiceUpdatedEvent) error { 20 | s.cache.ClearForService(ctx, msg.Service.ServiceID) 21 | return nil 22 | } 23 | 24 | func (s *Service) serviceDeleted(ctx context.Context, msg *models.ServiceDeletedEvent) error { 25 | s.cache.ClearForService(ctx, msg.ServiceID) 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /examples/ethereum-blocks/README.md: -------------------------------------------------------------------------------- 1 | To test locally: 2 | 3 | Set up poetry virtual environment 4 | 5 | ```bash 6 | poetry install 7 | poetry shell 8 | ``` 9 | 10 | Set a Web3 provider and run a simulation: 11 | 12 | ```bash 13 | WEB3_PROVIDER_URL=https://cloudflare-eth.com python main.py test 14 | ``` 15 | 16 | For reference, here's the complete list of build and deploy commands for the Kubernetes deployment: 17 | 18 | ```bash 19 | python main.py stage examples/ethereum/blocks-scraper --read-quota-mb 1000 --write-quota-mb 20000 20 | beneath service issue-secret examples/ethereum/blocks-scraper 21 | kubectl create secret generic ethereum-blocks -n models --from-literal=beneath-secret=XXX 22 | docker build -t gcr.io/beneath/examples-ethereum-blocks:latest . 23 | docker push gcr.io/beneath/examples-ethereum-blocks:latest 24 | kubectl apply -f kube.yaml -n models 25 | ``` 26 | -------------------------------------------------------------------------------- /web/apollo/queries/user.ts: -------------------------------------------------------------------------------- 1 | import gql from "graphql-tag"; 2 | 3 | export const REGISTER_USER_CONSENT = gql` 4 | mutation RegisterUserConsent($userID: UUID!, $terms: Boolean, $newsletter: Boolean) { 5 | registerUserConsent(userID: $userID, terms: $terms, newsletter: $newsletter) { 6 | userID 7 | updatedOn 8 | consentTerms 9 | consentNewsletter 10 | } 11 | } 12 | `; 13 | 14 | export const QUERY_AUTH_TICKET = gql` 15 | query AuthTicketByID($authTicketID: UUID!) { 16 | authTicketByID(authTicketID: $authTicketID) { 17 | authTicketID 18 | requesterName 19 | createdOn 20 | updatedOn 21 | } 22 | } 23 | `; 24 | 25 | export const UPDATE_AUTH_TICKET = gql` 26 | mutation UpdateAuthTicket($input: UpdateAuthTicketInput!) { 27 | updateAuthTicket(input: $input) { 28 | authTicketID 29 | updatedOn 30 | } 31 | } 32 | `; 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Question 3 | about: Ask a question 4 | title: '' 5 | labels: question 6 | assignees: '' 7 | 8 | --- 9 | 10 | 18 | 19 | ### Question 20 | 21 | 24 | 25 | ### Relevant code, logs, documentation and/or screenshots 26 | 27 | 30 | -------------------------------------------------------------------------------- /examples/earthquakes/kube.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: earthquakes 5 | labels: 6 | app.kubernetes.io/name: earthquakes 7 | app.kubernetes.io/instance: earthquakes 8 | app.kubernetes.io/managed-by: Manual 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | app.kubernetes.io/name: earthquakes 14 | app.kubernetes.io/instance: earthquakes 15 | template: 16 | metadata: 17 | labels: 18 | app.kubernetes.io/name: earthquakes 19 | app.kubernetes.io/instance: earthquakes 20 | spec: 21 | containers: 22 | - name: earthquakes 23 | image: gcr.io/beneath/examples-earthquakes:latest 24 | imagePullPolicy: Always 25 | env: 26 | - name: BENEATH_SECRET 27 | valueFrom: 28 | secretKeyRef: 29 | name: earthquakes 30 | key: secret 31 | -------------------------------------------------------------------------------- /migrations/012_stream_instances_counts.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | // Stream.InstancesCreatedCount, Stream.InstancesCommittedCount 10 | _, err = db.Exec(` 11 | ALTER TABLE streams 12 | ADD instances_created_count integer NOT NULL DEFAULT 0, 13 | ADD instances_committed_count integer NOT NULL DEFAULT 0; 14 | `) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | // Done 20 | return nil 21 | }, func(db migrations.DB) (err error) { 22 | // Stream.InstancesCreatedCount, Stream.InstancesCommittedCount 23 | _, err = db.Exec(` 24 | ALTER TABLE streams DROP instances_created_count; 25 | ALTER TABLE streams DROP instances_committed_count; 26 | `) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | // Done 32 | return nil 33 | }) 34 | } 35 | -------------------------------------------------------------------------------- /migrations/048_auth_ticket.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | _, err = db.Exec(` 10 | CREATE TABLE auth_tickets ( 11 | auth_ticket_id uuid DEFAULT uuid_generate_v4(), 12 | requester_name text NOT NULL, 13 | approver_user_id uuid, 14 | created_on timestamptz NOT NULL DEFAULT now(), 15 | updated_on timestamptz NOT NULL DEFAULT now(), 16 | PRIMARY KEY (auth_ticket_id), 17 | FOREIGN KEY (approver_user_id) REFERENCES users(user_id) ON DELETE CASCADE 18 | ); 19 | `) 20 | if err != nil { 21 | return err 22 | } 23 | return nil 24 | }, func(db migrations.DB) (err error) { 25 | _, err = db.Exec(` 26 | DROP TABLE IF EXISTS auth_tickets; 27 | `) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | // Done 33 | return nil 34 | }) 35 | } 36 | -------------------------------------------------------------------------------- /services/project/starter.go: -------------------------------------------------------------------------------- 1 | package project 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/beneath-hq/beneath/models" 7 | ) 8 | 9 | // CreateUserStarterProject creates a starter project when a user is created 10 | func (s *Service) CreateUserStarterProject(ctx context.Context, msg *models.UserCreatedEvent) error { 11 | // stage initial project 12 | starterProject := &models.Project{ 13 | Name: "starter_project", 14 | DisplayName: "Starter project", 15 | Description: "We automatically created this project for you to help you get started", 16 | Public: true, 17 | OrganizationID: msg.User.BillingOrganizationID, 18 | } 19 | 20 | err := s.CreateWithUser(ctx, starterProject, nil, nil, nil, nil, nil, msg.User.UserID, models.ProjectPermissions{ 21 | View: true, 22 | Create: true, 23 | Admin: true, 24 | }) 25 | if err != nil { 26 | return err 27 | } 28 | 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /services/user/user.go: -------------------------------------------------------------------------------- 1 | package user 2 | 3 | import ( 4 | "context" 5 | 6 | uuid "github.com/satori/go.uuid" 7 | 8 | "github.com/beneath-hq/beneath/bus" 9 | "github.com/beneath-hq/beneath/infra/db" 10 | "github.com/beneath-hq/beneath/models" 11 | ) 12 | 13 | // Service contains functionality for finding and creating users 14 | type Service struct { 15 | Bus *bus.Bus 16 | DB db.DB 17 | } 18 | 19 | // New creates a new user service 20 | func New(bus *bus.Bus, db db.DB) *Service { 21 | return &Service{ 22 | Bus: bus, 23 | DB: db, 24 | } 25 | } 26 | 27 | // FindUser returns the matching user or nil 28 | func (s *Service) FindUser(ctx context.Context, userID uuid.UUID) *models.User { 29 | user := &models.User{ 30 | UserID: userID, 31 | } 32 | err := s.DB.GetDB(ctx).ModelContext(ctx, user).WherePK().Column("user.*").Select() 33 | if !db.AssertFoundOne(err) { 34 | return nil 35 | } 36 | return user 37 | } 38 | -------------------------------------------------------------------------------- /web/apollo/types/GetUsage.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | import { GetUsageInput, UsageLabel } from "./globalTypes"; 7 | 8 | // ==================================================== 9 | // GraphQL query operation: GetUsage 10 | // ==================================================== 11 | 12 | export interface GetUsage_getUsage { 13 | __typename: "Usage"; 14 | entityID: ControlUUID; 15 | label: UsageLabel; 16 | time: ControlTime; 17 | readOps: number; 18 | readBytes: number; 19 | readRecords: number; 20 | writeOps: number; 21 | writeBytes: number; 22 | writeRecords: number; 23 | scanOps: number; 24 | scanBytes: number; 25 | } 26 | 27 | export interface GetUsage { 28 | getUsage: GetUsage_getUsage[]; 29 | } 30 | 31 | export interface GetUsageVariables { 32 | input: GetUsageInput; 33 | } 34 | -------------------------------------------------------------------------------- /examples/clock/kube.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: clock 5 | labels: 6 | app.kubernetes.io/name: clock 7 | app.kubernetes.io/instance: clock 8 | app.kubernetes.io/managed-by: Manual 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | app.kubernetes.io/name: clock 14 | app.kubernetes.io/instance: clock 15 | template: 16 | metadata: 17 | labels: 18 | app.kubernetes.io/name: clock 19 | app.kubernetes.io/instance: clock 20 | spec: 21 | containers: 22 | - name: clock 23 | image: gcr.io/beneath/examples-clock:latest 24 | imagePullPolicy: Always 25 | args: ["run", "examples/clock/clock"] 26 | env: 27 | - name: BENEATH_SECRET 28 | valueFrom: 29 | secretKeyRef: 30 | name: clock 31 | key: secret 32 | -------------------------------------------------------------------------------- /web/apollo/types/GetTableUsage.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | import { GetEntityUsageInput, UsageLabel } from "./globalTypes"; 7 | 8 | // ==================================================== 9 | // GraphQL query operation: GetTableUsage 10 | // ==================================================== 11 | 12 | export interface GetTableUsage_getTableUsage { 13 | __typename: "Usage"; 14 | entityID: ControlUUID; 15 | label: UsageLabel; 16 | time: ControlTime; 17 | readOps: number; 18 | readBytes: number; 19 | readRecords: number; 20 | writeOps: number; 21 | writeBytes: number; 22 | writeRecords: number; 23 | } 24 | 25 | export interface GetTableUsage { 26 | getTableUsage: GetTableUsage_getTableUsage[]; 27 | } 28 | 29 | export interface GetTableUsageVariables { 30 | input: GetEntityUsageInput; 31 | } 32 | -------------------------------------------------------------------------------- /web/public/assets/favicon/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "App", 3 | "icons": [ 4 | { 5 | "src": "\/android-icon-36x36.png", 6 | "sizes": "36x36", 7 | "type": "image\/png", 8 | "density": "0.75" 9 | }, 10 | { 11 | "src": "\/android-icon-48x48.png", 12 | "sizes": "48x48", 13 | "type": "image\/png", 14 | "density": "1.0" 15 | }, 16 | { 17 | "src": "\/android-icon-72x72.png", 18 | "sizes": "72x72", 19 | "type": "image\/png", 20 | "density": "1.5" 21 | }, 22 | { 23 | "src": "\/android-icon-96x96.png", 24 | "sizes": "96x96", 25 | "type": "image\/png", 26 | "density": "2.0" 27 | }, 28 | { 29 | "src": "\/android-icon-144x144.png", 30 | "sizes": "144x144", 31 | "type": "image\/png", 32 | "density": "3.0" 33 | }, 34 | { 35 | "src": "\/android-icon-192x192.png", 36 | "sizes": "192x192", 37 | "type": "image\/png", 38 | "density": "4.0" 39 | } 40 | ] 41 | } -------------------------------------------------------------------------------- /server/data/README.md: -------------------------------------------------------------------------------- 1 | # `server/data/` 2 | 3 | This package implements the data-plane server, which handles loading data to/from tables. 4 | 5 | The `http/` and `grpc/` contains the REST and gRPC server handlers respectively. For live subscriptions, they respectively offer websockets and gRPC unary streaming interfaces. They both largely delegate to `services/data.Service`, which contains the non-protocol specific handler implementations. 6 | 7 | ## Adding and modifying endpoints 8 | 9 | We strive to offer both gRPC and REST interfaces for all data-plane functionality. These are implemented separately to create the most intuitive experience for each protocol. When adding or modifying endpoints, make sure that your changes are reflected in both the `http/` and `grpc/` subpackages. 10 | 11 | ## Updating protocol buffer definitions 12 | 13 | - Run `scripts/proto-build.sh` to (re)generate all protocol buffers classes in the repository 14 | -------------------------------------------------------------------------------- /migrations/040_different_stream_schema_types.go: -------------------------------------------------------------------------------- 1 | package migrations 2 | 3 | import ( 4 | "github.com/go-pg/migrations/v7" 5 | ) 6 | 7 | func init() { 8 | Migrator.MustRegisterTx(func(db migrations.DB) (err error) { 9 | _, err = db.Exec(` 10 | ALTER TABLE streams 11 | ADD canonical_indexes JSON NOT NULL default '[]', 12 | DROP bigquery_schema; 13 | 14 | UPDATE streams s 15 | SET canonical_indexes = cast(replace(concat('[{"fields":', si.fields, ',"key":true}]'), ' ', '') as json) 16 | FROM stream_indexes si 17 | WHERE si.stream_id = s.stream_id; 18 | `) 19 | if err != nil { 20 | return err 21 | } 22 | 23 | // Done 24 | return nil 25 | }, func(db migrations.DB) (err error) { 26 | _, err = db.Exec(` 27 | ALTER TABLE streams 28 | DROP canonical_indexes, 29 | ADD bigquery_schema JSON; 30 | `) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | // Done 36 | return nil 37 | }) 38 | } 39 | -------------------------------------------------------------------------------- /pkg/envutil/config.go: -------------------------------------------------------------------------------- 1 | package envutil 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | // Env represents a runtime environment 9 | type Env string 10 | 11 | const ( 12 | // Development is used for running locally 13 | Development Env = "development" 14 | 15 | // Production is used in normal operation 16 | Production Env = "production" 17 | 18 | // Test is used for running automated tests 19 | Test Env = "test" 20 | ) 21 | 22 | // GetEnv reads the ENV environment variable. It panics if it's not set. 23 | func GetEnv() Env { 24 | env := os.Getenv("BENEATH_ENV") 25 | switch env { 26 | case "production": 27 | return Production 28 | case "prod": 29 | return Production 30 | case "development": 31 | return Development 32 | case "dev": 33 | return Development 34 | case "test": 35 | return Test 36 | case "": 37 | return Production 38 | default: 39 | panic(fmt.Errorf("ENV <%s> not recognized", env)) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /web/components/service/ViewUsage.tsx: -------------------------------------------------------------------------------- 1 | import { FC } from "react"; 2 | 3 | import { EntityKind } from "apollo/types/globalTypes"; 4 | import { ServiceByOrganizationProjectAndName_serviceByOrganizationProjectAndName } from "apollo/types/ServiceByOrganizationProjectAndName"; 5 | import OwnerUsageView from "components/usage/OwnerUsageView"; 6 | 7 | export interface ViewUsageProps { 8 | service: ServiceByOrganizationProjectAndName_serviceByOrganizationProjectAndName; 9 | } 10 | 11 | const ViewUsage: FC = ({ service }) => { 12 | return ( 13 | 22 | ); 23 | }; 24 | 25 | export default ViewUsage; 26 | -------------------------------------------------------------------------------- /clients/java/src/main/java/dev/beneath/client/utils/SubscriptionIdentifier.java: -------------------------------------------------------------------------------- 1 | package dev.beneath.client.utils; 2 | 3 | public class SubscriptionIdentifier { 4 | public String organization; 5 | public String project; 6 | public String subscription; 7 | 8 | public SubscriptionIdentifier(String organization, String project, String subscription) { 9 | this.organization = Utils.prettyEntityName(organization); 10 | this.project = Utils.prettyEntityName(project); 11 | this.subscription = Utils.prettyEntityName(subscription); 12 | } 13 | 14 | public static SubscriptionIdentifier fromPath(String path) { 15 | String[] parts = Utils.splitResource("subscription", path); 16 | return new SubscriptionIdentifier(parts[0], parts[1], parts[2]); 17 | } 18 | 19 | @Override 20 | public String toString() { 21 | return String.format("%s/%s/subscription:%s", this.organization, this.project, this.subscription); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /examples/ethereum-blocks/kube.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: ethereum-blocks 5 | labels: 6 | app.kubernetes.io/name: ethereum-blocks 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app.kubernetes.io/name: ethereum-blocks 12 | template: 13 | metadata: 14 | labels: 15 | app.kubernetes.io/name: ethereum-blocks 16 | spec: 17 | containers: 18 | - name: ethereum-blocks 19 | image: gcr.io/beneath/examples-ethereum-blocks:latest 20 | imagePullPolicy: Always 21 | args: ["run", "examples/ethereum/blocks-scraper"] 22 | env: 23 | - name: WEB3_PROVIDER_URL 24 | value: https://cloudflare-eth.com 25 | - name: BENEATH_SECRET 26 | valueFrom: 27 | secretKeyRef: 28 | name: ethereum-blocks 29 | key: beneath-secret 30 | -------------------------------------------------------------------------------- /pkg/mathutil/mathutil.go: -------------------------------------------------------------------------------- 1 | package mathutil 2 | 3 | import "fmt" 4 | 5 | // MinInt implements min for ints 6 | func MinInt(a, b int) int { 7 | if a > b { 8 | return b 9 | } 10 | return a 11 | } 12 | 13 | // MinInt64 implements min for int64s 14 | func MinInt64(a, b int64) int64 { 15 | if a > b { 16 | return b 17 | } 18 | return a 19 | } 20 | 21 | // MaxInt implements max for ints 22 | func MaxInt(a, b int) int { 23 | if a > b { 24 | return a 25 | } 26 | return b 27 | } 28 | 29 | // MaxInt64 implements max for int64s 30 | func MaxInt64(a, b int64) int64 { 31 | if a > b { 32 | return a 33 | } 34 | return b 35 | } 36 | 37 | // MinInts returns the lowest input value 38 | func MinInts(xs ...int) int { 39 | if len(xs) == 0 { 40 | panic(fmt.Errorf("cannot compute min on empty list")) 41 | } 42 | 43 | y := xs[0] 44 | for i := 1; i < len(xs); i++ { 45 | if xs[i] < y { 46 | y = xs[i] 47 | } 48 | } 49 | 50 | return y 51 | } 52 | -------------------------------------------------------------------------------- /pkg/schemalang/indexes_test.go: -------------------------------------------------------------------------------- 1 | package schemalang 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | // Note: there's a fair bunch of tests that touch Check in the transpiler test cases. 10 | 11 | func TestIndexesCanonicalJSON(t *testing.T) { 12 | first := Indexes{ 13 | Index{Fields: []string{"aaa", "bbb"}, Key: false}, 14 | Index{Fields: []string{"aaa", "ccc"}, Key: true}, 15 | Index{Fields: []string{"ddd"}, Key: false, Normalize: true}, 16 | } 17 | 18 | second := Indexes{ 19 | Index{Fields: []string{"aaa", "ccc"}, Key: true}, 20 | Index{Fields: []string{"ddd"}, Key: false, Normalize: true}, 21 | Index{Fields: []string{"aaa", "bbb"}, Key: false}, 22 | } 23 | 24 | expected := `[{"fields":["aaa","ccc"],"key":true},{"fields":["aaa","bbb"]},{"fields":["ddd"],"normalize":true}]` 25 | assert.Equal(t, expected, first.CanonicalJSON()) 26 | assert.Equal(t, expected, second.CanonicalJSON()) 27 | } 28 | --------------------------------------------------------------------------------