├── .gitattributes ├── .github └── workflows │ ├── conventional-commits.yml │ ├── docs-and-deploy.yml │ ├── publish.yml │ └── test.yml ├── .gitignore ├── .husky ├── .gitignore └── pre-commit ├── .prettierrc ├── .vscode └── settings.json ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── Tiltfile ├── admin ├── .eslintrc.json ├── .gitignore ├── Dockerfile.dev ├── README.md ├── app │ ├── cli-auth │ │ └── page.tsx │ ├── clusters │ │ ├── CreateClusterButton.tsx │ │ ├── [clusterId] │ │ │ ├── ClusterLiveTables.tsx │ │ │ ├── ClusterSettings.tsx │ │ │ ├── Navigation.tsx │ │ │ ├── SecretKeyReveal.tsx │ │ │ ├── activity │ │ │ │ ├── deployment │ │ │ │ │ └── page.tsx │ │ │ │ └── page.tsx │ │ │ ├── advanced │ │ │ │ └── page.tsx │ │ │ ├── client-libraries │ │ │ │ └── page.tsx │ │ │ ├── helpers.tsx │ │ │ ├── layout.tsx │ │ │ ├── monitoring │ │ │ │ └── page.tsx │ │ │ ├── overview │ │ │ │ ├── ActivityChart.tsx │ │ │ │ ├── CopyButton.tsx │ │ │ │ └── page.tsx │ │ │ ├── services │ │ │ │ ├── ServiceSummary.tsx │ │ │ │ ├── [serviceName] │ │ │ │ │ ├── ServiceDeployments.tsx │ │ │ │ │ ├── ServiceLiveTables.tsx │ │ │ │ │ ├── ServiceMetrics.tsx │ │ │ │ │ └── page.tsx │ │ │ │ └── page.tsx │ │ │ └── tasks │ │ │ │ └── page.tsx │ │ ├── fill-dates.ts │ │ └── page.tsx │ ├── favicon.ico │ ├── globals.css │ ├── layout.tsx │ ├── logo.png │ └── page.tsx ├── client │ ├── client.ts │ └── contract.ts ├── components.json ├── components │ ├── composite │ │ └── activity.tsx │ └── ui │ │ ├── DataTable.tsx │ │ ├── badge.tsx │ │ ├── button.tsx │ │ ├── card.tsx │ │ ├── clipboard-button.tsx │ │ ├── collapsible.tsx │ │ ├── drawer.tsx │ │ ├── label.tsx │ │ ├── navigation-menu.tsx │ │ ├── scroll-area.tsx │ │ ├── switch.tsx │ │ └── table.tsx ├── kubernetes.dev.yaml ├── lib │ └── utils.ts ├── middleware.ts ├── next.config.js ├── package-lock.json ├── package.json ├── postcss.config.js ├── public │ ├── next.svg │ └── vercel.svg ├── tailwind.config.js ├── tailwind.config.ts └── tsconfig.json ├── assets ├── differential-cloud-2.png ├── differential-cloud.gif └── image-3.png ├── cli ├── CHANGELOG.md ├── README.md ├── differential.local.json ├── package.json ├── src │ ├── client │ │ └── contract.ts │ ├── commands │ │ ├── auth.ts │ │ ├── client-lib-publish.ts │ │ ├── client-lib.ts │ │ ├── clusters-create.ts │ │ ├── clusters-info.ts │ │ ├── clusters-list.ts │ │ ├── clusters-open.ts │ │ ├── clusters.ts │ │ ├── context-info.ts │ │ ├── context-set.ts │ │ ├── context.ts │ │ ├── deploy-create.ts │ │ ├── deploy-info.ts │ │ ├── deploy-list.ts │ │ ├── deploy-logs.ts │ │ ├── deploy.ts │ │ ├── repl.ts │ │ ├── services-register.ts │ │ ├── services.ts │ │ └── task.ts │ ├── constants.ts │ ├── index.ts │ ├── lib │ │ ├── auth.ts │ │ ├── client.ts │ │ ├── context.ts │ │ ├── package.ts │ │ ├── release.ts │ │ ├── service-discovery.ts │ │ └── upload.ts │ └── utils.ts └── tsconfig.json ├── control-plane ├── .dockerignore ├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── Dockerfile.dev ├── LICENSE ├── README.md ├── babel.config.js ├── drizzle │ ├── 0000_odd_tattoo.sql │ ├── 0001_tense_dreaming_celestial.sql │ ├── 0002_steady_korath.sql │ ├── 0003_living_captain_america.sql │ ├── 0004_confused_goliath.sql │ ├── 0005_overconfident_hex.sql │ ├── 0006_ambiguous_wild_pack.sql │ ├── 0007_icy_the_hand.sql │ ├── 0008_omniscient_mongu.sql │ ├── 0009_cuddly_medusa.sql │ ├── 0010_curious_amphibian.sql │ ├── 0011_thin_preak.sql │ ├── 0012_manual_make.sql │ ├── 0013_bitter_lily_hollister.sql │ ├── 0014_kind_mercury.sql │ ├── 0015_flimsy_microbe.sql │ ├── 0016_unique_wallflower.sql │ ├── 0017_purple_captain_stacy.sql │ ├── 0018_spooky_sasquatch.sql │ ├── 0019_early_silver_sable.sql │ ├── 0020_third_lockjaw.sql │ ├── 0021_brave_eternals.sql │ ├── 0022_bizarre_princess_powerful.sql │ ├── 0023_lazy_dakota_north.sql │ ├── 0024_thin_black_bolt.sql │ ├── 0025_hot_shinobi_shaw.sql │ ├── 0026_freezing_clint_barton.sql │ ├── 0027_lyrical_ken_ellis.sql │ ├── 0028_parallel_karen_page.sql │ ├── 0029_sad_thunderbird.sql │ ├── 0030_omniscient_jamie_braddock.sql │ ├── 0031_misty_slipstream.sql │ ├── 0032_clumsy_gressill.sql │ ├── 0033_high_komodo.sql │ ├── 0034_real_big_bertha.sql │ ├── 0035_eminent_fantastic_four.sql │ ├── 0036_ambiguous_human_robot.sql │ ├── 0037_empty_molecule_man.sql │ ├── 0038_woozy_tiger_shark.sql │ ├── 0039_puzzling_vampiro.sql │ ├── 0040_mean_jazinda.sql │ ├── 0041_youthful_steve_rogers.sql │ ├── 0042_kind_lester.sql │ ├── 0043_closed_weapon_omega.sql │ ├── 0044_fancy_landau.sql │ ├── 0045_icy_kid_colt.sql │ ├── 0046_needy_brood.sql │ ├── 0047_certain_penance.sql │ ├── 0048_awesome_joystick.sql │ ├── 0049_gray_ben_grimm.sql │ ├── 0050_slimy_hulk.sql │ ├── 0051_loving_phil_sheldon.sql │ ├── 0052_swift_microbe.sql │ ├── 0053_broken_zaran.sql │ └── meta │ │ ├── 0000_snapshot.json │ │ ├── 0001_snapshot.json │ │ ├── 0002_snapshot.json │ │ ├── 0003_snapshot.json │ │ ├── 0004_snapshot.json │ │ ├── 0005_snapshot.json │ │ ├── 0006_snapshot.json │ │ ├── 0007_snapshot.json │ │ ├── 0008_snapshot.json │ │ ├── 0009_snapshot.json │ │ ├── 0010_snapshot.json │ │ ├── 0011_snapshot.json │ │ ├── 0012_snapshot.json │ │ ├── 0013_snapshot.json │ │ ├── 0014_snapshot.json │ │ ├── 0015_snapshot.json │ │ ├── 0016_snapshot.json │ │ ├── 0017_snapshot.json │ │ ├── 0018_snapshot.json │ │ ├── 0019_snapshot.json │ │ ├── 0020_snapshot.json │ │ ├── 0021_snapshot.json │ │ ├── 0022_snapshot.json │ │ ├── 0023_snapshot.json │ │ ├── 0024_snapshot.json │ │ ├── 0025_snapshot.json │ │ ├── 0026_snapshot.json │ │ ├── 0027_snapshot.json │ │ ├── 0028_snapshot.json │ │ ├── 0029_snapshot.json │ │ ├── 0030_snapshot.json │ │ ├── 0031_snapshot.json │ │ ├── 0032_snapshot.json │ │ ├── 0033_snapshot.json │ │ ├── 0034_snapshot.json │ │ ├── 0035_snapshot.json │ │ ├── 0036_snapshot.json │ │ ├── 0037_snapshot.json │ │ ├── 0038_snapshot.json │ │ ├── 0039_snapshot.json │ │ ├── 0040_snapshot.json │ │ ├── 0041_snapshot.json │ │ ├── 0042_snapshot.json │ │ ├── 0043_snapshot.json │ │ ├── 0044_snapshot.json │ │ ├── 0045_snapshot.json │ │ ├── 0046_snapshot.json │ │ ├── 0047_snapshot.json │ │ ├── 0048_snapshot.json │ │ ├── 0049_snapshot.json │ │ ├── 0050_snapshot.json │ │ ├── 0051_snapshot.json │ │ ├── 0052_snapshot.json │ │ ├── 0053_snapshot.json │ │ └── _journal.json ├── fly.toml ├── internal.md ├── kubernetes.dev.yaml ├── media │ └── logo.png ├── package-lock.json ├── package.json ├── postgres.dev.yaml ├── src │ ├── index.ts │ ├── modules │ │ ├── admin.ts │ │ ├── agents │ │ │ └── agent.ts │ │ ├── assets.test.ts │ │ ├── assets.ts │ │ ├── cluster-activity.ts │ │ ├── cluster.ts │ │ ├── contract.ts │ │ ├── cron.ts │ │ ├── data.ts │ │ ├── deployment │ │ │ ├── cfn-manager.test.ts │ │ │ ├── cfn-manager.ts │ │ │ ├── deployment-provider.ts │ │ │ ├── deployment.test.ts │ │ │ ├── deployment.ts │ │ │ ├── lambda-cfn-provider.ts │ │ │ ├── mock-deployment-provider.ts │ │ │ └── scheduler.ts │ │ ├── health.ts │ │ ├── jobs │ │ │ ├── create-job.ts │ │ │ ├── job-metrics.ts │ │ │ ├── jobs.test.ts │ │ │ ├── jobs.ts │ │ │ ├── persist-result.test.ts │ │ │ └── persist-result.ts │ │ ├── jwt.test.ts │ │ ├── jwt.ts │ │ ├── management.test.ts │ │ ├── management.ts │ │ ├── names.ts │ │ ├── observability │ │ │ ├── event-aggregation.test.ts │ │ │ ├── event-aggregation.ts │ │ │ └── events.ts │ │ ├── packages │ │ │ ├── client-lib.ts │ │ │ ├── versioning.test.ts │ │ │ └── versioning.ts │ │ ├── predictor │ │ │ ├── client.ts │ │ │ ├── contract.ts │ │ │ ├── predictor.test.ts │ │ │ ├── predictor.ts │ │ │ └── serialize-error.js │ │ ├── router.test.ts │ │ ├── router.ts │ │ ├── routing-helpers.test.ts │ │ ├── routing-helpers.ts │ │ ├── s3.ts │ │ ├── service-definitions.ts │ │ ├── sns.test.ts │ │ ├── sns.ts │ │ ├── test │ │ │ └── util.ts │ │ └── util.ts │ └── utilities │ │ ├── env.ts │ │ ├── errors.ts │ │ ├── invariant.ts │ │ ├── logger.ts │ │ ├── migrate.ts │ │ └── profiling.ts └── tsconfig.json ├── docs ├── .github │ └── workflows │ │ └── retype-action.yml ├── 256.png ├── CNAME ├── Makefile ├── _includes │ └── head.html ├── advanced │ ├── calling-functions │ │ ├── arguments-and-return-values.md │ │ └── customizing-function-calls.md │ ├── handling-failures │ │ ├── compute-recovery.md │ │ ├── failure-modes.md │ │ └── how-functions-fail.md │ ├── how-things-work │ │ ├── architecture.md │ │ ├── concepts.md │ │ ├── image.png │ │ ├── limitations.md │ │ └── service-discovery.md │ ├── index.yml │ └── self-hosting.md ├── api │ ├── README.md │ └── classes │ │ └── Differential.md ├── assets │ └── image-1.png ├── deployments │ ├── index.yml │ └── managed-compute.md ├── getting-started │ ├── index.yml │ ├── quick-start.md │ └── thinking.md ├── guides │ ├── caching.md │ ├── deduplicating.md │ ├── end-to-end-encryption.md │ ├── idempotency.md │ ├── index.yml │ ├── predictive-alerting.md │ ├── predictive-retries.md │ ├── recovering-from-infrastructure-stalls.md │ └── shared-client-libraries.md ├── image.png ├── index.md ├── package.json └── retype.yml ├── infrastructure ├── README.md ├── cfn.yaml └── deployment-templates │ └── lambda-cfn.yaml ├── lerna.json ├── load-tester ├── README.md ├── package-lock.json ├── package.json ├── run.sh ├── scripts │ └── setup.sh ├── src │ ├── commands │ │ ├── basic.ts │ │ ├── parallel-100.ts │ │ ├── parallel-1000.ts │ │ ├── parallel-10000.ts │ │ └── parallel.ts │ ├── d.ts │ ├── services │ │ └── executor.ts │ └── t.ts └── tsconfig.json ├── package-lock.json ├── package.json └── sdk ├── Makefile ├── README.md ├── assets └── logo.png ├── babel.config.js ├── docs ├── .nojekyll ├── README.md └── modules.md ├── package-lock.json ├── package.json ├── src ├── Differential.test.ts ├── Differential.ts ├── call-config.ts ├── contract.ts ├── create-client.ts ├── errors.ts ├── events.test.ts ├── events.ts ├── index.ts ├── results-poller.ts ├── serialize-error.js ├── serialize.test.ts ├── serialize.ts ├── task-queue.test.ts ├── task-queue.ts ├── tests │ ├── e2ee │ │ ├── d.ts │ │ ├── e2ee.test.ts │ │ └── hello.ts │ ├── errors │ │ ├── animals.ts │ │ ├── d.ts │ │ └── errors.test.ts │ ├── monolith │ │ ├── d.ts │ │ ├── db.ts │ │ ├── differential-398308-5f1deaf67146.json │ │ ├── expert.ts │ │ ├── facade.ts │ │ ├── monolith.test.ts │ │ └── run.ts │ └── utility │ │ ├── caching.test.ts │ │ ├── d.ts │ │ ├── product.ts │ │ └── retry.test.ts ├── types.ts └── util.ts ├── tsconfig.json └── typedoc.json /.gitattributes: -------------------------------------------------------------------------------- 1 | # git-crypt files containing secrets under R&D 2 | control-plane/src/modules/agents/** filter=git-crypt diff=git-crypt 3 | control-plane/kubernetes.dev.yaml filter=git-crypt diff=git-crypt 4 | sdk/src/tests/monolith/** filter=git-crypt diff=git-crypt -------------------------------------------------------------------------------- /.github/workflows/conventional-commits.yml: -------------------------------------------------------------------------------- 1 | name: Check conventional commits 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | types: [opened, edited, reopened, synchronize, ready_for_review] 8 | 9 | jobs: 10 | commit-message-validation: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - name: Commit message validation 15 | uses: amannn/action-semantic-pull-request@v5 16 | env: 17 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/docs-and-deploy.yml: -------------------------------------------------------------------------------- 1 | name: Generate Docs and deploy 2 | 3 | on: 4 | workflow_run: 5 | workflows: ["Run Tests"] 6 | branches: [main] 7 | types: 8 | - completed 9 | 10 | jobs: 11 | docs: 12 | runs-on: ubuntu-latest 13 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 14 | 15 | permissions: 16 | contents: write 17 | 18 | steps: 19 | - name: Checkout code 20 | uses: actions/checkout@v2 21 | with: 22 | fetch-depth: "0" # fetch all history for all tags and branches so that changeset-builder can generate changelogs 23 | 24 | - name: Set up Node.js 25 | uses: actions/setup-node@v2 26 | with: 27 | node-version: 20 28 | registry-url: "https://registry.npmjs.org" 29 | scope: "@differentialhq" 30 | 31 | - name: Install dependencies on root 32 | run: npm install 33 | 34 | - name: Configure Git User 35 | run: | 36 | git config --global user.name "Differential CI" 37 | git config --global user.email "ci@differential.dev" 38 | 39 | - name: Generate docs 40 | run: | 41 | make docs 42 | git add . 43 | git commit -m "chore: Update documentation [skip ci]" || true # ignore if there's nothing to commit 44 | git push origin main 45 | 46 | - uses: retypeapp/action-build@latest 47 | with: 48 | config: docs 49 | 50 | - uses: retypeapp/action-github-pages@latest 51 | with: 52 | update-branch: true 53 | 54 | deploy: 55 | name: deploy 56 | runs-on: ubuntu-latest 57 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 58 | steps: 59 | - uses: actions/checkout@v3 60 | - uses: sliteteam/github-action-git-crypt-unlock@1.2.0 61 | env: 62 | GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }} 63 | - uses: superfly/flyctl-actions/setup-flyctl@master 64 | - run: cd control-plane && flyctl deploy --remote-only 65 | env: 66 | FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} 67 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Create Release 2 | 3 | on: workflow_dispatch 4 | 5 | jobs: 6 | create-release: 7 | runs-on: ubuntu-latest 8 | 9 | permissions: 10 | contents: write 11 | 12 | steps: 13 | - name: Checkout code 14 | uses: actions/checkout@v2 15 | with: 16 | fetch-depth: "0" # fetch all history for all tags and branches so that changeset-builder can generate changelogs 17 | 18 | - name: Set up Node.js 19 | uses: actions/setup-node@v2 20 | with: 21 | node-version: 20 22 | registry-url: "https://registry.npmjs.org" 23 | scope: "@differentialhq" 24 | 25 | - name: Install dependencies on root 26 | run: npm install 27 | 28 | - name: Set DIFFERENTIAL_API_SECRET 29 | run: echo "DIFFERENTIAL_API_SECRET=$(curl https://api.differential.dev/demo/token)" >> $GITHUB_ENV 30 | 31 | - uses: sliteteam/github-action-git-crypt-unlock@1.2.0 32 | env: 33 | GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }} 34 | 35 | - name: Run tests 36 | run: npx lerna run test 37 | env: 38 | DATABASE_URL: ${{ secrets.DATABASE_URL }} 39 | JWKS_URL: ${{ secrets.JWKS_URL }} 40 | PREDICTOR_API_URL: ${{ secrets.PREDICTOR_API_URL }} 41 | 42 | - name: Build packages 43 | run: npx lerna run build 44 | 45 | - name: Configure Git User 46 | run: | 47 | git config --global user.name "Differential CI" 48 | git config --global user.email "ci@differential.dev" 49 | 50 | - name: Version packages 51 | run: npx lerna version --conventional-commits --yes --no-private --no-push 52 | env: 53 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 54 | 55 | # - name: Version packages 56 | # run: npx lerna version --conventional-commits --yes --no-private --create-release github 57 | # env: 58 | # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 59 | 60 | # - name: Publish packages 61 | # run: npx lerna publish from-package --yes --no-private --registry https://registry.npmjs.org 62 | # env: 63 | # NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 64 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Run Tests 2 | 3 | 4 | on: 5 | push: 6 | branches: 7 | - main 8 | pull_request: 9 | branches: 10 | - main 11 | 12 | jobs: 13 | default: 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - name: Checkout code 18 | uses: actions/checkout@v2 19 | with: 20 | fetch-depth: "0" # fetch all history for all tags and branches so that changeset-builder can generate changelogs 21 | 22 | - name: Set up Node.js 23 | uses: actions/setup-node@v2 24 | with: 25 | node-version: 20 26 | cache: 'npm' 27 | 28 | - name: Install dependencies 29 | run: npm install 30 | 31 | # - name: Set DIFFERENTIAL_API_SECRET 32 | # run: echo "DIFFERENTIAL_API_SECRET=$(curl https://api.differential.dev/demo/token)" >> $GITHUB_ENV 33 | 34 | # - name: Run tests 35 | # run: npx lerna run test 36 | # env: 37 | # DATABASE_URL: ${{ secrets.DATABASE_URL }} 38 | # JWKS_URL: ${{ secrets.JWKS_URL }} 39 | # PREDICTOR_API_URL: ${{ secrets.PREDICTOR_API_URL }} 40 | 41 | # - uses: sliteteam/github-action-git-crypt-unlock@1.2.0 42 | # env: 43 | # GIT_CRYPT_KEY: ${{ secrets.GIT_CRYPT_KEY }} 44 | 45 | # - name: Test builds 46 | # run: npx lerna run build 47 | -------------------------------------------------------------------------------- /.husky/.gitignore: -------------------------------------------------------------------------------- 1 | _ 2 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | . "$(dirname "$0")/_/husky.sh" 3 | 4 | npx lint-staged 5 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "all", 3 | "tabWidth": 2, 4 | "semi": true, 5 | "singleQuote": false 6 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnSave": true, 3 | } -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Differential 2 | 3 | We love your input! We want to make contributing to this project as easy and transparent as possible, whether it's: 4 | 5 | - Reporting a bug 6 | - Discussing the current state of the code 7 | - Submitting a fix 8 | - Proposing new features 9 | 10 | ## Setting up the development environment 11 | 12 | We use [tilt.dev](https://tilt.dev) to manage our development environment. 13 | 14 | 1. [Install](https://docs.tilt.dev/install.html) tilt on your machine. 15 | 2. Run `tilt up` in the root of the project. 16 | 17 | ## Running the admin console locally 18 | 19 | The admin console is a Next.js app using Clerk.dev for authentication. To run it locally, you'll need to add a `.env.local` file to the `admin` directory with some test credentials from Clerk.dev. You can sign up for a free account at [clerk.dev](https://clerk.dev). 20 | 21 | ``` 22 | NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=pk_test_REPLACE_ME 23 | CLERK_SECRET_KEY=sk_test_REPLACE_ME 24 | ``` 25 | 26 | ## We Develop with Github 27 | 28 | We use github to host code, to track issues and feature requests, as well as accept pull requests. 29 | 30 | ## All Code Changes Happen Through Pull Requests 31 | 32 | Pull requests are the best way to propose changes to the codebase. We actively welcome your pull requests: 33 | 34 | 1. Fork the repo and create your branch from `main`. 35 | 2. If you've added code that should be tested, add tests. 36 | 3. If you've changed APIs, update the documentation. 37 | 4. Ensure the test suite passes. 38 | 5. Make sure your code lints. 39 | 6. Issue that pull request! 40 | 41 | ## Any contributions you make will be under the Apache-2.0 Software License 42 | 43 | In short, when you submit code changes, your submissions are understood to be under the same license that covers the project. Feel free to contact the maintainers if that's a concern. 44 | 45 | ## Report bugs using Github's [issues](https://github.com/differentialhq/differential/issues) 46 | 47 | We use GitHub issues to track public bugs. Report a bug by [opening a new issue](https://github.com/differentialhq/differential/issues/new). 48 | 49 | ## Write bug reports with detail, background, and sample code 50 | 51 | A few things to consider when writing a bug report: 52 | 53 | - A quick summary and/or background 54 | - Steps to reproduce 55 | - Be specific! 56 | - What you expected would happen 57 | - What actually happens 58 | - Notes (possibly including why you think this might be happening, or stuff you tried that didn't work) 59 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: docs 2 | 3 | docs: 4 | @echo "Generating docs..." 5 | 6 | cd sdk && \ 7 | npm run docs 8 | 9 | cd docs && \ 10 | cp -r ../sdk/docs/* ./api && \ 11 | rm ./api/modules.md 12 | 13 | @echo "Done" -------------------------------------------------------------------------------- /Tiltfile: -------------------------------------------------------------------------------- 1 | k8s_yaml('control-plane/postgres.dev.yaml') 2 | k8s_resource('postgres', port_forwards=5432) 3 | 4 | k8s_yaml('control-plane/kubernetes.dev.yaml') 5 | k8s_resource('control-plane', port_forwards=[ 6 | port_forward(4000, 4000, "api"), 7 | port_forward(9091, 9091, 'metrics') 8 | ], resource_deps=['postgres']) 9 | 10 | docker_build('control-plane-image', 'control-plane', dockerfile='control-plane/Dockerfile.dev', live_update=[ 11 | sync('./control-plane/.env', '/app/.env'), 12 | sync('./control-plane/src', '/app/src'), 13 | sync('./control-plane/package.json', '/app/package.json'), 14 | sync('./control-plane/package-lock.json', '/app/package-lock.json'), 15 | run( 16 | 'npm install', 17 | trigger=['./control-plane/package.json', './control-plane/package-lock.json'], 18 | ) 19 | ], network='host') 20 | 21 | k8s_yaml('admin/kubernetes.dev.yaml') 22 | k8s_resource('admin', port_forwards=['3000'], resource_deps=['control-plane']) 23 | 24 | docker_build('admin-image', 'admin', dockerfile='admin/Dockerfile.dev', live_update=[ 25 | sync('./admin/client', '/app/client'), 26 | sync('./admin/app', '/app/app'), 27 | sync('./admin/components', '/app/components'), 28 | sync('./admin/lib', '/app/lib'), 29 | sync('./admin/package.json', '/app/package.json'), 30 | sync('./admin/package-lock.json', '/app/package-lock.json'), 31 | run( 32 | 'npm install', 33 | trigger=['./admin/package.json', './admin/package-lock.json'], 34 | ) 35 | ], ignore='.next', network='host') -------------------------------------------------------------------------------- /admin/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /admin/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | .yarn/install-state.gz 8 | 9 | # testing 10 | /coverage 11 | 12 | # next.js 13 | /.next/ 14 | /out/ 15 | 16 | # production 17 | /build 18 | 19 | # misc 20 | .DS_Store 21 | *.pem 22 | 23 | # debug 24 | npm-debug.log* 25 | yarn-debug.log* 26 | yarn-error.log* 27 | 28 | # local env files 29 | .env*.local 30 | 31 | # vercel 32 | .vercel 33 | 34 | # typescript 35 | *.tsbuildinfo 36 | next-env.d.ts 37 | -------------------------------------------------------------------------------- /admin/Dockerfile.dev: -------------------------------------------------------------------------------- 1 | FROM node:20 2 | 3 | WORKDIR /app 4 | 5 | # copy package.json and package-lock.json 6 | COPY --link package.json package-lock.json ./ 7 | 8 | # install dependencies 9 | RUN npm install 10 | 11 | # copy source code 12 | COPY --link . . 13 | 14 | RUN npm install 15 | 16 | ENTRYPOINT [ "npm", "run", "dev" ] -------------------------------------------------------------------------------- /admin/README.md: -------------------------------------------------------------------------------- 1 | This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app). 2 | 3 | ## Getting Started 4 | 5 | First, run the development server: 6 | 7 | ```bash 8 | npm run dev 9 | # or 10 | yarn dev 11 | # or 12 | pnpm dev 13 | # or 14 | bun dev 15 | ``` 16 | 17 | Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. 18 | 19 | You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file. 20 | 21 | This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font. 22 | 23 | ## Learn More 24 | 25 | To learn more about Next.js, take a look at the following resources: 26 | 27 | - [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. 28 | - [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. 29 | 30 | You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome! 31 | 32 | ## Deploy on Vercel 33 | 34 | The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. 35 | 36 | Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details. 37 | -------------------------------------------------------------------------------- /admin/app/cli-auth/page.tsx: -------------------------------------------------------------------------------- 1 | import { auth } from "@clerk/nextjs"; 2 | import { redirect } from "next/navigation"; 3 | 4 | export default async function CliAuth() { 5 | const { getToken } = await auth(); 6 | 7 | const token = await getToken({ 8 | template: "extended-cli-token", 9 | }); 10 | 11 | if (!token) { 12 | return null; 13 | } 14 | 15 | const url = new URL("http://localhost:9999"); 16 | url.searchParams.append("token", token); 17 | return redirect(url.toString()); 18 | } 19 | -------------------------------------------------------------------------------- /admin/app/clusters/CreateClusterButton.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { client } from "@/client/client"; 4 | import { Button } from "@/components/ui/button"; 5 | import { toast } from "react-hot-toast"; 6 | 7 | export const CreateClusterButton = ({ token }: { token: string }) => { 8 | return ( 9 | 42 | ); 43 | }; 44 | -------------------------------------------------------------------------------- /admin/app/clusters/[clusterId]/Navigation.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { Button } from "@/components/ui/button"; 4 | import Link from "next/link"; 5 | import { usePathname } from "next/navigation"; 6 | 7 | export function Navigation({ clusterId }: { clusterId: string }) { 8 | const currentPath = usePathname(); 9 | 10 | return ( 11 |
12 | 18 | 24 | 30 | {/* */} 36 | 46 |
47 | ); 48 | } 49 | -------------------------------------------------------------------------------- /admin/app/clusters/[clusterId]/SecretKeyReveal.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { Button } from "@/components/ui/button"; 4 | import { clipboardSvg } from "@/components/ui/clipboard-button"; 5 | import { useState } from "react"; 6 | import { toast } from "react-hot-toast"; 7 | 8 | const padRightWithAsterisks = (str: string) => { 9 | const asterisksForPadding = str.length - 8; 10 | 11 | return `${str.slice(0, 8)}${"*".repeat(asterisksForPadding)}`; 12 | }; 13 | 14 | export const SecretKeyReveal = ({ secretKey }: { secretKey: string }) => { 15 | const [revealed, setRevealed] = useState(false); 16 | 17 | return ( 18 |
19 |

20 | {revealed ? secretKey : padRightWithAsterisks(secretKey)} 21 |

22 |
23 | 33 | 36 | {/* Revoke */} 37 | 48 |
49 |
50 | ); 51 | }; 52 | -------------------------------------------------------------------------------- /admin/app/clusters/[clusterId]/helpers.tsx: -------------------------------------------------------------------------------- 1 | export function LiveGreenCircle() { 2 | return ( 3 |
4 | ); 5 | } 6 | 7 | export function SmallLiveGreenCircle() { 8 | return ( 9 |
10 | ); 11 | } 12 | 13 | export function LiveAmberCircle() { 14 | return ( 15 |
16 | ); 17 | } 18 | 19 | export function DeadRedCircle() { 20 | return
; 21 | } 22 | 23 | export function DeadGreenCircle() { 24 | return
; 25 | } 26 | 27 | export function DeadGrayCircle() { 28 | // a gray circle that is gray when the machine is dead 29 | return
; 30 | } 31 | 32 | export function functionStatusToCircle(status: string) { 33 | // "pending", "running", "success", "failure" 34 | switch (status) { 35 | case "pending": 36 | return ; 37 | case "running": 38 | return ; 39 | case "success": 40 | return ; 41 | case "failure": 42 | return ; 43 | case "stalled": 44 | return ; 45 | default: 46 | return ; 47 | } 48 | } 49 | 50 | export function deploymentStatusToCircle(status: string) { 51 | // "uploading", "ready", "active", "inactive" 52 | switch (status) { 53 | case "uploading": 54 | case "ready": 55 | return ; 56 | case "active": 57 | return ; 58 | default: 59 | return ; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /admin/app/clusters/[clusterId]/layout.tsx: -------------------------------------------------------------------------------- 1 | import { Navigation } from "./Navigation"; 2 | 3 | export default function Layout({ 4 | params, 5 | children, 6 | }: { 7 | params: { clusterId: string }; 8 | children: React.ReactNode; 9 | }) { 10 | return ( 11 |
12 |
13 |

Differential Cluster

14 |

{params.clusterId}

15 |
16 | 17 | {children} 18 |
19 | ); 20 | } 21 | -------------------------------------------------------------------------------- /admin/app/clusters/[clusterId]/overview/CopyButton.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { Button } from "@/components/ui/button"; 4 | import toast from "react-hot-toast"; 5 | 6 | export const CopyButton = ({ script }: { script: string }) => ( 7 | 17 | ); 18 | -------------------------------------------------------------------------------- /admin/app/clusters/[clusterId]/services/ServiceSummary.tsx: -------------------------------------------------------------------------------- 1 | import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; 2 | import Link from "next/link"; 3 | import { SmallLiveGreenCircle } from "../helpers"; 4 | 5 | const Service = (props: { 6 | clusterId: string; 7 | name: string; 8 | functions?: Array<{ name: string }>; 9 | activeFunctions?: Array<{ name: string }>; 10 | }) => { 11 | return ( 12 | 16 | 17 | 18 |

service

19 | {props.name} 20 | 21 | 22 |

functions

23 |
24 | {props.functions 25 | ?.sort((a, b) => (a.name < b.name ? -1 : 1)) 26 | .map((fn) => ( 27 |
31 | {fn.name}() 32 | {props.activeFunctions?.find((f) => f.name === fn.name) && ( 33 | 34 | 35 | 36 | )} 37 |
38 | ))} 39 |
40 | 41 | 42 | 43 | ); 44 | }; 45 | 46 | export const ServiceSummary = (props: { 47 | clusterId: string; 48 | services: Array<{ name: string; functions?: Array<{ name: string }> }>; 49 | activeFunctions?: Array<{ name: string; service: string }>; 50 | }) => { 51 | return ( 52 |
53 | {props.services 54 | .sort((a, b) => (a.name < b.name ? -1 : 1)) 55 | .map((s) => ( 56 | 62 | ))} 63 |
64 | ); 65 | }; 66 | -------------------------------------------------------------------------------- /admin/app/clusters/[clusterId]/services/[serviceName]/page.tsx: -------------------------------------------------------------------------------- 1 | import { client } from "@/client/client"; 2 | import { auth } from "@clerk/nextjs"; 3 | import { ServiceDeployments } from "./ServiceDeployments"; 4 | import { ServiceLiveTables } from "./ServiceLiveTables"; 5 | import { ServiceMetrics } from "./ServiceMetrics"; 6 | 7 | export default async function Page({ 8 | params, 9 | }: { 10 | params: { clusterId: string; serviceName: string }; 11 | }) { 12 | const { getToken } = await auth(); 13 | 14 | const token = await getToken(); 15 | 16 | if (!token) { 17 | return null; 18 | } 19 | 20 | const clusterResult = await client.getClusterDetailsForUser({ 21 | headers: { 22 | authorization: `Bearer ${token}`, 23 | }, 24 | params: { 25 | clusterId: params.clusterId, 26 | }, 27 | }); 28 | 29 | if (clusterResult.status !== 200) { 30 | return null; 31 | } 32 | 33 | return ( 34 |
35 |
36 |

Differential Service

37 |

{params.serviceName}

38 |
39 | 40 | 44 | 45 | 50 | 51 | 56 |
57 | ); 58 | } 59 | -------------------------------------------------------------------------------- /admin/app/clusters/fill-dates.ts: -------------------------------------------------------------------------------- 1 | function thirtyMinutesAgo() { 2 | const now = new Date(); 3 | now.setMinutes(now.getMinutes() - 30); 4 | return now; 5 | } 6 | 7 | export function fillDates(data: Array): T[] { 8 | const sorted = data.sort( 9 | (a, b) => a.timestamp.getTime() - b.timestamp.getTime(), 10 | ); 11 | 12 | const earliestTimestamp = Math.min( 13 | sorted[0].timestamp.getTime(), 14 | thirtyMinutesAgo().getTime(), 15 | ); 16 | const latestTimestamp = new Date().getTime(); 17 | 18 | const dataApplicable = data.filter( 19 | (d) => d.timestamp?.getTime() >= earliestTimestamp, 20 | ); 21 | 22 | const timestamps = dataApplicable 23 | .map((d) => d.timestamp.getTime()) 24 | .filter((t) => t >= earliestTimestamp); 25 | 26 | const missingTimestamps = Array.from( 27 | { length: (latestTimestamp - earliestTimestamp) / 60000 }, 28 | (_, i) => i * 60000 + earliestTimestamp, 29 | ).filter((timestamp) => !timestamps.includes(timestamp)); 30 | 31 | return dataApplicable 32 | .concat( 33 | missingTimestamps.map((timestamp) => ({ 34 | timestamp: new Date(timestamp), 35 | })) as T[], 36 | ) 37 | .sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); 38 | } 39 | -------------------------------------------------------------------------------- /admin/app/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/admin/app/favicon.ico -------------------------------------------------------------------------------- /admin/app/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | @layer base { 6 | :root { 7 | --background: 0 0% 100%; 8 | --foreground: 222.2 84% 4.9%; 9 | 10 | --card: 0 0% 100%; 11 | --card-foreground: 222.2 84% 4.9%; 12 | 13 | --popover: 0 0% 100%; 14 | --popover-foreground: 222.2 84% 4.9%; 15 | 16 | --primary: 222.2 47.4% 11.2%; 17 | --primary-foreground: 210 40% 98%; 18 | 19 | --secondary: 210 40% 96.1%; 20 | --secondary-foreground: 222.2 47.4% 11.2%; 21 | 22 | --muted: 210 40% 96.1%; 23 | --muted-foreground: 215.4 16.3% 46.9%; 24 | 25 | --accent: 210 40% 96.1%; 26 | --accent-foreground: 222.2 47.4% 11.2%; 27 | 28 | --destructive: 0 84.2% 60.2%; 29 | --destructive-foreground: 210 40% 98%; 30 | 31 | --border: 214.3 31.8% 91.4%; 32 | --input: 214.3 31.8% 91.4%; 33 | --ring: 222.2 84% 4.9%; 34 | 35 | --radius: 0.5rem; 36 | } 37 | 38 | .dark { 39 | --background: 222.2 84% 4.9%; 40 | --foreground: 210 40% 98%; 41 | 42 | --card: 222.2 84% 4.9%; 43 | --card-foreground: 210 40% 98%; 44 | 45 | --popover: 222.2 84% 4.9%; 46 | --popover-foreground: 210 40% 98%; 47 | 48 | --primary: 210 40% 98%; 49 | --primary-foreground: 222.2 47.4% 11.2%; 50 | 51 | --secondary: 217.2 32.6% 17.5%; 52 | --secondary-foreground: 210 40% 98%; 53 | 54 | --muted: 217.2 32.6% 17.5%; 55 | --muted-foreground: 215 20.2% 65.1%; 56 | 57 | --accent: 217.2 32.6% 17.5%; 58 | --accent-foreground: 210 40% 98%; 59 | 60 | --destructive: 0 62.8% 30.6%; 61 | --destructive-foreground: 210 40% 98%; 62 | 63 | --border: 217.2 32.6% 17.5%; 64 | --input: 217.2 32.6% 17.5%; 65 | --ring: 212.7 26.8% 83.9%; 66 | } 67 | } 68 | 69 | @layer base { 70 | * { 71 | @apply border-border; 72 | } 73 | body { 74 | @apply bg-background text-foreground; 75 | } 76 | } -------------------------------------------------------------------------------- /admin/app/layout.tsx: -------------------------------------------------------------------------------- 1 | import { ClerkProvider, UserButton } from "@clerk/nextjs"; 2 | import type { Metadata } from "next"; 3 | import { Inter } from "next/font/google"; 4 | import Image from "next/image"; 5 | import { Toaster } from "react-hot-toast"; 6 | import "./globals.css"; 7 | import logo from "./logo.png"; 8 | import { Inter as FontSans } from "next/font/google"; 9 | import { cn } from "@/lib/utils"; 10 | 11 | const fontSans = FontSans({ 12 | subsets: ["latin"], 13 | variable: "--font-sans", 14 | }); 15 | 16 | const json = require("../package.json"); 17 | 18 | export const metadata: Metadata = { 19 | title: "Admin Console", 20 | description: "Admin Console for Differential", 21 | }; 22 | 23 | export default function RootLayout({ 24 | children, 25 | }: { 26 | children: React.ReactNode; 27 | }) { 28 | return ( 29 | 30 | 31 | 37 | 38 |
39 |
40 | 55 | 56 |
57 |
{children}
58 |
59 | 60 | 61 | 62 | ); 63 | } 64 | -------------------------------------------------------------------------------- /admin/app/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/admin/app/logo.png -------------------------------------------------------------------------------- /admin/app/page.tsx: -------------------------------------------------------------------------------- 1 | import { redirect } from "next/navigation"; 2 | 3 | export default function Home() { 4 | // redirect to /clusters 5 | redirect("/clusters"); 6 | } 7 | -------------------------------------------------------------------------------- /admin/client/client.ts: -------------------------------------------------------------------------------- 1 | import { initClient } from "@ts-rest/core"; 2 | import { contract } from "./contract"; 3 | 4 | export const client = initClient(contract, { 5 | baseUrl: 6 | process.env.DIFFERENTIAL_API_URL || 7 | process.env.NEXT_PUBLIC_DIFFERENTIAL_API_URL || 8 | "https://api.differential.dev", 9 | baseHeaders: {}, 10 | }); 11 | -------------------------------------------------------------------------------- /admin/components.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://ui.shadcn.com/schema.json", 3 | "style": "default", 4 | "rsc": true, 5 | "tsx": true, 6 | "tailwind": { 7 | "config": "tailwind.config.js", 8 | "css": "app/globals.css", 9 | "baseColor": "slate", 10 | "cssVariables": true, 11 | "prefix": "" 12 | }, 13 | "aliases": { 14 | "components": "@/components", 15 | "utils": "@/lib/utils" 16 | } 17 | } -------------------------------------------------------------------------------- /admin/components/ui/badge.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import { cva, type VariantProps } from "class-variance-authority"; 3 | 4 | import { cn } from "@/lib/utils"; 5 | 6 | const badgeVariants = cva( 7 | "inline-flex items-center rounded-full border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2", 8 | { 9 | variants: { 10 | variant: { 11 | default: 12 | "border-transparent bg-primary text-primary-foreground hover:bg-primary/80", 13 | secondary: 14 | "border-transparent bg-secondary text-secondary-foreground hover:bg-secondary/80", 15 | destructive: 16 | "border-transparent bg-destructive text-destructive-foreground hover:bg-destructive/80", 17 | outline: "text-foreground", 18 | }, 19 | }, 20 | defaultVariants: { 21 | variant: "default", 22 | }, 23 | }, 24 | ); 25 | 26 | export interface BadgeProps 27 | extends React.HTMLAttributes, 28 | VariantProps {} 29 | 30 | function Badge({ className, variant, ...props }: BadgeProps) { 31 | return ( 32 |
33 | ); 34 | } 35 | 36 | export { Badge, badgeVariants }; 37 | -------------------------------------------------------------------------------- /admin/components/ui/button.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react" 2 | import { Slot } from "@radix-ui/react-slot" 3 | import { cva, type VariantProps } from "class-variance-authority" 4 | 5 | import { cn } from "@/lib/utils" 6 | 7 | const buttonVariants = cva( 8 | "inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50", 9 | { 10 | variants: { 11 | variant: { 12 | default: "bg-primary text-primary-foreground hover:bg-primary/90", 13 | destructive: 14 | "bg-destructive text-destructive-foreground hover:bg-destructive/90", 15 | outline: 16 | "border border-input bg-background hover:bg-accent hover:text-accent-foreground", 17 | secondary: 18 | "bg-secondary text-secondary-foreground hover:bg-secondary/80", 19 | ghost: "hover:bg-accent hover:text-accent-foreground", 20 | link: "text-primary underline-offset-4 hover:underline", 21 | }, 22 | size: { 23 | default: "h-10 px-4 py-2", 24 | sm: "h-9 rounded-md px-3", 25 | lg: "h-11 rounded-md px-8", 26 | icon: "h-10 w-10", 27 | }, 28 | }, 29 | defaultVariants: { 30 | variant: "default", 31 | size: "default", 32 | }, 33 | } 34 | ) 35 | 36 | export interface ButtonProps 37 | extends React.ButtonHTMLAttributes, 38 | VariantProps { 39 | asChild?: boolean 40 | } 41 | 42 | const Button = React.forwardRef( 43 | ({ className, variant, size, asChild = false, ...props }, ref) => { 44 | const Comp = asChild ? Slot : "button" 45 | return ( 46 | 51 | ) 52 | } 53 | ) 54 | Button.displayName = "Button" 55 | 56 | export { Button, buttonVariants } 57 | -------------------------------------------------------------------------------- /admin/components/ui/card.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react" 2 | 3 | import { cn } from "@/lib/utils" 4 | 5 | const Card = React.forwardRef< 6 | HTMLDivElement, 7 | React.HTMLAttributes 8 | >(({ className, ...props }, ref) => ( 9 |
17 | )) 18 | Card.displayName = "Card" 19 | 20 | const CardHeader = React.forwardRef< 21 | HTMLDivElement, 22 | React.HTMLAttributes 23 | >(({ className, ...props }, ref) => ( 24 |
29 | )) 30 | CardHeader.displayName = "CardHeader" 31 | 32 | const CardTitle = React.forwardRef< 33 | HTMLParagraphElement, 34 | React.HTMLAttributes 35 | >(({ className, ...props }, ref) => ( 36 |

44 | )) 45 | CardTitle.displayName = "CardTitle" 46 | 47 | const CardDescription = React.forwardRef< 48 | HTMLParagraphElement, 49 | React.HTMLAttributes 50 | >(({ className, ...props }, ref) => ( 51 |

56 | )) 57 | CardDescription.displayName = "CardDescription" 58 | 59 | const CardContent = React.forwardRef< 60 | HTMLDivElement, 61 | React.HTMLAttributes 62 | >(({ className, ...props }, ref) => ( 63 |

64 | )) 65 | CardContent.displayName = "CardContent" 66 | 67 | const CardFooter = React.forwardRef< 68 | HTMLDivElement, 69 | React.HTMLAttributes 70 | >(({ className, ...props }, ref) => ( 71 |
76 | )) 77 | CardFooter.displayName = "CardFooter" 78 | 79 | export { Card, CardHeader, CardFooter, CardTitle, CardDescription, CardContent } 80 | -------------------------------------------------------------------------------- /admin/components/ui/clipboard-button.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | export const clipboardSvg = () => ( 4 | 10 | 16 | 17 | ); 18 | -------------------------------------------------------------------------------- /admin/components/ui/collapsible.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import * as CollapsiblePrimitive from "@radix-ui/react-collapsible"; 4 | 5 | const Collapsible = CollapsiblePrimitive.Root; 6 | 7 | const CollapsibleTrigger = CollapsiblePrimitive.CollapsibleTrigger; 8 | 9 | const CollapsibleContent = CollapsiblePrimitive.CollapsibleContent; 10 | 11 | export { Collapsible, CollapsibleTrigger, CollapsibleContent }; 12 | -------------------------------------------------------------------------------- /admin/components/ui/label.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import * as React from "react"; 4 | import * as LabelPrimitive from "@radix-ui/react-label"; 5 | import { cva, type VariantProps } from "class-variance-authority"; 6 | 7 | import { cn } from "@/lib/utils"; 8 | 9 | const labelVariants = cva( 10 | "text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70", 11 | ); 12 | 13 | const Label = React.forwardRef< 14 | React.ElementRef, 15 | React.ComponentPropsWithoutRef & 16 | VariantProps 17 | >(({ className, ...props }, ref) => ( 18 | 23 | )); 24 | Label.displayName = LabelPrimitive.Root.displayName; 25 | 26 | export { Label }; 27 | -------------------------------------------------------------------------------- /admin/components/ui/scroll-area.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import * as React from "react"; 4 | import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area"; 5 | 6 | import { cn } from "@/lib/utils"; 7 | 8 | const ScrollArea = React.forwardRef< 9 | React.ElementRef, 10 | React.ComponentPropsWithoutRef 11 | >(({ className, children, ...props }, ref) => ( 12 | 17 | 18 | {children} 19 | 20 | 21 | 22 | 23 | )); 24 | ScrollArea.displayName = ScrollAreaPrimitive.Root.displayName; 25 | 26 | const ScrollBar = React.forwardRef< 27 | React.ElementRef, 28 | React.ComponentPropsWithoutRef 29 | >(({ className, orientation = "vertical", ...props }, ref) => ( 30 | 43 | 44 | 45 | )); 46 | ScrollBar.displayName = ScrollAreaPrimitive.ScrollAreaScrollbar.displayName; 47 | 48 | export { ScrollArea, ScrollBar }; 49 | -------------------------------------------------------------------------------- /admin/components/ui/switch.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import * as React from "react"; 4 | import * as SwitchPrimitives from "@radix-ui/react-switch"; 5 | 6 | import { cn } from "@/lib/utils"; 7 | 8 | const Switch = React.forwardRef< 9 | React.ElementRef, 10 | React.ComponentPropsWithoutRef 11 | >(({ className, ...props }, ref) => ( 12 | 20 | 25 | 26 | )); 27 | Switch.displayName = SwitchPrimitives.Root.displayName; 28 | 29 | export { Switch }; 30 | -------------------------------------------------------------------------------- /admin/kubernetes.dev.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: admin 5 | labels: 6 | app: admin 7 | spec: 8 | selector: 9 | matchLabels: 10 | app: admin 11 | template: 12 | metadata: 13 | labels: 14 | app: admin 15 | spec: 16 | containers: 17 | - name: admin 18 | image: admin-image 19 | ports: 20 | - containerPort: 3000 21 | env: 22 | - name: DIFFERENTIAL_API_URL 23 | value: http://host.docker.internal:4000 24 | - name: NEXT_PUBLIC_DIFFERENTIAL_API_URL 25 | value: http://localhost:4000 26 | -------------------------------------------------------------------------------- /admin/lib/utils.ts: -------------------------------------------------------------------------------- 1 | import { type ClassValue, clsx } from "clsx" 2 | import { twMerge } from "tailwind-merge" 3 | 4 | export function cn(...inputs: ClassValue[]) { 5 | return twMerge(clsx(inputs)) 6 | } 7 | -------------------------------------------------------------------------------- /admin/middleware.ts: -------------------------------------------------------------------------------- 1 | import { authMiddleware } from "@clerk/nextjs"; 2 | 3 | // This example protects all routes including api/trpc routes 4 | // Please edit this to allow other routes to be public as needed. 5 | // See https://clerk.com/docs/references/nextjs/auth-middleware for more information about configuring your Middleware 6 | export default authMiddleware({}); 7 | 8 | export const config = { 9 | matcher: ["/((?!.+\\.[\\w]+$|_next).*)", "/", "/(api|trpc)(.*)"], 10 | }; 11 | -------------------------------------------------------------------------------- /admin/next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = {}; 3 | 4 | module.exports = nextConfig; 5 | -------------------------------------------------------------------------------- /admin/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "admin", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint", 10 | "update-contract": "mkdir -p client && cp ../control-plane/src/modules/contract.ts client/" 11 | }, 12 | "dependencies": { 13 | "@clerk/clerk-sdk-node": "^4.13.4", 14 | "@clerk/nextjs": "^4.29.3", 15 | "@hookform/resolvers": "^3.3.4", 16 | "@radix-ui/react-collapsible": "^1.0.3", 17 | "@radix-ui/react-dialog": "^1.0.5", 18 | "@radix-ui/react-icons": "^1.3.0", 19 | "@radix-ui/react-label": "^2.0.2", 20 | "@radix-ui/react-navigation-menu": "^1.1.4", 21 | "@radix-ui/react-scroll-area": "^1.0.5", 22 | "@radix-ui/react-slot": "^1.0.2", 23 | "@radix-ui/react-switch": "^1.0.3", 24 | "@tanstack/react-table": "^8.11.2", 25 | "@ts-rest/core": "^3.30.5", 26 | "@ts-rest/react-query": "^3.30.5", 27 | "@types/react-syntax-highlighter": "^15.5.11", 28 | "class-variance-authority": "^0.7.0", 29 | "clsx": "^2.0.0", 30 | "date-fns": "^3.0.6", 31 | "jwks-rsa": "^3.1.0", 32 | "lucide-react": "^0.309.0", 33 | "msgpackr": "^1.10.1", 34 | "next": "14.2.5", 35 | "react": "^18", 36 | "react-dom": "^18", 37 | "react-hook-form": "^7.50.1", 38 | "react-hot-toast": "^2.4.1", 39 | "react-json-pretty": "^2.2.0", 40 | "react-loader-spinner": "^6.1.6", 41 | "react-syntax-highlighter": "^15.5.0", 42 | "recharts": "^2.10.4", 43 | "tailwind-merge": "^2.2.0", 44 | "tailwindcss-animate": "^1.0.7", 45 | "vaul": "^0.8.0", 46 | "zod": "^3.22.4" 47 | }, 48 | "devDependencies": { 49 | "@types/node": "^20", 50 | "@types/react": "^18", 51 | "@types/react-dom": "^18", 52 | "autoprefixer": "^10.0.1", 53 | "eslint": "^8", 54 | "eslint-config-next": "14.0.4", 55 | "postcss": "^8", 56 | "tailwindcss": "^3.3.0", 57 | "typescript": "^5" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /admin/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /admin/public/next.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /admin/public/vercel.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /admin/tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | darkMode: ["class"], 4 | content: [ 5 | './pages/**/*.{ts,tsx}', 6 | './components/**/*.{ts,tsx}', 7 | './app/**/*.{ts,tsx}', 8 | './src/**/*.{ts,tsx}', 9 | ], 10 | prefix: "", 11 | theme: { 12 | container: { 13 | center: true, 14 | padding: "2rem", 15 | screens: { 16 | "2xl": "1400px", 17 | }, 18 | }, 19 | extend: { 20 | colors: { 21 | border: "hsl(var(--border))", 22 | input: "hsl(var(--input))", 23 | ring: "hsl(var(--ring))", 24 | background: "hsl(var(--background))", 25 | foreground: "hsl(var(--foreground))", 26 | primary: { 27 | DEFAULT: "hsl(var(--primary))", 28 | foreground: "hsl(var(--primary-foreground))", 29 | }, 30 | secondary: { 31 | DEFAULT: "hsl(var(--secondary))", 32 | foreground: "hsl(var(--secondary-foreground))", 33 | }, 34 | destructive: { 35 | DEFAULT: "hsl(var(--destructive))", 36 | foreground: "hsl(var(--destructive-foreground))", 37 | }, 38 | muted: { 39 | DEFAULT: "hsl(var(--muted))", 40 | foreground: "hsl(var(--muted-foreground))", 41 | }, 42 | accent: { 43 | DEFAULT: "hsl(var(--accent))", 44 | foreground: "hsl(var(--accent-foreground))", 45 | }, 46 | popover: { 47 | DEFAULT: "hsl(var(--popover))", 48 | foreground: "hsl(var(--popover-foreground))", 49 | }, 50 | card: { 51 | DEFAULT: "hsl(var(--card))", 52 | foreground: "hsl(var(--card-foreground))", 53 | }, 54 | }, 55 | borderRadius: { 56 | lg: "var(--radius)", 57 | md: "calc(var(--radius) - 2px)", 58 | sm: "calc(var(--radius) - 4px)", 59 | }, 60 | keyframes: { 61 | "accordion-down": { 62 | from: { height: "0" }, 63 | to: { height: "var(--radix-accordion-content-height)" }, 64 | }, 65 | "accordion-up": { 66 | from: { height: "var(--radix-accordion-content-height)" }, 67 | to: { height: "0" }, 68 | }, 69 | }, 70 | animation: { 71 | "accordion-down": "accordion-down 0.2s ease-out", 72 | "accordion-up": "accordion-up 0.2s ease-out", 73 | }, 74 | }, 75 | }, 76 | plugins: [require("tailwindcss-animate")], 77 | } -------------------------------------------------------------------------------- /admin/tailwind.config.ts: -------------------------------------------------------------------------------- 1 | import type { Config } from "tailwindcss"; 2 | const { fontFamily } = require("tailwindcss/defaultTheme"); 3 | 4 | const config: Config = { 5 | content: [ 6 | "./pages/**/*.{js,ts,jsx,tsx,mdx}", 7 | "./components/**/*.{js,ts,jsx,tsx,mdx}", 8 | "./app/**/*.{js,ts,jsx,tsx,mdx}", 9 | ], 10 | theme: { 11 | extend: { 12 | fontFamily: { 13 | sans: ["var(--font-sans)", ...fontFamily.sans], 14 | }, 15 | }, 16 | }, 17 | }; 18 | export default config; 19 | -------------------------------------------------------------------------------- /admin/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "strict": true, 12 | "noEmit": true, 13 | "esModuleInterop": true, 14 | "module": "esnext", 15 | "moduleResolution": "bundler", 16 | "resolveJsonModule": true, 17 | "isolatedModules": true, 18 | "jsx": "preserve", 19 | "incremental": true, 20 | "plugins": [ 21 | { 22 | "name": "next" 23 | } 24 | ], 25 | "paths": { 26 | "@/*": [ 27 | "./*" 28 | ] 29 | } 30 | }, 31 | "include": [ 32 | "next-env.d.ts", 33 | "**/*.ts", 34 | "**/*.tsx", 35 | ".next/types/**/*.ts", 36 | "app/clusters/[clusterId]/page.tsx_", 37 | ], 38 | "exclude": [ 39 | "node_modules" 40 | ] 41 | } -------------------------------------------------------------------------------- /assets/differential-cloud-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/assets/differential-cloud-2.png -------------------------------------------------------------------------------- /assets/differential-cloud.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/assets/differential-cloud.gif -------------------------------------------------------------------------------- /assets/image-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/assets/image-3.png -------------------------------------------------------------------------------- /cli/differential.local.json: -------------------------------------------------------------------------------- 1 | { 2 | "apiUrl": "http://localhost:4000", 3 | "consoleUrl": "http://localhost:3000" 4 | } -------------------------------------------------------------------------------- /cli/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@differentialhq/cli", 3 | "version": "3.19.0", 4 | "description": "CLI for differential.dev", 5 | "bin": { 6 | "differential": "bin/index.js" 7 | }, 8 | "scripts": { 9 | "test": "exit 0", 10 | "start": "node ./bin/index.js", 11 | "build": "tsc", 12 | "clean": "rm -rf ./bin", 13 | "dev": "npm run build && npm run start" 14 | }, 15 | "author": "", 16 | "license": "ISC", 17 | "dependencies": { 18 | "@differentialhq/core": "^3.18.0", 19 | "@inquirer/prompts": "^4.0.0", 20 | "@ts-rest/core": "^3.28.0", 21 | "acorn": "^8.11.3", 22 | "acorn-walk": "^8.3.2", 23 | "debug": "^4.3.4", 24 | "detective": "^5.2.1", 25 | "tslib": "^2.6.2", 26 | "typescript": "^5.2.2", 27 | "ulid": "^2.3.0", 28 | "yargs": "^17.7.2", 29 | "zip-a-folder": "^3.1.6", 30 | "zod": "^3.22.2" 31 | }, 32 | "devDependencies": { 33 | "@types/debug": "^4.1.8", 34 | "@types/detective": "^5.1.5", 35 | "@types/jest": "^29.5.4", 36 | "@types/yargs": "^17.0.32", 37 | "jest": "^29.6.4", 38 | "ts-morph": "^22.0.0" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /cli/src/commands/auth.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule, showHelp } from "yargs"; 2 | import { startTokenFlow } from "../lib/auth"; 3 | 4 | export const Auth: CommandModule = { 5 | command: "auth", 6 | describe: "Authenticate with the Differential API.", 7 | builder: (yargs) => 8 | yargs.command({ 9 | command: "login", 10 | describe: "Authenticate the Differential CLI.", 11 | handler: () => { 12 | startTokenFlow(); 13 | }, 14 | }), 15 | handler: async () => { 16 | showHelp(); 17 | }, 18 | }; 19 | -------------------------------------------------------------------------------- /cli/src/commands/client-lib.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule, showHelp } from "yargs"; 2 | import { ClientLibraryPublish } from "./client-lib-publish"; 3 | 4 | export const ClientLibrary: CommandModule = { 5 | command: "client", 6 | describe: "Manage Differential client libraries", 7 | builder: (yargs) => yargs.command(ClientLibraryPublish), 8 | handler: async () => { 9 | showHelp(); 10 | }, 11 | }; 12 | -------------------------------------------------------------------------------- /cli/src/commands/clusters-create.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from "yargs"; 2 | import { client } from "../lib/client"; 3 | 4 | interface ClusterCreateArgs { 5 | description?: string; 6 | } 7 | export const ClusterCreate: CommandModule<{}, ClusterCreateArgs> = { 8 | command: "create", 9 | describe: "Create a new Differential cluster", 10 | builder: (yargs) => 11 | yargs.option("description", { 12 | describe: "Cluster Description", 13 | demandOption: false, 14 | type: "string", 15 | }), 16 | handler: async ({ description }) => { 17 | const d = await client.createClusterForUser({ 18 | body: { 19 | description: description ?? "CLI Created Cluster", 20 | }, 21 | }); 22 | 23 | if (d.status !== 204) { 24 | console.error(`Failed to create cluster: ${d.status}`); 25 | return; 26 | } else { 27 | console.log("Cluster created successfully"); 28 | 29 | const clusters = await client.getClustersForUser(); 30 | 31 | if (clusters.status === 200) { 32 | const cluster = clusters.body.sort((a, b) => 33 | a.createdAt > b.createdAt ? -1 : 1, 34 | )[0]; 35 | 36 | console.log(cluster); 37 | } 38 | } 39 | }, 40 | }; 41 | -------------------------------------------------------------------------------- /cli/src/commands/clusters-info.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from "yargs"; 2 | import { client } from "../lib/client"; 3 | import { selectCluster } from "../utils"; 4 | 5 | interface ClusterInfoArgs { 6 | cluster?: string; 7 | } 8 | export const ClusterInfo: CommandModule<{}, ClusterInfoArgs> = { 9 | command: "info", 10 | describe: "Display information about a Differential cluster", 11 | builder: (yargs) => 12 | yargs.option("cluster", { 13 | describe: "Cluster ID", 14 | demandOption: false, 15 | type: "string", 16 | }), 17 | handler: async ({ cluster }) => { 18 | if (!cluster) { 19 | cluster = await selectCluster(); 20 | if (!cluster) { 21 | console.log("No cluster selected"); 22 | return; 23 | } 24 | } 25 | 26 | getClusterDetails(cluster); 27 | }, 28 | }; 29 | 30 | const getClusterDetails = async (clusterId: string) => { 31 | const d = await client.getClusterDetailsForUser({ 32 | params: { 33 | clusterId, 34 | }, 35 | }); 36 | if (d.status !== 200) { 37 | console.error(`Failed to get cluster details: ${d.status}`); 38 | return; 39 | } 40 | console.log(d.body); 41 | }; 42 | -------------------------------------------------------------------------------- /cli/src/commands/clusters-list.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from "yargs"; 2 | import { client } from "../lib/client"; 3 | 4 | interface ClusterCreateArgs { 5 | description?: string; 6 | } 7 | export const ClusterList: CommandModule<{}, ClusterCreateArgs> = { 8 | command: "list", 9 | aliases: ["ls"], 10 | describe: "List Differential clusters", 11 | handler: async () => { 12 | const d = await client.getClustersForUser(); 13 | if (d.status !== 200) { 14 | console.error(`Failed to list clusters: ${d.status}`); 15 | return; 16 | } 17 | 18 | if (!d.body) { 19 | console.log("No clusters found"); 20 | return; 21 | } 22 | 23 | console.log( 24 | d.body.map((c) => ({ 25 | ...c, 26 | apiSecret: 27 | c.apiSecret.substring(0, 8) + "*".repeat(c.apiSecret.length - 8), 28 | info: `differential clusters info --cluster ${c.id}`, 29 | })), 30 | ); 31 | }, 32 | }; 33 | -------------------------------------------------------------------------------- /cli/src/commands/clusters-open.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from "yargs"; 2 | import { openBrowser, selectCluster } from "../utils"; 3 | import { readCurrentContext } from "../lib/context"; 4 | 5 | interface ClusterOpenArgs { 6 | cluster?: string; 7 | } 8 | export const ClusterOpen: CommandModule<{}, ClusterOpenArgs> = { 9 | command: "open", 10 | describe: "Open a Differential cluster in the console", 11 | builder: (yargs) => 12 | yargs.option("cluster", { 13 | describe: "Cluster ID", 14 | demandOption: false, 15 | type: "string", 16 | }), 17 | handler: async ({ cluster }) => { 18 | if (!cluster) { 19 | cluster = await selectCluster(); 20 | if (!cluster) { 21 | console.log("No cluster selected"); 22 | return; 23 | } 24 | } 25 | 26 | openBrowser( 27 | `${readCurrentContext().npmRegistryUrl}/clusters/${cluster}/overview`, 28 | ); 29 | }, 30 | }; 31 | -------------------------------------------------------------------------------- /cli/src/commands/clusters.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule, showHelp } from "yargs"; 2 | import { ClusterCreate } from "./clusters-create"; 3 | import { ClusterList } from "./clusters-list"; 4 | import { ClusterOpen } from "./clusters-open"; 5 | import { ClusterInfo } from "./clusters-info"; 6 | 7 | export const Clusters: CommandModule = { 8 | command: "clusters", 9 | aliases: ["cluster", "c"], 10 | describe: "Manage Differential clusters", 11 | builder: (yargs) => 12 | yargs 13 | .command(ClusterCreate) 14 | .command(ClusterList) 15 | .command(ClusterOpen) 16 | .command(ClusterInfo), 17 | handler: async () => { 18 | showHelp(); 19 | }, 20 | }; 21 | -------------------------------------------------------------------------------- /cli/src/commands/context-info.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from "yargs"; 2 | import { readCurrentContext } from "../lib/context"; 3 | 4 | export const ContextInfo: CommandModule = { 5 | command: "info", 6 | describe: "View CLI context details", 7 | handler: async () => { 8 | console.log(readCurrentContext()); 9 | }, 10 | }; 11 | -------------------------------------------------------------------------------- /cli/src/commands/context-set.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from "yargs"; 2 | import { CliContext, getCurrentContext, saveContext } from "../lib/context"; 3 | 4 | type ContextSetArgs = Partial & { 5 | context?: string; 6 | }; 7 | 8 | export const ContextSet: CommandModule<{}, ContextSetArgs> = { 9 | command: "set", 10 | describe: "Update CLI context values", 11 | builder: (yargs) => 12 | yargs 13 | .option("apiUrl", { 14 | describe: "Control Plane API host", 15 | demandOption: false, 16 | type: "string", 17 | }) 18 | .option("consoleUrl", { 19 | describe: "Management Console host", 20 | demandOption: false, 21 | type: "string", 22 | }) 23 | .option("cluster", { 24 | describe: "Cluster ID", 25 | demandOption: false, 26 | type: "string", 27 | }) 28 | .option("service", { 29 | describe: "Service name", 30 | demandOption: false, 31 | type: "string", 32 | }) 33 | .option("deployment", { 34 | describe: "Deployment id", 35 | demandOption: false, 36 | type: "string", 37 | }), 38 | handler: async ({ 39 | context, 40 | apiUrl, 41 | consoleUrl, 42 | cluster, 43 | service, 44 | deployment, 45 | }: ContextSetArgs) => { 46 | if (!context) { 47 | context = getCurrentContext(); 48 | } 49 | saveContext( 50 | { 51 | ...mapKey("apiUrl", apiUrl), 52 | ...mapKey("consoleUrl", consoleUrl), 53 | ...mapKey("cluster", cluster), 54 | ...mapKey("service", service), 55 | ...mapKey("deployment", deployment), 56 | }, 57 | context, 58 | ); 59 | }, 60 | }; 61 | 62 | const mapKey = (key: string, value?: string) => { 63 | if (value === undefined) return {}; 64 | 65 | if (value === "") { 66 | console.log(`Clearing context key: ${key}`); 67 | return { [key]: undefined }; 68 | } 69 | 70 | return { [key]: value }; 71 | }; 72 | -------------------------------------------------------------------------------- /cli/src/commands/context.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from "yargs"; 2 | import { ContextSet } from "./context-set"; 3 | import { ContextInfo } from "./context-info"; 4 | 5 | export const Context: CommandModule = { 6 | command: "context", 7 | aliases: ["context"], 8 | describe: "Manage Differential CLI context", 9 | builder: (yargs) => yargs.command(ContextInfo).command(ContextSet), 10 | handler: ContextInfo.handler, 11 | }; 12 | -------------------------------------------------------------------------------- /cli/src/commands/deploy-info.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from "yargs"; 2 | import { client } from "../lib/client"; 3 | import { cloudEnabledCheck } from "../lib/auth"; 4 | 5 | interface DeployInfoArgs { 6 | cluster: string; 7 | service: string; 8 | deployment: string; 9 | } 10 | export const DeployInfo: CommandModule<{}, DeployInfoArgs> = { 11 | command: "info", 12 | describe: "Display information about a Differential cloud deployment", 13 | builder: (yargs) => 14 | yargs 15 | .option("cluster", { 16 | describe: "Cluster ID", 17 | demandOption: true, 18 | type: "string", 19 | }) 20 | .option("service", { 21 | describe: "Service name", 22 | demandOption: true, 23 | type: "string", 24 | }) 25 | .option("deployment", { 26 | describe: "Deployment ID", 27 | demandOption: true, 28 | type: "string", 29 | }), 30 | handler: async ({ cluster, service, deployment }) => { 31 | if (!(await cloudEnabledCheck(cluster))) { 32 | return; 33 | } 34 | 35 | const d = await client.getDeployment({ 36 | params: { 37 | clusterId: cluster, 38 | serviceName: service, 39 | deploymentId: deployment, 40 | }, 41 | }); 42 | if (d.status !== 200) { 43 | console.error(`Failed to fetch deployment info: ${d.status}`); 44 | return; 45 | } 46 | 47 | console.log(d.body); 48 | }, 49 | }; 50 | -------------------------------------------------------------------------------- /cli/src/commands/deploy-list.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from "yargs"; 2 | import { client } from "../lib/client"; 3 | import { selectCluster, selectService } from "../utils"; 4 | import { cloudEnabledCheck } from "../lib/auth"; 5 | 6 | interface DeployListArgs { 7 | cluster?: string; 8 | service?: string; 9 | } 10 | export const DeployList: CommandModule<{}, DeployListArgs> = { 11 | command: "list", 12 | aliases: ["ls"], 13 | describe: "List Differential cloud deployments", 14 | builder: (yargs) => 15 | yargs 16 | .option("cluster", { 17 | describe: "Cluster ID", 18 | demandOption: false, 19 | type: "string", 20 | }) 21 | .option("service", { 22 | describe: "Service name", 23 | demandOption: false, 24 | type: "string", 25 | }), 26 | handler: async ({ cluster, service }) => { 27 | if (!cluster) { 28 | cluster = await selectCluster(); 29 | if (!cluster) { 30 | console.log("No cluster selected"); 31 | return; 32 | } 33 | } 34 | 35 | if (!(await cloudEnabledCheck(cluster))) { 36 | return; 37 | } 38 | 39 | if (!service) { 40 | service = await selectService(cluster); 41 | if (!service) { 42 | console.log("No service selected"); 43 | return; 44 | } 45 | } 46 | 47 | const d = await client.getDeployments({ 48 | params: { 49 | clusterId: cluster, 50 | serviceName: service, 51 | }, 52 | }); 53 | if (d.status !== 200) { 54 | console.error(`Failed to fetch deployment info: ${d.status}`); 55 | return; 56 | } 57 | 58 | console.log(d.body); 59 | }, 60 | }; 61 | -------------------------------------------------------------------------------- /cli/src/commands/deploy.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule, showHelp } from "yargs"; 2 | import { DeployCreate } from "./deploy-create"; 3 | import { DeployInfo } from "./deploy-info"; 4 | import { DeployList } from "./deploy-list"; 5 | import { DeployLogs } from "./deploy-logs"; 6 | 7 | export const Deploy: CommandModule = { 8 | command: "deployments", 9 | aliases: ["deployment", "deploy", "d"], 10 | describe: "Manage Differential cloud deployments", 11 | builder: (yargs) => 12 | yargs 13 | .command(DeployCreate) 14 | .command(DeployList) 15 | .command(DeployInfo) 16 | .command(DeployLogs), 17 | handler: async () => { 18 | showHelp(); 19 | }, 20 | }; 21 | -------------------------------------------------------------------------------- /cli/src/commands/services-register.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import { Project } from "ts-morph"; 3 | import { CommandModule } from "yargs"; 4 | import { client } from "../lib/client"; 5 | import { selectCluster } from "../utils"; 6 | 7 | interface ServicesRegisterArgs { 8 | cluster?: string; 9 | service: string; 10 | } 11 | 12 | export const ServicesRegister: CommandModule<{}, ServicesRegisterArgs> = { 13 | command: "register", 14 | describe: "Explicitly register a service with Cluster", 15 | builder: (yargs) => 16 | yargs 17 | .option("cluster", { 18 | describe: "Cluster ID", 19 | demandOption: false, 20 | type: "string", 21 | }) 22 | .option("service", { 23 | describe: "Service name", 24 | demandOption: true, 25 | type: "string", 26 | }), 27 | handler: async ({ cluster, service }) => { 28 | if (!cluster) { 29 | cluster = await selectCluster(); 30 | if (!cluster) { 31 | console.log("No cluster selected"); 32 | process.exit(1); 33 | } 34 | } 35 | 36 | // check if the name.service.ts exists 37 | if (!fs.existsSync(`${service}.service.ts`)) { 38 | throw new Error(`Service file ${service}.service.ts not found`); 39 | } 40 | 41 | const project = new Project({ 42 | compilerOptions: { 43 | declaration: true, 44 | emitDecoratorMetadata: true, 45 | emitDeclarationOnly: true, 46 | outFile: "service.d.ts", 47 | }, 48 | }); 49 | 50 | project.addSourceFileAtPath(`${service}.service.ts`); 51 | 52 | const result = project.emitToMemory(); 53 | 54 | const typesText = result 55 | .getFiles() 56 | .find((x) => x.filePath.includes("service.d.ts"))?.text; 57 | 58 | if (!typesText) { 59 | throw new Error("Failed to generate types for service"); 60 | } 61 | 62 | const storeResult = await client.storeSchema({ 63 | body: { 64 | schema: typesText, 65 | }, 66 | params: { 67 | clusterId: cluster, 68 | serviceName: service, 69 | }, 70 | }); 71 | 72 | if (storeResult.status !== 204) { 73 | console.log("Failed to register service", { 74 | result: storeResult, 75 | }); 76 | } 77 | 78 | console.log(`Service ${service} registered with cluster ${cluster}`); 79 | }, 80 | }; 81 | -------------------------------------------------------------------------------- /cli/src/commands/services.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule, showHelp } from "yargs"; 2 | import { ServicesRegister } from "./services-register"; 3 | 4 | export const Services: CommandModule = { 5 | command: "services", 6 | aliases: ["s"], 7 | describe: "Manage Differential services", 8 | builder: (yargs) => yargs.command(ServicesRegister), 9 | handler: async () => { 10 | showHelp(); 11 | }, 12 | }; 13 | -------------------------------------------------------------------------------- /cli/src/commands/task.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from "yargs"; 2 | import { selectCluster } from "../utils"; 3 | import { client } from "../lib/client"; 4 | import { input } from "@inquirer/prompts"; 5 | 6 | interface TaskArgs { 7 | cluster?: string; 8 | task?: string; 9 | } 10 | 11 | export const Task: CommandModule<{}, TaskArgs> = { 12 | command: "task", 13 | describe: "Execute a task in the cluster using a human readable prompt", 14 | builder: (yargs) => 15 | yargs 16 | .option("cluster", { 17 | describe: "Cluster ID", 18 | demandOption: false, 19 | type: "string", 20 | }) 21 | .option("task", { 22 | describe: "Task for the cluster to perform", 23 | demandOption: false, 24 | type: "string", 25 | }), 26 | handler: async ({ cluster, task }) => { 27 | if (!cluster) { 28 | cluster = await selectCluster(); 29 | if (!cluster) { 30 | console.log("No cluster selected"); 31 | return; 32 | } 33 | } 34 | 35 | if (!task) { 36 | task = await input({ 37 | message: "Human readable prompt for the cluster to perform", 38 | validate: (value) => { 39 | if (!value) { 40 | return "Prompt is required"; 41 | } 42 | return true; 43 | }, 44 | }); 45 | } 46 | 47 | try { 48 | const result = await executeTask(cluster, task); 49 | console.log(result); 50 | } catch (e) { 51 | console.error(e); 52 | } 53 | }, 54 | }; 55 | 56 | const executeTask = async (clusterId: string, task: string) => { 57 | const result = await client.executeTask({ 58 | params: { 59 | clusterId, 60 | }, 61 | body: { 62 | task, 63 | }, 64 | }); 65 | 66 | if (result.status !== 200) { 67 | throw new Error(`Failed to prompt cluster: ${result.status}`); 68 | } 69 | return result.body.result; 70 | }; 71 | -------------------------------------------------------------------------------- /cli/src/constants.ts: -------------------------------------------------------------------------------- 1 | import { CliContext } from "./lib/context"; 2 | 3 | export const DEFAULT_CONSOLE_URL = "https://console.differential.dev"; 4 | export const DEFAULT_API_URL = "https://api.differential.dev"; 5 | export const NPM_REGISTRY_PATH = `/packages/npm/`; 6 | export const CLIENT_PACKAGE_SCOPE = "@differential.dev"; 7 | 8 | export const DEFAULT_CLI_CONTEXT: CliContext = { 9 | apiUrl: DEFAULT_API_URL, 10 | consoleUrl: DEFAULT_CONSOLE_URL, 11 | npmRegistryUrl: DEFAULT_API_URL + NPM_REGISTRY_PATH, 12 | }; 13 | -------------------------------------------------------------------------------- /cli/src/index.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import yargs from "yargs"; 4 | import { hideBin } from "yargs/helpers"; 5 | 6 | import { Deploy } from "./commands/deploy"; 7 | import { Auth } from "./commands/auth"; 8 | import { getToken } from "./lib/auth"; 9 | import { Clusters } from "./commands/clusters"; 10 | import { ClientLibrary } from "./commands/client-lib"; 11 | import { Repl } from "./commands/repl"; 12 | import { Context } from "./commands/context"; 13 | import { setCurrentContext } from "./lib/context"; 14 | import { Services } from "./commands/services"; 15 | import { Task } from "./commands/task"; 16 | 17 | const cli = yargs(hideBin(process.argv)) 18 | .scriptName("differential") 19 | .strict() 20 | .hide("version") 21 | .option("context", { 22 | describe: "Configuration context", 23 | demandOption: false, 24 | default: "default", 25 | type: "string", 26 | }) 27 | .middleware(setCurrentContext) 28 | .demandCommand() 29 | .command(Auth); 30 | 31 | const authenticated = getToken(); 32 | if (authenticated) { 33 | cli 34 | .command(Deploy) 35 | .command(Clusters) 36 | .command(ClientLibrary) 37 | .command(Repl) 38 | .command(Context) 39 | .command(Task) 40 | .command(Services); 41 | } else { 42 | console.log( 43 | "Diffential CLI is not authenticated. Please run `differential auth login` to authenticate.", 44 | ); 45 | } 46 | 47 | cli.argv; 48 | -------------------------------------------------------------------------------- /cli/src/lib/client.ts: -------------------------------------------------------------------------------- 1 | import { initClient, tsRestFetchApi } from "@ts-rest/core"; 2 | import { contract } from "../client/contract"; 3 | 4 | import { getToken } from "../lib/auth"; 5 | import { readCurrentContext } from "./context"; 6 | 7 | export const client = initClient(contract, { 8 | baseUrl: readCurrentContext().apiUrl, 9 | baseHeaders: { 10 | authorization: `Bearer ${getToken()}`, 11 | }, 12 | api: tsRestFetchApi, 13 | }); 14 | 15 | export const waitForDeploymentStatus = async ( 16 | deploymentId: string, 17 | serviceName: string, 18 | clusterId: string, 19 | targets: string[], 20 | maxAttempts: number = 10, 21 | ): Promise => { 22 | const checkDeployment = () => 23 | client.getDeployment({ 24 | params: { 25 | deploymentId, 26 | clusterId, 27 | serviceName, 28 | }, 29 | }); 30 | 31 | let attempts = 0; 32 | while (attempts < maxAttempts) { 33 | attempts++; 34 | const result = await checkDeployment(); 35 | 36 | if (result.status == 200 && targets.includes(result.body.status)) { 37 | return result.body.status; 38 | } 39 | await new Promise((resolve) => setTimeout(resolve, 1000)); 40 | } 41 | 42 | throw new Error( 43 | "Failed to check deployment status. Please check provided options and cluster configuration.", 44 | ); 45 | }; 46 | -------------------------------------------------------------------------------- /cli/src/lib/context.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import fs from "fs"; 3 | import { DEFAULT_CLI_CONTEXT } from "../constants"; 4 | 5 | export type CliContext = { 6 | apiUrl: string; 7 | consoleUrl: string; 8 | npmRegistryUrl: string; 9 | cluster?: string; 10 | service?: string; 11 | deployment?: string; 12 | }; 13 | 14 | let CURRENT_CONTEXT = "default"; 15 | 16 | export const setCurrentContext = ({ context }: { context: string }) => { 17 | console.log(`Running with configuration context: ${context}`); 18 | CURRENT_CONTEXT = context; 19 | }; 20 | 21 | export const getCurrentContext = () => { 22 | return CURRENT_CONTEXT; 23 | }; 24 | 25 | export const saveContext = ( 26 | inputContext: Partial, 27 | name?: string, 28 | ) => { 29 | const updatedContext = { 30 | ...readContext(name), 31 | ...inputContext, 32 | }; 33 | 34 | fs.writeFileSync(getContextPath(name), JSON.stringify(updatedContext)); 35 | }; 36 | 37 | // Loads the context specified or the "default" context if not. 38 | export const readContext = (name?: string): CliContext => { 39 | let context = DEFAULT_CLI_CONTEXT; 40 | 41 | // Load the default context for merging 42 | if (name) { 43 | context = readContext(); 44 | } 45 | 46 | const contextFile = getContextPath(name); 47 | if (fs.existsSync(contextFile)) { 48 | const file = fs.readFileSync(contextFile, { encoding: "utf-8" }); 49 | // Merge default context 50 | context = { 51 | ...context, 52 | ...JSON.parse(file), 53 | }; 54 | } 55 | return context; 56 | }; 57 | 58 | export const readCurrentContext = () => { 59 | return readContext(getCurrentContext()); 60 | }; 61 | 62 | const getContextPath = (name: string = "default") => { 63 | return path.join( 64 | process.cwd(), 65 | name === "default" ? "differential.json" : `differential.${name}.json`, 66 | ); 67 | }; 68 | -------------------------------------------------------------------------------- /cli/src/lib/release.ts: -------------------------------------------------------------------------------- 1 | import { client } from "./client"; 2 | 3 | export const release = async ({ 4 | deploymentId, 5 | clusterId, 6 | serviceName, 7 | }: { 8 | deploymentId: string; 9 | clusterId: string; 10 | serviceName: string; 11 | }) => { 12 | const result = await client.releaseDeployment({ 13 | params: { 14 | deploymentId, 15 | clusterId, 16 | serviceName, 17 | }, 18 | }); 19 | if (result.status !== 200) { 20 | throw new Error( 21 | "Failed to deploy service. Please check provided options and cluster configuration.", 22 | ); 23 | } 24 | }; 25 | -------------------------------------------------------------------------------- /cli/src/lib/upload.ts: -------------------------------------------------------------------------------- 1 | import debug from "debug"; 2 | import { readFileSync } from "fs"; 3 | import { client } from "./client"; 4 | 5 | const log = debug("differential:cli:upload"); 6 | 7 | export const uploadAsset = async ({ 8 | path, 9 | target, 10 | contentType, 11 | type, 12 | cluster, 13 | }: { 14 | path: string; 15 | target: string; 16 | contentType: string; 17 | type: "client_library" | "service_bundle"; 18 | cluster: string; 19 | }): Promise => { 20 | log(`Uploading asset`); 21 | 22 | const upload = await client.createAsset({ 23 | body: { 24 | type, 25 | target, 26 | }, 27 | params: { 28 | clusterId: cluster, 29 | }, 30 | }); 31 | 32 | log("Response from createAsset", upload); 33 | 34 | if (upload.status !== 201) { 35 | throw new Error( 36 | "Failed to upload asset. Please check provided options and cluster configuration.", 37 | ); 38 | } 39 | 40 | const { presignedUrl } = upload.body; 41 | 42 | log("Uploading asset to s3", { presignedUrl }); 43 | 44 | const response = await fetch(presignedUrl, { 45 | method: "PUT", 46 | body: readFileSync(path), 47 | headers: { 48 | "Content-Type": contentType, 49 | }, 50 | }); 51 | 52 | log("Response from S3 put", response); 53 | 54 | if (response.status !== 200) { 55 | throw new Error( 56 | "Failed to upload asset. Please check provided options and cluster configuration.", 57 | ); 58 | } 59 | }; 60 | -------------------------------------------------------------------------------- /cli/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "commonjs", 5 | "outDir": "bin", 6 | "rootDir": "src", 7 | "declaration": true, 8 | "sourceMap": true, 9 | "strict": true, 10 | "esModuleInterop": true, 11 | "allowJs": true 12 | }, 13 | "include": [ 14 | "src/**/*" 15 | ], 16 | "exclude": [ 17 | "node_modules" 18 | ] 19 | } -------------------------------------------------------------------------------- /control-plane/.dockerignore: -------------------------------------------------------------------------------- 1 | .env -------------------------------------------------------------------------------- /control-plane/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## [0.0.16](https://github.com/differentialHQ/differential/compare/v0.0.0...v0.0.16) (2023-12-24) 2 | 3 | 4 | ### Features 5 | 6 | * Adding function support ([372b897](https://github.com/differentialHQ/differential/commit/372b897dbad2ea3b871d6e5c0bdb28d121995cd8)) 7 | * Adding long polling support for control-plane ([98a6b69](https://github.com/differentialHQ/differential/commit/98a6b69b340cfa67c3fa1759d0de9cfdf6c8f7ec)) 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /control-plane/Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax = docker/dockerfile:1 2 | 3 | # Adjust NODE_VERSION as desired 4 | ARG NODE_VERSION=18.7.0 5 | FROM node:${NODE_VERSION}-slim as base 6 | 7 | LABEL fly_launch_runtime="NodeJS" 8 | 9 | # NodeJS app lives here 10 | WORKDIR /app 11 | 12 | # Throw-away build stage to reduce size of final image 13 | FROM base as build 14 | 15 | # Install packages needed to build node modules 16 | RUN apt-get update -qq && \ 17 | apt-get install -y python-is-python3 pkg-config build-essential 18 | 19 | # Install node modules 20 | COPY --link package.json package-lock.json ./ 21 | RUN npm install 22 | 23 | # Copy application code 24 | COPY --link . . 25 | 26 | # Remove src/test directory 27 | RUN rm -rf src/test 28 | 29 | RUN npm run build 30 | 31 | # Remove development dependencies 32 | RUN npm install --production=true 33 | 34 | # Final stage for app image 35 | FROM base 36 | 37 | # Set production environment 38 | ENV NODE_ENV=production 39 | 40 | # Copy built application 41 | COPY --from=build /app /app 42 | 43 | # Start the server by default, this can be overwritten at runtime 44 | CMD [ "npm", "run", "start" ] 45 | -------------------------------------------------------------------------------- /control-plane/Dockerfile.dev: -------------------------------------------------------------------------------- 1 | FROM node:20 2 | 3 | WORKDIR /app 4 | 5 | ADD . . 6 | 7 | RUN npm install 8 | 9 | ENTRYPOINT [ "npm", "run", "dev" ] -------------------------------------------------------------------------------- /control-plane/README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 | ## Overview 6 | 7 | This repo contains the source code for the Differential control plane. Control plane acts as an application code aware service mesh, and a distributed orchestrator. It implements a few things on top of Postgres: 8 | 9 | - Queue with exactly-once processing semantics 10 | - Service & function registry 11 | - Utilities for running functions in a distributed environment 12 | - APIs for interacting with the control plane 13 | - APIs for ingesting metrics and logs 14 | 15 | See the [Documentation](https://docs.differential.dev) for more information. 16 | 17 | ## Contributing 18 | 19 | We welcome contributions to Differential! Please see the [Contributing Guide](../CONTRIBUTING.md) for more information. 20 | -------------------------------------------------------------------------------- /control-plane/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | ["@babel/preset-env", { targets: { node: "current" } }], 4 | "@babel/preset-typescript", 5 | ], 6 | }; 7 | -------------------------------------------------------------------------------- /control-plane/drizzle/0000_odd_tattoo.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS "environments" ( 2 | "id" varchar(70) PRIMARY KEY NOT NULL, 3 | "user_id" varchar NOT NULL, 4 | "name" varchar(255) NOT NULL 5 | ); 6 | --> statement-breakpoint 7 | CREATE TABLE IF NOT EXISTS "jobs" ( 8 | "id" varchar(70) PRIMARY KEY NOT NULL, 9 | "environment_id" varchar NOT NULL, 10 | "target_fn" varchar(255) NOT NULL, 11 | "target_args" text NOT NULL, 12 | "idempotency_key" varchar(255) NOT NULL, 13 | "status" text NOT NULL, 14 | "result" text, 15 | "created_at" timestamp DEFAULT now() NOT NULL, 16 | "updated_at" timestamp DEFAULT now() NOT NULL 17 | ); 18 | --> statement-breakpoint 19 | CREATE TABLE IF NOT EXISTS "machine_callable_targets" ( 20 | "id" varchar(70) PRIMARY KEY NOT NULL, 21 | "machine_id" varchar NOT NULL, 22 | "target_fn" varchar(255) NOT NULL 23 | ); 24 | --> statement-breakpoint 25 | CREATE TABLE IF NOT EXISTS "machines" ( 26 | "id" varchar(70) PRIMARY KEY NOT NULL, 27 | "environment_id" varchar NOT NULL, 28 | "description" varchar(255), 29 | "class" varchar(255), 30 | "last_ping_at" timestamp 31 | ); 32 | --> statement-breakpoint 33 | CREATE TABLE IF NOT EXISTS "users" ( 34 | "id" varchar(70) PRIMARY KEY NOT NULL, 35 | "secret_key" varchar(255) NOT NULL 36 | ); 37 | --> statement-breakpoint 38 | DO $$ BEGIN 39 | ALTER TABLE "environments" ADD CONSTRAINT "environments_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE no action ON UPDATE no action; 40 | EXCEPTION 41 | WHEN duplicate_object THEN null; 42 | END $$; 43 | --> statement-breakpoint 44 | DO $$ BEGIN 45 | ALTER TABLE "jobs" ADD CONSTRAINT "jobs_environment_id_environments_id_fk" FOREIGN KEY ("environment_id") REFERENCES "environments"("id") ON DELETE no action ON UPDATE no action; 46 | EXCEPTION 47 | WHEN duplicate_object THEN null; 48 | END $$; 49 | --> statement-breakpoint 50 | DO $$ BEGIN 51 | ALTER TABLE "machine_callable_targets" ADD CONSTRAINT "machine_callable_targets_machine_id_machines_id_fk" FOREIGN KEY ("machine_id") REFERENCES "machines"("id") ON DELETE no action ON UPDATE no action; 52 | EXCEPTION 53 | WHEN duplicate_object THEN null; 54 | END $$; 55 | --> statement-breakpoint 56 | DO $$ BEGIN 57 | ALTER TABLE "machines" ADD CONSTRAINT "machines_environment_id_environments_id_fk" FOREIGN KEY ("environment_id") REFERENCES "environments"("id") ON DELETE no action ON UPDATE no action; 58 | EXCEPTION 59 | WHEN duplicate_object THEN null; 60 | END $$; 61 | -------------------------------------------------------------------------------- /control-plane/drizzle/0001_tense_dreaming_celestial.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" DROP CONSTRAINT "jobs_environment_id_environments_id_fk"; 2 | --> statement-breakpoint 3 | ALTER TABLE "machines" DROP CONSTRAINT "machines_environment_id_environments_id_fk"; 4 | --> statement-breakpoint 5 | DROP TABLE "environments";--> statement-breakpoint 6 | DROP TABLE "machine_callable_targets";--> statement-breakpoint 7 | DROP TABLE "users"; -------------------------------------------------------------------------------- /control-plane/drizzle/0002_steady_korath.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" ALTER COLUMN "id" SET DATA TYPE varchar(1024);--> statement-breakpoint 2 | ALTER TABLE "jobs" ALTER COLUMN "target_fn" SET DATA TYPE varchar(1024);--> statement-breakpoint 3 | ALTER TABLE "jobs" ALTER COLUMN "idempotency_key" SET DATA TYPE varchar(1024);--> statement-breakpoint 4 | ALTER TABLE "machines" ALTER COLUMN "id" SET DATA TYPE varchar(1024);--> statement-breakpoint 5 | ALTER TABLE "machines" ALTER COLUMN "description" SET DATA TYPE varchar(1024);--> statement-breakpoint 6 | ALTER TABLE "machines" ALTER COLUMN "class" SET DATA TYPE varchar(1024); -------------------------------------------------------------------------------- /control-plane/drizzle/0003_living_captain_america.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" ADD COLUMN "result_type" text;--> statement-breakpoint 2 | ALTER TABLE "jobs" ADD COLUMN "remaining" integer DEFAULT 1; -------------------------------------------------------------------------------- /control-plane/drizzle/0004_confused_goliath.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" ADD COLUMN "owner_id" varchar(1024); -------------------------------------------------------------------------------- /control-plane/drizzle/0005_overconfident_hex.sql: -------------------------------------------------------------------------------- 1 | UPDATE "jobs" SET "owner_id" = 'fake_hash' WHERE "owner_id" IS NULL;--> statement-breakpoint 2 | 3 | ALTER TABLE "jobs" RENAME COLUMN "owner_id" TO "owner_hash";--> statement-breakpoint 4 | ALTER TABLE "jobs" ALTER COLUMN "owner_hash" SET DATA TYPE text;--> statement-breakpoint 5 | ALTER TABLE "jobs" ALTER COLUMN "owner_hash" SET NOT NULL; -------------------------------------------------------------------------------- /control-plane/drizzle/0006_ambiguous_wild_pack.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" ADD COLUMN "machine_type" text; -------------------------------------------------------------------------------- /control-plane/drizzle/0007_icy_the_hand.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS "credentials" ( 2 | "id" varchar(1024) PRIMARY KEY NOT NULL, 3 | "api_key" varchar(1024) NOT NULL, 4 | "api_secret" varchar(1024) NOT NULL, 5 | "environment_id" varchar NOT NULL, 6 | "user_id" varchar NOT NULL, 7 | "created_at" timestamp DEFAULT now() NOT NULL 8 | ); 9 | -------------------------------------------------------------------------------- /control-plane/drizzle/0008_omniscient_mongu.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "credentials" DROP COLUMN IF EXISTS "api_key"; -------------------------------------------------------------------------------- /control-plane/drizzle/0009_cuddly_medusa.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "credentials" RENAME COLUMN "user_id" TO "organization_id";--> statement-breakpoint 2 | ALTER TABLE "credentials" DROP COLUMN IF EXISTS "environment_id";--> statement-breakpoint 3 | ALTER TABLE "jobs" DROP COLUMN IF EXISTS "environment_id";--> statement-breakpoint 4 | ALTER TABLE "machines" DROP COLUMN IF EXISTS "environment_id"; -------------------------------------------------------------------------------- /control-plane/drizzle/0010_curious_amphibian.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "credentials" RENAME TO "clusters"; -------------------------------------------------------------------------------- /control-plane/drizzle/0011_thin_preak.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "machines" ADD COLUMN "ip" varchar(1024);--> statement-breakpoint 2 | ALTER TABLE "machines" ADD COLUMN "cluster_id" varchar NOT NULL; -------------------------------------------------------------------------------- /control-plane/drizzle/0012_manual_make.sql: -------------------------------------------------------------------------------- 1 | -- fill NULL cluster_id in the table machines with -1 2 | UPDATE machines SET cluster_id = '-1' WHERE cluster_id IS NULL; 3 | -------------------------------------------------------------------------------- /control-plane/drizzle/0013_bitter_lily_hollister.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "machines" ALTER COLUMN "cluster_id" SET NOT NULL; -------------------------------------------------------------------------------- /control-plane/drizzle/0014_kind_mercury.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "clusters" ALTER COLUMN "created_at" SET DATA TYPE timestamp with time zone;--> statement-breakpoint 2 | ALTER TABLE "jobs" ALTER COLUMN "created_at" SET DATA TYPE timestamp with time zone;--> statement-breakpoint 3 | ALTER TABLE "jobs" ALTER COLUMN "updated_at" SET DATA TYPE timestamp with time zone;--> statement-breakpoint 4 | ALTER TABLE "machines" ALTER COLUMN "last_ping_at" SET DATA TYPE timestamp with time zone; -------------------------------------------------------------------------------- /control-plane/drizzle/0015_flimsy_microbe.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" ADD COLUMN "timed_out_at" timestamp with time zone DEFAULT now() + interval '300 seconds';--> statement-breakpoint 2 | ALTER TABLE "jobs" ADD COLUMN "timeout_interval" integer DEFAULT 300; -------------------------------------------------------------------------------- /control-plane/drizzle/0016_unique_wallflower.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "clusters" ADD COLUMN "wake_up_config" json; -------------------------------------------------------------------------------- /control-plane/drizzle/0017_purple_captain_stacy.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS "users" ( 2 | "id" varchar(1024) PRIMARY KEY NOT NULL, 3 | "email" varchar(1024) NOT NULL, 4 | "created_at" timestamp with time zone DEFAULT now() NOT NULL 5 | ); 6 | --> statement-breakpoint 7 | ALTER TABLE "clusters" ALTER COLUMN "organization_id" DROP NOT NULL;--> statement-breakpoint 8 | ALTER TABLE "clusters" ADD COLUMN "description" varchar(1024);--> statement-breakpoint 9 | ALTER TABLE "clusters" ADD COLUMN "owner_id" varchar;--> statement-breakpoint 10 | DO $$ BEGIN 11 | ALTER TABLE "clusters" ADD CONSTRAINT "clusters_owner_id_users_id_fk" FOREIGN KEY ("owner_id") REFERENCES "users"("id") ON DELETE no action ON UPDATE no action; 12 | EXCEPTION 13 | WHEN duplicate_object THEN null; 14 | END $$; 15 | -------------------------------------------------------------------------------- /control-plane/drizzle/0018_spooky_sasquatch.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "clusters" DROP CONSTRAINT "clusters_owner_id_users_id_fk"; 2 | -------------------------------------------------------------------------------- /control-plane/drizzle/0019_early_silver_sable.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE "users";--> statement-breakpoint 2 | ALTER TABLE "jobs" ADD COLUMN "resulted_at" timestamp with time zone;--> statement-breakpoint 3 | ALTER TABLE "jobs" ADD COLUMN "function_execution_time_ms" integer;--> statement-breakpoint 4 | ALTER TABLE "jobs" ADD COLUMN IF NOT EXISTS "service" varchar(1024); 5 | -------------------------------------------------------------------------------- /control-plane/drizzle/0020_third_lockjaw.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" DROP CONSTRAINT "jobs_pkey";--> statement-breakpoint 2 | ALTER TABLE "jobs" ADD CONSTRAINT "jobs_owner_hash_target_fn_idempotency_key" PRIMARY KEY("owner_hash","target_fn","idempotency_key"); -------------------------------------------------------------------------------- /control-plane/drizzle/0021_brave_eternals.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS "services" ( 2 | "cluster_id" varchar NOT NULL, 3 | "machine_id" varchar NOT NULL, 4 | "service" varchar(1024) NOT NULL, 5 | "definition" json NOT NULL 6 | ); 7 | --> statement-breakpoint 8 | DO $$ BEGIN 9 | ALTER TABLE "services" ADD CONSTRAINT "services_cluster_id_clusters_id_fk" FOREIGN KEY ("cluster_id") REFERENCES "clusters"("id") ON DELETE no action ON UPDATE no action; 10 | EXCEPTION 11 | WHEN duplicate_object THEN null; 12 | END $$; 13 | --> statement-breakpoint 14 | DO $$ BEGIN 15 | ALTER TABLE "services" ADD CONSTRAINT "services_machine_id_machines_id_fk" FOREIGN KEY ("machine_id") REFERENCES "machines"("id") ON DELETE no action ON UPDATE no action; 16 | EXCEPTION 17 | WHEN duplicate_object THEN null; 18 | END $$; 19 | -------------------------------------------------------------------------------- /control-plane/drizzle/0022_bizarre_princess_powerful.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "services" DROP CONSTRAINT "services_machine_id_machines_id_fk"; 2 | --> statement-breakpoint 3 | ALTER TABLE "services" DROP COLUMN IF EXISTS "machine_id";--> statement-breakpoint 4 | ALTER TABLE "services" ADD CONSTRAINT "services_cluster_id_service" PRIMARY KEY("cluster_id","service"); -------------------------------------------------------------------------------- /control-plane/drizzle/0023_lazy_dakota_north.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" ALTER COLUMN "service" SET NOT NULL;--> statement-breakpoint 2 | ALTER TABLE "jobs" ADD COLUMN "cache_key" varchar(1024); -------------------------------------------------------------------------------- /control-plane/drizzle/0024_thin_black_bolt.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" RENAME COLUMN "remaining" TO "remaining_attempts";--> statement-breakpoint 2 | ALTER TABLE "jobs" ALTER COLUMN "timeout_interval" DROP DEFAULT;--> statement-breakpoint 3 | ALTER TABLE "jobs" ADD COLUMN "last_retrieved_at" timestamp with time zone;--> statement-breakpoint 4 | ALTER TABLE "jobs" DROP COLUMN IF EXISTS "timed_out_at"; -------------------------------------------------------------------------------- /control-plane/drizzle/0025_hot_shinobi_shaw.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" RENAME COLUMN "timeout_interval" TO "timeout_interval_seconds"; -------------------------------------------------------------------------------- /control-plane/drizzle/0026_freezing_clint_barton.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" ADD CONSTRAINT "jobs_id_unique" UNIQUE("id"); 2 | --> statement-breakpoint 3 | CREATE TABLE IF NOT EXISTS "events" ( 4 | "id" varchar(1024) PRIMARY KEY NOT NULL, 5 | "cluster_id" varchar, 6 | "type" varchar(1024), 7 | "job_id" varchar(1024), 8 | "machine_id" varchar(1024), 9 | "service" varchar(1024), 10 | "created_at" timestamp with time zone NOT NULL, 11 | "meta" json 12 | ); 13 | --> statement-breakpoint 14 | DO $$ BEGIN 15 | ALTER TABLE "events" ADD CONSTRAINT "events_cluster_id_clusters_id_fk" FOREIGN KEY ("cluster_id") REFERENCES "clusters"("id") ON DELETE no action ON UPDATE no action; 16 | EXCEPTION 17 | WHEN duplicate_object THEN null; 18 | END $$; 19 | --> statement-breakpoint 20 | DO $$ BEGIN 21 | ALTER TABLE "events" ADD CONSTRAINT "events_job_id_jobs_id_fk" FOREIGN KEY ("job_id") REFERENCES "jobs"("id") ON DELETE no action ON UPDATE no action; 22 | EXCEPTION 23 | WHEN duplicate_object THEN null; 24 | END $$; 25 | --> statement-breakpoint 26 | DO $$ BEGIN 27 | ALTER TABLE "events" ADD CONSTRAINT "events_machine_id_machines_id_fk" FOREIGN KEY ("machine_id") REFERENCES "machines"("id") ON DELETE no action ON UPDATE no action; 28 | EXCEPTION 29 | WHEN duplicate_object THEN null; 30 | END $$; -------------------------------------------------------------------------------- /control-plane/drizzle/0027_lyrical_ken_ellis.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "events" ALTER COLUMN "cluster_id" SET NOT NULL;--> statement-breakpoint 2 | ALTER TABLE "events" ALTER COLUMN "type" SET NOT NULL; -------------------------------------------------------------------------------- /control-plane/drizzle/0028_parallel_karen_page.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "events" ALTER COLUMN "created_at" SET DATA TYPE timestamp (3) with time zone; -------------------------------------------------------------------------------- /control-plane/drizzle/0029_sad_thunderbird.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "events" ALTER COLUMN "created_at" SET DATA TYPE timestamp (6) with time zone; -------------------------------------------------------------------------------- /control-plane/drizzle/0030_omniscient_jamie_braddock.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "clusters" ADD COLUMN IF NOT EXISTS "cloud_enabled" boolean DEFAULT false; -------------------------------------------------------------------------------- /control-plane/drizzle/0031_misty_slipstream.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS "deployments" ( 2 | "id" varchar(1024) PRIMARY KEY NOT NULL, 3 | "cluster_id" varchar NOT NULL, 4 | "service" varchar(1024) NOT NULL, 5 | "created_at" timestamp (6) with time zone DEFAULT now() NOT NULL, 6 | "package_upload_path" varchar(1024) NOT NULL, 7 | "definition_upload_url" varchar(1024) NOT NULL, 8 | "status" text DEFAULT 'uploading' NOT NULL 9 | ); 10 | --> statement-breakpoint 11 | DO $$ BEGIN 12 | ALTER TABLE "deployments" ADD CONSTRAINT "deployments_cluster_id_clusters_id_fk" FOREIGN KEY ("cluster_id") REFERENCES "clusters"("id") ON DELETE no action ON UPDATE no action; 13 | EXCEPTION 14 | WHEN duplicate_object THEN null; 15 | END $$; 16 | -------------------------------------------------------------------------------- /control-plane/drizzle/0032_clumsy_gressill.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS "deployment_provider_config" ( 2 | "provider" varchar(1024) PRIMARY KEY NOT NULL, 3 | "config" json 4 | ); 5 | --> statement-breakpoint 6 | ALTER TABLE "deployments" ADD COLUMN "meta" json; 7 | -------------------------------------------------------------------------------- /control-plane/drizzle/0033_high_komodo.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "clusters" ADD COLUMN "predictive_retries_enabled" boolean DEFAULT false; -------------------------------------------------------------------------------- /control-plane/drizzle/0034_real_big_bertha.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "deployments" ADD COLUMN "provider" text NOT NULL;--> statement-breakpoint 2 | ALTER TABLE "services" ADD COLUMN "deployment_provider" text; -------------------------------------------------------------------------------- /control-plane/drizzle/0035_eminent_fantastic_four.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" ADD COLUMN "predicted_to_be_retryable" boolean;--> statement-breakpoint 2 | ALTER TABLE "jobs" ADD COLUMN "predicted_to_be_retryable_reason" text; -------------------------------------------------------------------------------- /control-plane/drizzle/0036_ambiguous_human_robot.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS "deployment_notifications" ( 2 | "id" varchar(1024) PRIMARY KEY NOT NULL, 3 | "deployment_id" varchar(1024) NOT NULL, 4 | "created_at" timestamp (6) with time zone DEFAULT now() NOT NULL 5 | ); 6 | --> statement-breakpoint 7 | ALTER TABLE "jobs" ADD COLUMN "deployment_id" varchar(1024);--> statement-breakpoint 8 | ALTER TABLE "machines" ADD COLUMN "deployment_id" varchar;--> statement-breakpoint 9 | DO $$ BEGIN 10 | ALTER TABLE "deployment_notifications" ADD CONSTRAINT "deployment_notifications_deployment_id_deployments_id_fk" FOREIGN KEY ("deployment_id") REFERENCES "deployments"("id") ON DELETE no action ON UPDATE no action; 11 | EXCEPTION 12 | WHEN duplicate_object THEN null; 13 | END $$; 14 | -------------------------------------------------------------------------------- /control-plane/drizzle/0037_empty_molecule_man.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "events" ADD COLUMN "deployment_id" varchar(1024);--> statement-breakpoint 2 | DO $$ BEGIN 3 | ALTER TABLE "events" ADD CONSTRAINT "events_deployment_id_deployments_id_fk" FOREIGN KEY ("deployment_id") REFERENCES "deployments"("id") ON DELETE no action ON UPDATE no action; 4 | EXCEPTION 5 | WHEN duplicate_object THEN null; 6 | END $$; 7 | -------------------------------------------------------------------------------- /control-plane/drizzle/0038_woozy_tiger_shark.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" ADD COLUMN IF NOT EXISTS "predictive_retry_count" integer DEFAULT 0; -------------------------------------------------------------------------------- /control-plane/drizzle/0039_puzzling_vampiro.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" ALTER COLUMN "remaining_attempts" DROP DEFAULT;--> statement-breakpoint 2 | ALTER TABLE "jobs" ALTER COLUMN "remaining_attempts" SET NOT NULL;--> statement-breakpoint 3 | ALTER TABLE "clusters" ADD COLUMN "retry_on_stall_enabled" boolean DEFAULT true NOT NULL; -------------------------------------------------------------------------------- /control-plane/drizzle/0040_mean_jazinda.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" ADD COLUMN "executing_machine_id" text;--> statement-breakpoint 2 | ALTER TABLE "machines" ADD COLUMN "status" text DEFAULT 'active';--> statement-breakpoint 3 | ALTER TABLE "jobs" DROP COLUMN IF EXISTS "machine_type"; -------------------------------------------------------------------------------- /control-plane/drizzle/0041_youthful_steve_rogers.sql: -------------------------------------------------------------------------------- 1 | DELETE FROM "events"; 2 | DELETE FROM "machines"; 3 | ALTER TABLE "events" DROP CONSTRAINT IF EXISTS "events_machine_id_machines_id_fk"; 4 | 5 | --> statement-breakpoint 6 | 7 | ALTER TABLE "machines" ADD CONSTRAINT "machines_id_cluster_id_u" UNIQUE("id","cluster_id"); 8 | 9 | --> statement-breakpoint 10 | 11 | ALTER TABLE "machines" DROP CONSTRAINT IF EXISTS "machines_pkey" CASCADE;--> statement-breakpoint 12 | DO $$ BEGIN 13 | ALTER TABLE "events" ADD CONSTRAINT "events_machine_id_cluster_id_machines_id_cluster_id_fk" FOREIGN KEY ("machine_id","cluster_id") REFERENCES "machines"("id","cluster_id") ON DELETE no action ON UPDATE no action; 14 | EXCEPTION 15 | WHEN duplicate_object THEN null; 16 | END $$; 17 | 18 | --> statement-breakpoint 19 | 20 | ALTER TABLE "machines" ADD CONSTRAINT "machines_id_cluster_id" PRIMARY KEY("id","cluster_id"); 21 | 22 | 23 | --> statement-breakpoint 24 | 25 | ALTER TABLE "machines" DROP CONSTRAINT IF EXISTS "machines_id_cluster_id" CASCADE; 26 | -------------------------------------------------------------------------------- /control-plane/drizzle/0042_kind_lester.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS "asset_uploads" ( 2 | "id" varchar(1024) PRIMARY KEY NOT NULL, 3 | "type" text NOT NULL, 4 | "bucket" varchar(1024) NOT NULL, 5 | "key" varchar(1024) NOT NULL, 6 | "created_at" timestamp (6) with time zone DEFAULT now() NOT NULL 7 | ); 8 | --> statement-breakpoint 9 | ALTER TABLE "deployments" ADD COLUMN "asset_upload_id" varchar(1024) NOT NULL;--> statement-breakpoint 10 | DO $$ BEGIN 11 | ALTER TABLE "deployments" ADD CONSTRAINT "deployments_asset_upload_id_asset_uploads_id_fk" FOREIGN KEY ("asset_upload_id") REFERENCES "asset_uploads"("id") ON DELETE no action ON UPDATE no action; 12 | EXCEPTION 13 | WHEN duplicate_object THEN null; 14 | END $$; 15 | --> statement-breakpoint 16 | ALTER TABLE "deployments" DROP COLUMN IF EXISTS "package_upload_path";--> statement-breakpoint 17 | ALTER TABLE "deployments" DROP COLUMN IF EXISTS "definition_upload_url"; -------------------------------------------------------------------------------- /control-plane/drizzle/0043_closed_weapon_omega.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS "client_library_versions" ( 2 | "id" varchar(1024) NOT NULL, 3 | "cluster_id" varchar NOT NULL, 4 | "version" varchar(1024) NOT NULL, 5 | "asset_upload_id" varchar(1024), 6 | CONSTRAINT client_library_versions_cluster_id_id PRIMARY KEY("cluster_id","id") 7 | ); 8 | --> statement-breakpoint 9 | ALTER TABLE "deployments" ALTER COLUMN "asset_upload_id" DROP NOT NULL;--> statement-breakpoint 10 | DO $$ BEGIN 11 | ALTER TABLE "client_library_versions" ADD CONSTRAINT "client_library_versions_cluster_id_clusters_id_fk" FOREIGN KEY ("cluster_id") REFERENCES "clusters"("id") ON DELETE no action ON UPDATE no action; 12 | EXCEPTION 13 | WHEN duplicate_object THEN null; 14 | END $$; 15 | --> statement-breakpoint 16 | DO $$ BEGIN 17 | ALTER TABLE "client_library_versions" ADD CONSTRAINT "client_library_versions_asset_upload_id_asset_uploads_id_fk" FOREIGN KEY ("asset_upload_id") REFERENCES "asset_uploads"("id") ON DELETE no action ON UPDATE no action; 18 | EXCEPTION 19 | WHEN duplicate_object THEN null; 20 | END $$; 21 | -------------------------------------------------------------------------------- /control-plane/drizzle/0044_fancy_landau.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" DROP CONSTRAINT "jobs_owner_hash_target_fn_idempotency_key";--> statement-breakpoint 2 | ALTER TABLE "jobs" DROP COLUMN IF EXISTS "idempotency_key";--> statement-breakpoint 3 | ALTER TABLE "jobs" ADD CONSTRAINT "jobs_owner_hash_target_fn_id" PRIMARY KEY("owner_hash","target_fn","id"); -------------------------------------------------------------------------------- /control-plane/drizzle/0045_icy_kid_colt.sql: -------------------------------------------------------------------------------- 1 | UPDATE "jobs" SET "timeout_interval_seconds" = 3600 WHERE "timeout_interval_seconds" IS NULL; 2 | 3 | ALTER TABLE "jobs" ALTER COLUMN "timeout_interval_seconds" SET DEFAULT 3600;--> statement-breakpoint 4 | ALTER TABLE "jobs" ALTER COLUMN "timeout_interval_seconds" SET NOT NULL; -------------------------------------------------------------------------------- /control-plane/drizzle/0046_needy_brood.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS "predictive_retries_cache" ( 2 | "error_name" varchar(1024) NOT NULL, 3 | "error_message" varchar(1024) NOT NULL, 4 | "retryable" boolean NOT NULL, 5 | "created_at" timestamp (6) with time zone DEFAULT now() NOT NULL, 6 | CONSTRAINT predictive_retries_cache_error_name_error_message PRIMARY KEY("error_name","error_message") 7 | ); 8 | -------------------------------------------------------------------------------- /control-plane/drizzle/0047_certain_penance.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "services" RENAME COLUMN "deployment_provider" TO "preferred_deployment_provider"; -------------------------------------------------------------------------------- /control-plane/drizzle/0048_awesome_joystick.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" ALTER COLUMN "timeout_interval_seconds" SET DEFAULT 300;--> statement-breakpoint 2 | ALTER TABLE "jobs" ADD COLUMN "predictive_retry_enabled" boolean DEFAULT false;--> statement-breakpoint 3 | ALTER TABLE "clusters" DROP COLUMN IF EXISTS "retry_on_stall_enabled"; -------------------------------------------------------------------------------- /control-plane/drizzle/0049_gray_ben_grimm.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "jobs" DROP CONSTRAINT "jobs_owner_hash_target_fn_id";--> statement-breakpoint 2 | CREATE INDEX IF NOT EXISTS "idx1" ON "jobs" ("owner_hash","service","status");--> statement-breakpoint 3 | CREATE INDEX IF NOT EXISTS "idx2" ON "jobs" ("owner_hash","service","target_fn","status");--> statement-breakpoint 4 | ALTER TABLE "jobs" ADD CONSTRAINT "jobs_owner_hash_id" PRIMARY KEY("owner_hash","id"); -------------------------------------------------------------------------------- /control-plane/drizzle/0050_slimy_hulk.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS "cluster_access_points" ( 2 | "cluster_id" varchar NOT NULL, 3 | "name" varchar(1024) NOT NULL, 4 | "allowed_services_csv" text NOT NULL, 5 | "token" varchar(1024) NOT NULL, 6 | "created_at" timestamp (6) with time zone DEFAULT now() NOT NULL, 7 | "updated_at" timestamp (6) with time zone NOT NULL 8 | ); 9 | --> statement-breakpoint 10 | DO $$ BEGIN 11 | ALTER TABLE "cluster_access_points" ADD CONSTRAINT "cluster_access_points_cluster_id_clusters_id_fk" FOREIGN KEY ("cluster_id") REFERENCES "clusters"("id") ON DELETE no action ON UPDATE no action; 12 | EXCEPTION 13 | WHEN duplicate_object THEN null; 14 | END $$; 15 | -------------------------------------------------------------------------------- /control-plane/drizzle/0051_loving_phil_sheldon.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "cluster_access_points" ADD CONSTRAINT "cluster_access_points_cluster_id_name" PRIMARY KEY("cluster_id","name"); -------------------------------------------------------------------------------- /control-plane/drizzle/0052_swift_microbe.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "services" ADD COLUMN "types" text; -------------------------------------------------------------------------------- /control-plane/drizzle/0053_broken_zaran.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE "services" ALTER COLUMN "definition" DROP NOT NULL;--> statement-breakpoint 2 | ALTER TABLE "services" ADD COLUMN "json_schema" json;--> statement-breakpoint 3 | ALTER TABLE "services" DROP COLUMN IF EXISTS "types"; -------------------------------------------------------------------------------- /control-plane/fly.toml: -------------------------------------------------------------------------------- 1 | app = "differential-core" 2 | primary_region = "lax" 3 | 4 | [build] 5 | 6 | [http_service] 7 | internal_port = 4000 8 | force_https = true 9 | auto_stop_machines = true 10 | auto_start_machines = true 11 | min_machines_running = 1 12 | processes = ["app"] 13 | 14 | [http_service.concurrency] 15 | type = "requests" 16 | hard_limit = 500 17 | soft_limit = 400 18 | 19 | [[http_service.checks]] 20 | grace_period = "2s" 21 | interval = "2s" 22 | method = "GET" 23 | timeout = "2s" 24 | path = "/live" 25 | 26 | [metrics] 27 | port = 9091 28 | path = "/metrics" 29 | -------------------------------------------------------------------------------- /control-plane/internal.md: -------------------------------------------------------------------------------- 1 | # differential-core 2 | 3 | ## How to 4 | 5 | 1. Generating migrations 6 | 7 | ```sh 8 | DATABASE_SSL_DISABLED="true" DATABASE_URL="postgres://" npm run migrations 9 | ``` -------------------------------------------------------------------------------- /control-plane/kubernetes.dev.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/control-plane/kubernetes.dev.yaml -------------------------------------------------------------------------------- /control-plane/media/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/control-plane/media/logo.png -------------------------------------------------------------------------------- /control-plane/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@differentialhq/control-plane", 3 | "version": "0.0.16", 4 | "description": "", 5 | "scripts": { 6 | "test": "tsx src/utilities/migrate.ts && jest src --runInBand --setupFiles dotenv/config --forceExit --detectOpenHandles", 7 | "test:dev": "tsx src/utilities/migrate.ts && jest --runInBand --watch src --setupFiles dotenv/config --forceExit --onlyChanged", 8 | "dev": "tsx src/utilities/migrate.ts && nodemon src/index.ts", 9 | "migrations": "npx drizzle-kit generate:pg --schema src/modules/data.ts", 10 | "build": "tsc", 11 | "start": "node dist/utilities/migrate.js && node dist/index.js", 12 | "deploy": "git add . && git commit -m 'update' && git push origin main && fly deploy" 13 | }, 14 | "author": "", 15 | "license": "ISC", 16 | "dependencies": { 17 | "@aws-sdk/client-cloudformation": "^3.529.1", 18 | "@aws-sdk/client-cloudwatch-logs": "^3.540.0", 19 | "@aws-sdk/client-lambda": "^3.504.0", 20 | "@aws-sdk/client-s3": "^3.496.0", 21 | "@aws-sdk/client-sns": "^3.529.1", 22 | "@aws-sdk/s3-request-presigner": "^3.496.0", 23 | "@babel/preset-env": "^7.22.10", 24 | "@babel/preset-typescript": "^7.22.11", 25 | "@fastify/cors": "^8.5.0", 26 | "@langchain/core": "^0.1.61", 27 | "@langchain/openai": "^0.0.28", 28 | "@sentry/node": "^7.93.0", 29 | "@sentry/profiling-node": "^1.3.5", 30 | "@ts-rest/core": "^3.27.0", 31 | "@ts-rest/fastify": "^3.27.0", 32 | "@types/jest": "^29.5.4", 33 | "@types/semver": "^7.5.8", 34 | "axios": "^1.6.2", 35 | "dotenv": "^16.3.1", 36 | "drizzle-kit": "^0.19.12", 37 | "drizzle-orm": "^0.27.2", 38 | "fastify": "^4.21.0", 39 | "jest": "^29.6.4", 40 | "json-schema-to-zod": "^2.1.0", 41 | "jsonwebtoken": "^9.0.2", 42 | "jwks-rsa": "^3.1.0", 43 | "langchain": "^0.1.36", 44 | "msgpackr": "^1.10.1", 45 | "node-cache": "^5.1.2", 46 | "nodemon": "^3.0.1", 47 | "pg": "^8.11.2", 48 | "prom-client": "^15.1.0", 49 | "semver": "^7.6.0", 50 | "sns-validator": "^0.3.5", 51 | "ts-node": "^10.9.1", 52 | "typescript": "^5.1.6", 53 | "ulid": "^2.3.0", 54 | "winston": "^3.12.0", 55 | "zod": "^3.22.4" 56 | }, 57 | "devDependencies": { 58 | "@types/jsonwebtoken": "^9.0.2", 59 | "@types/pg": "^8.10.2", 60 | "@types/sns-validator": "^0.3.3", 61 | "@types/ws": "^8.5.5", 62 | "tsx": "^4.7.0" 63 | }, 64 | "private": true, 65 | "jest": { 66 | "testTimeout": 20000 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /control-plane/postgres.dev.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: postgres-pv-claim 5 | spec: 6 | accessModes: 7 | - ReadWriteOnce 8 | resources: 9 | requests: 10 | storage: 1Gi 11 | 12 | --- 13 | apiVersion: v1 14 | kind: Service 15 | metadata: 16 | name: postgres 17 | spec: 18 | type: ClusterIP 19 | ports: 20 | - port: 5432 21 | selector: 22 | app: postgres 23 | 24 | --- 25 | apiVersion: apps/v1 26 | kind: Deployment 27 | metadata: 28 | name: postgres 29 | spec: 30 | selector: 31 | matchLabels: 32 | app: postgres 33 | template: 34 | metadata: 35 | labels: 36 | app: postgres 37 | spec: 38 | containers: 39 | - name: postgres 40 | image: postgres:latest 41 | env: 42 | - name: POSTGRES_DB 43 | value: postgres 44 | - name: POSTGRES_USER 45 | value: postgres 46 | - name: POSTGRES_PASSWORD 47 | value: postgres 48 | ports: 49 | - containerPort: 5432 50 | volumeMounts: 51 | - mountPath: /var/lib/postgresql/data 52 | name: postgres-storage 53 | volumes: 54 | - name: postgres-storage 55 | persistentVolumeClaim: 56 | claimName: postgres-pv-claim 57 | -------------------------------------------------------------------------------- /control-plane/src/modules/agents/agent.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/control-plane/src/modules/agents/agent.ts -------------------------------------------------------------------------------- /control-plane/src/modules/cluster-activity.ts: -------------------------------------------------------------------------------- 1 | import Cache from "node-cache"; 2 | 3 | const cache = new Cache({ stdTTL: 60, maxKeys: 1000 }); 4 | 5 | export const setClusterActivityToHigh = (clusterId: string) => { 6 | cache.set(clusterId, true); 7 | }; 8 | 9 | export const isClusterActivityHigh = (clusterId: string) => { 10 | return cache.get(clusterId) === true; 11 | }; 12 | -------------------------------------------------------------------------------- /control-plane/src/modules/cluster.ts: -------------------------------------------------------------------------------- 1 | import { eq } from "drizzle-orm"; 2 | import NodeCache from "node-cache"; 3 | import * as data from "./data"; 4 | 5 | const cache = new NodeCache({ stdTTL: 60, checkperiod: 65, maxKeys: 5000 }); 6 | 7 | export type OperationalCluster = { 8 | predictiveRetriesEnabled: boolean | null; 9 | cloudEnabled: boolean | null; 10 | }; 11 | 12 | export const operationalCluster = async ( 13 | clusterId: string, 14 | ): Promise => { 15 | const cached = cache.get(clusterId); 16 | 17 | if (cached) { 18 | return cached; 19 | } 20 | 21 | const results = await data.db 22 | .select({ 23 | predictiveRetriesEnabled: data.clusters.predictive_retries_enabled, 24 | cloudEnabled: data.clusters.cloud_enabled, 25 | }) 26 | .from(data.clusters) 27 | .where(eq(data.clusters.id, clusterId)); 28 | 29 | if (results.length === 0) { 30 | throw new Error(`Cluster not found: ${clusterId}`); 31 | } 32 | 33 | cache.set(clusterId, results[0]); 34 | 35 | return results[0]; 36 | }; 37 | 38 | export const getCluster = async (clusterId: string) => { 39 | const [cluster] = await data.db 40 | .select({ 41 | id: data.clusters.id, 42 | apiSecret: data.clusters.api_secret, 43 | }) 44 | .from(data.clusters) 45 | .where(eq(data.clusters.id, clusterId)); 46 | 47 | if (!cluster) { 48 | throw new Error(`Cluster not found: ${clusterId}`); 49 | } 50 | 51 | return cluster; 52 | }; 53 | -------------------------------------------------------------------------------- /control-plane/src/modules/cron.ts: -------------------------------------------------------------------------------- 1 | // a naive cron implementation which will consume from a CDC later 2 | 3 | const intervals: NodeJS.Timeout[] = []; 4 | 5 | export const registerCron = async ( 6 | fn: () => Promise, 7 | { interval }: { interval: number }, 8 | ) => { 9 | const intervalId = setInterval(fn, interval); 10 | intervals.push(intervalId); 11 | }; 12 | 13 | process.on("beforeExit", () => { 14 | intervals.forEach((intervalId) => { 15 | clearInterval(intervalId); 16 | }); 17 | }); 18 | -------------------------------------------------------------------------------- /control-plane/src/modules/deployment/cfn-manager.test.ts: -------------------------------------------------------------------------------- 1 | import { deploymentResultFromNotification } from "./cfn-manager"; 2 | 3 | describe("deploymentResultFromNotification", () => { 4 | const stackId = "arn:aws:cloudformation:xxxx"; 5 | const clientRequestToken = "xxxx"; 6 | it("throws if status is missing", () => { 7 | expect(() => deploymentResultFromNotification({})).toThrow(); 8 | }); 9 | 10 | it.each(["CREATE_COMPLETE", "UPDATE_COMPLETE"])( 11 | "%s status is treated as a success", 12 | (status) => { 13 | const result = deploymentResultFromNotification({ 14 | ResourceStatus: status, 15 | ResourceStatusReason: "", 16 | ClientRequestToken: clientRequestToken, 17 | StackId: stackId, 18 | }); 19 | expect(result).toMatchObject({ 20 | success: true, 21 | pending: false, 22 | status, 23 | stackId, 24 | clientRequestToken, 25 | }); 26 | }, 27 | ); 28 | 29 | it.each([ 30 | "UPDATE_ROLLBACK_FAILED", 31 | "ROLLBACK_FAILED", 32 | "UPDATE_ROLLBACK_COMPLETE", 33 | "DELETE_COMPLETE", 34 | ])("%s status is treated as a failure", (status) => { 35 | const result = deploymentResultFromNotification({ 36 | ResourceStatus: status, 37 | ResourceStatusReason: "Something went wrong!", 38 | ClientRequestToken: clientRequestToken, 39 | StackId: stackId, 40 | }); 41 | expect(result).toMatchObject({ 42 | success: false, 43 | pending: false, 44 | reason: "Something went wrong!", 45 | status, 46 | stackId, 47 | clientRequestToken, 48 | }); 49 | }); 50 | }); 51 | -------------------------------------------------------------------------------- /control-plane/src/modules/deployment/deployment-provider.ts: -------------------------------------------------------------------------------- 1 | import { ZodSchema } from "zod"; 2 | import { Deployment } from "./deployment"; 3 | import * as data from "../data"; 4 | import { eq } from "drizzle-orm"; 5 | import { MockProvider } from "./mock-deployment-provider"; 6 | import { LambdaCfnProvider } from "./lambda-cfn-provider"; 7 | 8 | const mockProvider = new MockProvider(); 9 | const lambdaProvider = new LambdaCfnProvider(); 10 | 11 | export const getDeploymentProvider = (provider: string): DeploymentProvider => { 12 | switch (provider) { 13 | case "lambda": 14 | return lambdaProvider; 15 | case "mock": 16 | return mockProvider; 17 | default: 18 | throw new Error(`Unknown provider ${provider}`); 19 | } 20 | }; 21 | 22 | export const fetchConfig = async ( 23 | provider: DeploymentProvider, 24 | ): Promise => { 25 | const config = await data.db 26 | .select({ 27 | config: data.deploymentProvividerConfig.config, 28 | }) 29 | .from(data.deploymentProvividerConfig) 30 | .where(eq(data.deploymentProvividerConfig.provider, provider.name())); 31 | 32 | if (config.length === 0) { 33 | throw new Error(`No configuration found for provider ${provider.name()}`); 34 | } 35 | 36 | return provider.schema().parse(config[0].config); 37 | }; 38 | 39 | export interface DeploymentProvider { 40 | name: () => string; 41 | schema: () => ZodSchema; 42 | // Create a new deployment 43 | create: (deployment: Deployment) => Promise; 44 | // Update an existing deployment 45 | update: (deployment: Deployment) => Promise; 46 | // Notify the provider of a a new job 47 | notify: ( 48 | deployment: Deployment, 49 | pendingJobs: number, 50 | runningMachines: number, 51 | ) => Promise; 52 | getLogs: ( 53 | deployment: Deployment, 54 | options: { 55 | start?: Date; 56 | end?: Date; 57 | filter?: string; 58 | next?: string; 59 | }, 60 | ) => Promise<{ message: string }[]>; 61 | // How frequently the provider should be notified of new jobs 62 | minimumNotificationInterval: () => number; 63 | } 64 | -------------------------------------------------------------------------------- /control-plane/src/modules/deployment/mock-deployment-provider.ts: -------------------------------------------------------------------------------- 1 | import { ZodSchema, z } from "zod"; 2 | import { Deployment } from "./deployment"; 3 | import { DeploymentProvider } from "./deployment-provider"; 4 | import { logger } from "../../utilities/logger"; 5 | 6 | export class MockProvider implements DeploymentProvider { 7 | public name(): string { 8 | return "mock"; 9 | } 10 | 11 | public schema(): ZodSchema<{}> { 12 | return z.object({}); 13 | } 14 | 15 | public minimumNotificationInterval(): number { 16 | return 10000; 17 | } 18 | 19 | public async create(deployment: Deployment): Promise { 20 | logger.info("Would create new deployment", { 21 | deployment, 22 | }); 23 | } 24 | 25 | public async update(deployment: Deployment): Promise { 26 | logger.info("Would update existing deployment", { 27 | deployment, 28 | }); 29 | } 30 | 31 | public async notify( 32 | deployment: Deployment, 33 | pendingJobs: number, 34 | runningMachines: number, 35 | ): Promise { 36 | logger.info("Would notify provider of new jobs", { 37 | deployment, 38 | pendingJobs, 39 | runningMachines, 40 | }); 41 | } 42 | 43 | public async getLogs(): Promise<{ message: string }[]> { 44 | return [{ message: "Test log" }]; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /control-plane/src/modules/health.ts: -------------------------------------------------------------------------------- 1 | import { db } from "./data"; 2 | import { sql } from "drizzle-orm"; 3 | 4 | export const isOk = () => 5 | db 6 | .execute(sql`SELECT 1`) 7 | .then(() => true) 8 | .catch(() => false); 9 | -------------------------------------------------------------------------------- /control-plane/src/modules/jobs/job-metrics.ts: -------------------------------------------------------------------------------- 1 | import { Summary } from "prom-client"; 2 | 3 | export const jobDurations = new Summary({ 4 | name: "differential_job_operations_duration_ms", 5 | help: "Duration of job operations in milliseconds", 6 | labelNames: ["operation"], 7 | }); 8 | -------------------------------------------------------------------------------- /control-plane/src/modules/jwt.test.ts: -------------------------------------------------------------------------------- 1 | import * as jwt from "./jwt"; 2 | import { env } from "../utilities/env"; 3 | 4 | jest.mock("../utilities/env"); 5 | 6 | describe("verifyManagementToken", () => { 7 | afterEach(() => { 8 | jest.resetAllMocks(); 9 | }); 10 | 11 | it("should verify a correct token", async () => { 12 | env.JWT_IGNORE_EXPIRATION = true; 13 | 14 | const token = `eyJhbGciOiJSUzI1NiIsImtpZCI6Imluc18yYTF3ZjlWUWhwbFpXdzZkYlJ1M3NSWTN2RkgiLCJ0eXAiOiJKV1QifQ.eyJhenAiOiJodHRwOi8vbG9jYWxob3N0OjMwMDAiLCJleHAiOjE3MDM1Mzk1NDIsImlhdCI6MTcwMzUzOTQ4MiwiaXNzIjoiaHR0cHM6Ly9odW1hbmUtZ3JvdXBlci05OC5jbGVyay5hY2NvdW50cy5kZXYiLCJuYmYiOjE3MDM1Mzk0NzIsInNpZCI6InNlc3NfMmExeEU4OG02bkVSZmlkQnUyZUxDbjF6dkRxIiwic3ViIjoidXNlcl8yYTF4RTVwdlNUUGxsVng2aTh3Q0E3cFZWSEoifQ.RsNtbucx6zpEemvKlgheDJ4n_ygNKpwiBBVsH_Vnb_jV1z4g7XMGYYmuKlMJLqSqNj2jc96J_VyiULcDlCC2ylEFwuIsPSn8qgsF6M-7LIUsZ9IRosT3m3K0s-wWWXBBOKZsxqD38aUh6A8AKwTNxEgs_JnWmFCdzvlftPmRNd0_QFiOiwTX1PZk3XFG2ETM8t-t9Q8AVf0T09kdrZ6CZphV93VDmCSh7zBouMtitO5hU58GERIomEagRRI-NV4310WT8YNt-XXiG8hIcejIw6wRGyYDdHoV66QJLF9cs8HRjxHLHqVqpJu8QL8yRj6wzFVtj22fE_guYSD1S0bMPQ`; 15 | 16 | const result = await jwt.verifyManagementToken({ managementToken: token }); 17 | 18 | expect(result).toEqual({ 19 | userId: "user_2a1xE5pvSTPllVx6i8wCA7pVVHJ", 20 | }); 21 | }); 22 | 23 | it("should throw on a malformed token", async () => { 24 | const token = `1234`; 25 | 26 | await expect( 27 | jwt.verifyManagementToken({ managementToken: token }), 28 | ).rejects.toThrowError("jwt malformed"); 29 | }); 30 | 31 | it("should accept a correct management secret", async () => { 32 | const token = 33 | "sk_management_98a7oysidtghfkjbaslnd;fuays87otdiygfukahjsbdlf;a;soihudyfgajvshkbdlnf;asdfh"; 34 | 35 | env.MANAGEMENT_SECRET = token; 36 | 37 | const result = await jwt.verifyManagementToken({ 38 | managementToken: token, 39 | }); 40 | 41 | expect(result).toEqual({ 42 | userId: "control-plane-administrator", 43 | }); 44 | }); 45 | 46 | it("should throw on an incorrect management secret", async () => { 47 | const token = 48 | "sk_management_98a7oysidtghfkjbaslnd;fuays87otdiygfukahjsbdlf;a;soihudyfgajvshkbdlnf;asdfh"; 49 | 50 | env.MANAGEMENT_SECRET = token; 51 | 52 | await expect( 53 | jwt.verifyManagementToken({ managementToken: "sk_management_1234" }), 54 | ).rejects.toThrowError("Invalid token"); 55 | }); 56 | }); 57 | -------------------------------------------------------------------------------- /control-plane/src/modules/jwt.ts: -------------------------------------------------------------------------------- 1 | import jwt, { GetPublicKeyOrSecret } from "jsonwebtoken"; 2 | import jwksClient from "jwks-rsa"; 3 | import { env } from "../utilities/env"; 4 | 5 | export class AuthenticationError extends Error { 6 | statusCode = 401; 7 | 8 | constructor(message: string) { 9 | super(message); 10 | this.name = "AuthenticationError"; 11 | } 12 | } 13 | 14 | const client = env.JWKS_URL 15 | ? jwksClient({ 16 | jwksUri: env.JWKS_URL, 17 | }) 18 | : null; 19 | 20 | const getKey: GetPublicKeyOrSecret = (header, callback) => { 21 | if (!client) { 22 | return callback( 23 | new Error( 24 | "JWKS client not initialized. Probably missing JWKS_URL in env.", 25 | ), 26 | ); 27 | } 28 | 29 | return client.getSigningKey(header.kid, function (err, key) { 30 | var signingKey = key?.getPublicKey(); 31 | callback(err, signingKey); 32 | }); 33 | }; 34 | 35 | export const CONTROL_PLANE_ADMINISTRATOR = "control-plane-administrator"; 36 | 37 | export const verifyManagementToken = async ({ 38 | managementToken, 39 | }: { 40 | managementToken: string; 41 | }): Promise<{ 42 | userId: string; 43 | }> => { 44 | const managementSecretAuthEnabled = Boolean(env.MANAGEMENT_SECRET); 45 | 46 | if (managementSecretAuthEnabled && managementToken) { 47 | const secretsMatch = managementToken === env.MANAGEMENT_SECRET; 48 | 49 | if (secretsMatch) { 50 | return { 51 | userId: CONTROL_PLANE_ADMINISTRATOR, 52 | }; 53 | } else { 54 | throw new AuthenticationError("Invalid token"); 55 | } 56 | } 57 | 58 | return new Promise((resolve, reject) => { 59 | jwt.verify( 60 | managementToken, 61 | getKey, 62 | { 63 | algorithms: ["RS256"], 64 | ignoreExpiration: env.JWT_IGNORE_EXPIRATION, 65 | }, 66 | function (err, decoded) { 67 | if (err) { 68 | return reject(new AuthenticationError(err.message)); 69 | } 70 | 71 | if (!decoded) { 72 | return reject(new AuthenticationError("No decoded token")); 73 | } 74 | 75 | if (typeof decoded.sub !== "string") { 76 | return reject(new AuthenticationError("No sub in decoded token")); 77 | } 78 | 79 | return resolve({ 80 | userId: decoded.sub, 81 | }); 82 | }, 83 | ); 84 | }); 85 | }; 86 | -------------------------------------------------------------------------------- /control-plane/src/modules/management.test.ts: -------------------------------------------------------------------------------- 1 | import { getClusterSettings, setClusterSettings } from "./management"; 2 | import { createOwner } from "./test/util"; 3 | 4 | describe("management", () => { 5 | describe("cluster settings", () => { 6 | it("should be able to get and set cluster settings", async () => { 7 | const owner = await createOwner(); 8 | 9 | const initial = await getClusterSettings(owner.clusterId); 10 | 11 | expect(initial).toStrictEqual({ 12 | predictiveRetriesEnabled: false, 13 | cloudEnabled: false, 14 | }); 15 | 16 | await setClusterSettings(owner.clusterId, { 17 | predictiveRetriesEnabled: true, 18 | }); 19 | 20 | const updated = await getClusterSettings(owner.clusterId); 21 | 22 | expect(updated).toStrictEqual({ 23 | predictiveRetriesEnabled: true, 24 | cloudEnabled: false, 25 | }); 26 | }); 27 | }); 28 | }); 29 | -------------------------------------------------------------------------------- /control-plane/src/modules/packages/versioning.test.ts: -------------------------------------------------------------------------------- 1 | import { ulid } from "ulid"; 2 | import * as data from "../data"; 3 | import { createOwner } from "../test/util"; 4 | import { 5 | previousVersion, 6 | incrementVersion, 7 | SemVerIncrement, 8 | } from "./versioning"; 9 | 10 | describe("versioning", () => { 11 | let cluster1: string; 12 | 13 | beforeAll(async () => { 14 | cluster1 = (await createOwner()).clusterId; 15 | 16 | const versions = ["1.0.0", "1.0.1", "1.1.0", "2.9.9", "3.0.0"]; 17 | 18 | const upload = await data.db 19 | .insert(data.assetUploads) 20 | .values({ 21 | id: ulid(), 22 | type: "client_library", 23 | bucket: "bucket", 24 | key: "key", 25 | }) 26 | .returning({ 27 | id: data.assetUploads.id, 28 | }); 29 | 30 | for (const version of versions) { 31 | await data.db.insert(data.clientLibraryVersions).values({ 32 | id: ulid(), 33 | cluster_id: cluster1, 34 | asset_upload_id: upload[0].id, 35 | version, 36 | }); 37 | } 38 | }); 39 | 40 | afterEach(() => { 41 | jest.clearAllMocks(); 42 | }); 43 | 44 | it("should return the highest version library", async () => { 45 | const result = await previousVersion({ clusterId: cluster1 }); 46 | expect(result).toBe("3.0.0"); 47 | }); 48 | 49 | it("should return 0.0.0 if no version exist", async () => { 50 | const result = await previousVersion({ clusterId: "nonexistent" }); 51 | expect(result).toBe("0.0.0"); 52 | }); 53 | 54 | it.each([ 55 | ["patch", "3.0.1"], 56 | ["minor", "3.1.0"], 57 | ["major", "4.0.0"], 58 | ])(`should increment version by %s`, async (increment, expected) => { 59 | const result = incrementVersion({ 60 | version: "3.0.0", 61 | increment: increment as SemVerIncrement, 62 | }); 63 | expect(result).toBe(expected); 64 | }); 65 | }); 66 | -------------------------------------------------------------------------------- /control-plane/src/modules/packages/versioning.ts: -------------------------------------------------------------------------------- 1 | import { and, eq, isNotNull } from "drizzle-orm"; 2 | import * as data from "../data"; 3 | 4 | export type SemVerIncrement = "patch" | "minor" | "major"; 5 | 6 | import * as semver from "semver"; 7 | 8 | export const previousVersion = async ({ 9 | clusterId, 10 | }: { 11 | clusterId: string; 12 | }): Promise => { 13 | const previousVersions = await data.db 14 | .select({ 15 | version: data.clientLibraryVersions.version, 16 | }) 17 | .from(data.clientLibraryVersions) 18 | .where( 19 | and( 20 | eq(data.clientLibraryVersions.cluster_id, clusterId), 21 | isNotNull(data.clientLibraryVersions.asset_upload_id), 22 | ), 23 | ); 24 | 25 | if (previousVersions.length === 0) { 26 | return "0.0.0"; 27 | } 28 | 29 | return previousVersions.map((v) => v.version).sort(semver.rcompare)[0]; 30 | }; 31 | 32 | export const incrementVersion = ({ 33 | version, 34 | increment, 35 | }: { 36 | version: string; 37 | increment: SemVerIncrement; 38 | }): string => { 39 | const nextVersion = semver.inc(version, increment); 40 | 41 | if (!nextVersion) { 42 | throw new Error(`Could not increment version: ${version} ${increment}`); 43 | } 44 | 45 | return nextVersion; 46 | }; 47 | -------------------------------------------------------------------------------- /control-plane/src/modules/predictor/client.ts: -------------------------------------------------------------------------------- 1 | import { initClient } from "@ts-rest/core"; 2 | import { contract } from "./contract"; 3 | import { env } from "../../utilities/env"; 4 | import { logger } from "../../utilities/logger"; 5 | 6 | if (!env.PREDICTOR_API_URL) { 7 | logger.warn("PREDICTOR_API_URL is not set"); 8 | } 9 | 10 | export const client = env.PREDICTOR_API_URL 11 | ? initClient(contract, { 12 | baseUrl: env.PREDICTOR_API_URL, 13 | baseHeaders: {}, 14 | }) 15 | : null; 16 | -------------------------------------------------------------------------------- /control-plane/src/modules/predictor/contract.ts: -------------------------------------------------------------------------------- 1 | import { initContract } from "@ts-rest/core"; 2 | import { z } from "zod"; 3 | 4 | const c = initContract(); 5 | 6 | export const contract = c.router({ 7 | predictRetryability: { 8 | method: "POST", 9 | path: "/is-retryable", 10 | headers: z.object({ 11 | authorization: z.string(), 12 | }), 13 | body: z.object({ 14 | errorName: z.string(), 15 | errorMessage: z.string(), 16 | }), 17 | responses: { 18 | 200: z.object({ 19 | retryable: z.boolean(), 20 | }), 21 | }, 22 | }, 23 | patchFunction: { 24 | method: "POST", 25 | path: "/patch-function", 26 | headers: z.object({ 27 | authorization: z.string(), 28 | }), 29 | body: z.object({ 30 | errorName: z.string(), 31 | errorMessage: z.string(), 32 | fn: z.string(), 33 | }), 34 | responses: { 35 | 200: z.object({ 36 | patch: z.string().nullable(), 37 | }), 38 | }, 39 | }, 40 | live: { 41 | method: "GET", 42 | path: "/live", 43 | responses: { 44 | 200: z.object({ 45 | live: z.boolean(), 46 | }), 47 | }, 48 | }, 49 | }); 50 | -------------------------------------------------------------------------------- /control-plane/src/modules/predictor/predictor.test.ts: -------------------------------------------------------------------------------- 1 | import msgpackr from "msgpackr"; 2 | import { isRetryable } from "./predictor"; 3 | 4 | describe("predictor", () => { 5 | it("should cache the prediction", async () => { 6 | const nonce = Math.random().toString(36).substring(7); 7 | 8 | const errorName = `Error${nonce}`; 9 | 10 | const status = await isRetryable( 11 | msgpackr 12 | .pack({ name: errorName, message: "This error is retryable" }) 13 | .toString("base64"), 14 | ); 15 | 16 | expect(status.retryable).toBe(true); 17 | expect(status.cached).toBeFalsy(); 18 | 19 | const status2 = await isRetryable( 20 | msgpackr 21 | .pack({ name: errorName, message: "This error is retryable" }) 22 | .toString("base64"), 23 | ); 24 | 25 | expect(status2.retryable).toBe(true); 26 | expect(status2.cached).toBe(true); 27 | }); 28 | }); 29 | -------------------------------------------------------------------------------- /control-plane/src/modules/s3.ts: -------------------------------------------------------------------------------- 1 | import { 2 | GetObjectCommand, 3 | PutObjectCommand, 4 | S3Client, 5 | } from "@aws-sdk/client-s3"; 6 | import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; 7 | 8 | const s3Client = new S3Client(); 9 | 10 | export const getPresignedURL = async ( 11 | bucket: string, 12 | key: string, 13 | ): Promise => { 14 | const command = new PutObjectCommand({ 15 | Bucket: bucket, 16 | Key: key, 17 | ContentType: "application/zip", 18 | }); 19 | 20 | return await getSignedUrl(s3Client, command, { expiresIn: 3600 }); 21 | }; 22 | 23 | export const getObject = async ({ 24 | bucket, 25 | key, 26 | }: { 27 | bucket: string; 28 | key: string; 29 | }): Promise => { 30 | const command = new GetObjectCommand({ 31 | Bucket: bucket, 32 | Key: key, 33 | }); 34 | 35 | const { Body } = await s3Client.send(command); 36 | if (!Body) { 37 | throw new Error("No body in S3 response"); 38 | } 39 | return Body; 40 | }; 41 | -------------------------------------------------------------------------------- /control-plane/src/modules/sns.test.ts: -------------------------------------------------------------------------------- 1 | import { parseCloudFormationMessage } from "./sns"; 2 | 3 | describe("parseCloudFormationMessages", () => { 4 | const testNotification = ` 5 | 'StackId'='arn:aws:cloudformation:xxxx' 6 | 'ResourceStatus'='UPDATE_FAILED' 7 | 'ResourceStatusReason'='Something went wrong!' 8 | 'ClientRequestToken'='xxxx' 9 | `; 10 | it("parses the stackName", () => { 11 | const result = parseCloudFormationMessage(testNotification); 12 | expect(result.StackId).toBe("arn:aws:cloudformation:xxxx"); 13 | expect(result.ResourceStatus).toBe("UPDATE_FAILED"); 14 | expect(result.ResourceStatusReason).toBe("Something went wrong!"); 15 | expect(result.ClientRequestToken).toBe("xxxx"); 16 | }); 17 | }); 18 | -------------------------------------------------------------------------------- /control-plane/src/modules/sns.ts: -------------------------------------------------------------------------------- 1 | import { ConfirmSubscriptionCommand, SNSClient } from "@aws-sdk/client-sns"; 2 | import MessageValidator from "sns-validator"; 3 | 4 | const validator = new MessageValidator(); 5 | const sns = new SNSClient(); 6 | 7 | // "'stackId'='XXXX'\n"; 8 | export const parseCloudFormationMessage = (notification: string) => 9 | notification 10 | .replace(/"/g, "") 11 | .replace(/'/g, "") 12 | .split("\n") 13 | .reduce( 14 | (acc, line) => { 15 | const [key, value] = line.split("="); 16 | acc[key] = value; 17 | return acc; 18 | }, 19 | {} as Record, 20 | ); 21 | 22 | export const confirmSubscription = async (options: { 23 | Token: string; 24 | TopicArn: string; 25 | }) => { 26 | await sns.send(new ConfirmSubscriptionCommand(options)); 27 | }; 28 | 29 | export const validateSignature = async ( 30 | message: Record, 31 | ): Promise => { 32 | return new Promise((resolve, reject) => 33 | validator.validate(message, (err: any) => { 34 | if (err) { 35 | reject(err); 36 | } else { 37 | resolve(true); 38 | } 39 | }), 40 | ); 41 | }; 42 | -------------------------------------------------------------------------------- /control-plane/src/modules/test/util.ts: -------------------------------------------------------------------------------- 1 | import * as data from "../data"; 2 | 3 | export const createOwner = async (params?: { clusterId?: string }) => { 4 | const clusterId = params?.clusterId || `test-cluster-${Math.random()}`; 5 | 6 | const apiSecret = `test-secret-${Math.random()}`; 7 | 8 | await data.db 9 | .insert(data.clusters) 10 | .values({ 11 | id: clusterId, 12 | api_secret: apiSecret, 13 | }) 14 | .execute(); 15 | 16 | return { clusterId, apiSecret }; 17 | }; 18 | -------------------------------------------------------------------------------- /control-plane/src/modules/util.ts: -------------------------------------------------------------------------------- 1 | import { logger } from "../utilities/logger"; 2 | 3 | export const backgrounded = Promise>( 4 | fn: T, 5 | ): ((...args: Parameters) => void) => { 6 | return (...args) => { 7 | fn(...args).catch((err) => { 8 | logger.error(`Error in backgrounded function`, { 9 | funcion: fn.name, 10 | error: err, 11 | }); 12 | }); 13 | }; 14 | }; 15 | -------------------------------------------------------------------------------- /control-plane/src/utilities/errors.ts: -------------------------------------------------------------------------------- 1 | export class NotFoundError extends Error { 2 | statusCode: number = 404; 3 | 4 | constructor(message: string) { 5 | super(message); 6 | this.name = "NotFoundError"; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /control-plane/src/utilities/invariant.ts: -------------------------------------------------------------------------------- 1 | type WithoutNullOrUndefined = T extends null | undefined ? never : T; 2 | 3 | export const invariant = ( 4 | value: T, 5 | message: string 6 | ): WithoutNullOrUndefined => { 7 | if (value === null || value === undefined) { 8 | throw new Error(message); 9 | } 10 | 11 | return value as WithoutNullOrUndefined; 12 | }; 13 | -------------------------------------------------------------------------------- /control-plane/src/utilities/logger.ts: -------------------------------------------------------------------------------- 1 | import { createLogger, format, transports } from "winston"; 2 | import { AsyncLocalStorage } from "async_hooks"; 3 | import { env } from "../utilities/env"; 4 | 5 | export const logContext = new AsyncLocalStorage(); 6 | 7 | const winston = createLogger({ 8 | level: env.LOG_LEVEL, 9 | format: 10 | env.NODE_ENV === "development" 11 | ? format.combine(format.colorize(), format.simple()) 12 | : format.simple(), 13 | transports: [new transports.Console()], 14 | }); 15 | 16 | type LogMeta = Record; 17 | type LogLevel = "error" | "warn" | "info" | "debug"; 18 | 19 | const log = (level: LogLevel, message: string, meta?: LogMeta) => { 20 | const store = logContext.getStore(); 21 | if (store) { 22 | winston.log(level, message, { 23 | ...meta, 24 | ...store, 25 | }); 26 | } else { 27 | winston.log(level, message, { 28 | ...meta, 29 | }); 30 | } 31 | }; 32 | 33 | export const logger = { 34 | error: (message: string, meta?: LogMeta) => log("error", message, meta), 35 | warn: (message: string, meta?: LogMeta) => log("warn", message, meta), 36 | info: (message: string, meta?: LogMeta) => log("info", message, meta), 37 | debug: (message: string, meta?: LogMeta) => log("debug", message, meta), 38 | }; 39 | -------------------------------------------------------------------------------- /control-plane/src/utilities/migrate.ts: -------------------------------------------------------------------------------- 1 | import "./env"; 2 | import { migrate } from "drizzle-orm/node-postgres/migrator"; 3 | import * as data from "../modules/data"; 4 | import { logger } from "./logger"; 5 | 6 | (async function runMigrations() { 7 | logger.info("Migrating database..."); 8 | 9 | try { 10 | await migrate(data.db, { migrationsFolder: "./drizzle" }); 11 | logger.info("Database migrated successfully"); 12 | } catch (e) { 13 | logger.error("Error migrating database", { 14 | error: e, 15 | }); 16 | process.exit(1); 17 | } 18 | 19 | await data.pool.end(); 20 | })(); 21 | -------------------------------------------------------------------------------- /control-plane/src/utilities/profiling.ts: -------------------------------------------------------------------------------- 1 | import * as Sentry from "@sentry/node"; 2 | import { ProfilingIntegration } from "@sentry/profiling-node"; 3 | import { env } from "./env"; 4 | 5 | if (env.SENTRY_DSN) { 6 | Sentry.init({ 7 | dsn: env.SENTRY_DSN, 8 | integrations: [new ProfilingIntegration()], 9 | tracesSampleRate: 1.0, 10 | profilesSampleRate: 1.0, 11 | }); 12 | } 13 | -------------------------------------------------------------------------------- /control-plane/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2016", 4 | "module": "commonjs", 5 | "outDir": "./dist", 6 | "esModuleInterop": true, 7 | "forceConsistentCasingInFileNames": true, 8 | "strict": true, 9 | "skipLibCheck": true, 10 | "allowJs": true, 11 | }, 12 | "include": [ 13 | "src/**/*" 14 | ] 15 | } -------------------------------------------------------------------------------- /docs/.github/workflows/retype-action.yml: -------------------------------------------------------------------------------- 1 | name: Publish Retype powered website to GitHub Pages 2 | on: 3 | workflow_dispatch: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | publish: 10 | name: Publish to retype branch 11 | 12 | runs-on: ubuntu-latest 13 | 14 | permissions: 15 | contents: write 16 | 17 | steps: 18 | - uses: actions/checkout@v3 19 | 20 | - uses: retypeapp/action-build@latest 21 | 22 | - uses: retypeapp/action-github-pages@latest 23 | with: 24 | update-branch: true 25 | -------------------------------------------------------------------------------- /docs/256.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/docs/256.png -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | docs.differential.dev -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | dev: 2 | retype start -------------------------------------------------------------------------------- /docs/_includes/head.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/advanced/calling-functions/arguments-and-return-values.md: -------------------------------------------------------------------------------- 1 | # Argument and return values 2 | 3 | One benefit of using Differential is that is abstracts away many of the complexities of network and data serialization. However, as with any abstraction - it is optimised for the most common use-cases and leaks some of the underlying complexity in the form of limitations. 4 | 5 | Differential serializes function arguments and return values with the same algorithm, so the limitations are the same for both. 6 | 7 | ## Serializability 8 | 9 | Differential uses Message Pack for serializing function arguments and return values. It serializes the actual values, not the references to the values. Therefore, if your function arguments or return values contain references to objects, they will not be serialized correctly. 10 | 11 | ## Data Type Compatibility 12 | 13 | This is a table of data types and their compatibility with Differential as validated by the test suite: 14 | 15 | | Data Type | Supported | 16 | | ----------- | --------- | 17 | | `undefined` | ✅ | 18 | | `null` | ✅ | 19 | | `boolean` | ✅ | 20 | | `number` | ✅ | 21 | | `string` | ✅ | 22 | | `bigint` | ✅ | 23 | | `object` | ✅ | 24 | | `Array` | ✅ | 25 | | `Buffer` | ✅ | 26 | | `Date` | ✅ | 27 | 28 | ## Advanced Data Types 29 | 30 | It is theoretically possible to pass more complex data types such as `Map`, `Set`, `TypedArray`, `Error`, `RegExp`, `Function`, `Promise`, `Symbol`, `WeakMap`, `WeakSet`, `ArrayBuffer`, `SharedArrayBuffer`, `DataView`, `Int8Array`, `Uint8Array` etc, as long as they are not nested and do not contain references to objects. However, they go through a serialization process that requires extra work to be properly deserialized on the other side. 31 | 32 | For this purpose, they are left out of the initial offering. 33 | 34 | ## Safety 35 | 36 | Differential adds a layer of safety by validating the data types of the function arguments and return values before serializing them. If the data types are not supported, Differential will throw an error `DifferentialError.INVALID_DATA_TYPE`. The peformance overhead of this validation is negligible. However, you can disable this behaviour by setting the `validate` option to `false`. 37 | -------------------------------------------------------------------------------- /docs/advanced/handling-failures/compute-recovery.md: -------------------------------------------------------------------------------- 1 | --- 2 | order: 1980 3 | --- 4 | 5 | # Recovering from machine failures 6 | 7 | In a cloud environment, machines can fail at any time. Differential transparently handles machine failures by periodically sending heartbeats to the control-plane, quickly catching and retrying failed operations on a healthy worker. This means that you don't have to worry about your service being unavailable due to a machine failure. 8 | 9 | If a machine fails to send any heartbeats within an interval (default 90 seconds): 10 | 11 | 1. It is marked as unhealthy, and Differential will not send any new requests to it. 12 | 2. The functions in progress are marked as failed, and Differential will retry them on a healthy worker. 13 | 14 | If the machine comes back online, Differential will mark it as healthy, and start sending new requests to it. However, it will disregard any results from the machine for the functions that were marked as failed. 15 | 16 | ```mermaid 17 | sequenceDiagram 18 | participant ControlPlane as Control Plane 19 | participant UnhealthyMachine as Unhealthy Machine 20 | participant HealthyMachine as Healthy Machine 21 | 22 | Note over UnhealthyMachine,ControlPlane: Periodic Heartbeat Check 23 | UnhealthyMachine->>+ControlPlane: Send Heartbeat (Fail) 24 | ControlPlane->>-UnhealthyMachine: Mark as Unhealthy 25 | Note over UnhealthyMachine: No new requests received 26 | 27 | ControlPlane->>+HealthyMachine: Redirect New Requests 28 | Note over HealthyMachine: Continues receiving tasks 29 | 30 | UnhealthyMachine->>+ControlPlane: Attempt to Reconnect 31 | alt Reconnection Successful 32 | ControlPlane->>UnhealthyMachine: Mark as Healthy 33 | Note over UnhealthyMachine: Can now receive new requests 34 | else Reconnection Failed 35 | ControlPlane->>UnhealthyMachine: Keep as Unhealthy 36 | Note over UnhealthyMachine: No new requests 37 | end 38 | 39 | Note over UnhealthyMachine, HealthyMachine: Handling Failed Tasks 40 | ControlPlane->>HealthyMachine: Retry Failed Tasks from Unhealthy Machine 41 | Note over HealthyMachine: Executes retried tasks 42 | ``` 43 | 44 | However, it's possible that the particular workload that you're executing on the machine is what makes it crash. To account for this, there's a retry limit for any function call that results in a machine stall (default 1 time). If the function fails more than the retry limit, Differential will mark the function as permanently failed. 45 | -------------------------------------------------------------------------------- /docs/advanced/handling-failures/how-functions-fail.md: -------------------------------------------------------------------------------- 1 | --- 2 | order: 2000 3 | --- 4 | 5 | # How functions fail 6 | 7 | Functions in Differential can "fail" in 2 main ways. 8 | 9 | ## 1. Promise rejection 10 | 11 | A function can fail via a promise rejection due to an explicity `throw` statement in the function code or due to an unhandled exception. 12 | 13 | Differential does not automatically retry failed function calls - i.e., if a function call fails, it fails. It is up to the client to decide if it wants to retry the function call. This is the scenarios denoted by 1 and 2 above. The exception to this is [predictive retries](../guides/predictive-retries.md), which is an opt-in feature to retry function calls based on the error metadata. 14 | 15 | ## 2. Stalling 16 | 17 | A function can stall if it does not complete within the alotted time. This usually happens if: 18 | 19 | 1. Function fails to complete within the alotted `timeoutSeconds` time. 20 | 2. Worker stalls due to a machine crash or network problems. (See [Recovering from machine failures](./compute-recovery.md) for more information) 21 | 22 | Differential does automatically retry function calls that stall. This can be customized by setting the `retryCountOnStall` key in the call configuration object. 23 | -------------------------------------------------------------------------------- /docs/advanced/how-things-work/architecture.md: -------------------------------------------------------------------------------- 1 | # Architecture 2 | 3 | !!! 4 | Differential is currently in technical preview. We've seen promising early results, but our architecture is still evolving. We welcome your thoughts on our design choices. Join the discussion on our [GitHub](https://github.com/differentialhq/differential). 5 | !!! 6 | 7 | Differential’s architecture is intentionally straightforward. We adhere to the principle that simplicity leads to easier scalability. 8 | 9 | ![Architecture](image.png) 10 | 11 | 1. A stateful control-plane is backed by postgres and served via a REST API. 12 | 2. Services in the user land uses the Differential SDK (`new Differential()`) to register services and ask/poll for incoming function calls. 13 | 3. Clients in the user land uses the Differential SDK (`new Differential()`) to queue function calls and poll for completion. 14 | 4. All "work" is logically grouped into clusters (`$ differential clusters create`). 15 | 5. Console and CLI uses the REST API to manage clusters. 16 | -------------------------------------------------------------------------------- /docs/advanced/how-things-work/concepts.md: -------------------------------------------------------------------------------- 1 | --- 2 | order: 1400 3 | --- 4 | 5 | # Concepts 6 | 7 | ## Control Plane 8 | 9 | A central book-keeping stateful service that keeps track of all the services that are running, and their health. It acts as a service registry, and a service mesh. Backed by a database, and deployed with multiple replicas. The managed control plane that Differential runs can be accessed at https://console.differential.dev. 10 | 11 | ## Cluster 12 | 13 | A collection of services and workers. All services and workers connect to the control plane via a cluster. 14 | 15 | ```bash 16 | $ differential clusters create 17 | Cluster created successfully 18 | { 19 | id: 'cluster-aged-paper-cb2142f716', 20 | apiSecret: 'sk_my_super_secret' 21 | } 22 | ``` 23 | 24 | ## Service 25 | 26 | A collection of functions that can be called by a consumer using the differential client. Defined using the SDK. 27 | 28 | ```typescript 29 | const d = new Differential("sk_my_super_secret"); 30 | 31 | export const myService = d.service({ 32 | name: "myService", 33 | functions: async function echo(value: string) { 34 | return value; 35 | }, 36 | }); 37 | ``` 38 | 39 | ## Machine 40 | 41 | A computer that runs one or more services. For a machine to identify itself with the control plane, at least one service must be started. 42 | 43 | ```typescript 44 | await myService.start(); 45 | ``` 46 | 47 | ## Client 48 | 49 | A type-safe entrypoint to your services. 50 | 51 | ```typescript 52 | const d = new Differential("sk_my_super_secret"); 53 | 54 | const client = d.client("myService"); 55 | 56 | await d.echo("hello"); // hello 57 | ``` 58 | -------------------------------------------------------------------------------- /docs/advanced/how-things-work/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/docs/advanced/how-things-work/image.png -------------------------------------------------------------------------------- /docs/advanced/how-things-work/limitations.md: -------------------------------------------------------------------------------- 1 | # Limitations 2 | 3 | ## Programming language 4 | 5 | Differential only supports TypeScript until it gets to a stable release. We do plan to support other languages in the future, however we are intensely focused on creating the right abstractions for a single language first. 6 | 7 | There's nothing preventing your from using Differential with JavaScript, however you will lose out on the type safety that Differential provides for your clients. 8 | -------------------------------------------------------------------------------- /docs/advanced/how-things-work/service-discovery.md: -------------------------------------------------------------------------------- 1 | # Service Discovery 2 | 3 | Status: **General Availability** 4 | 5 | Differential comes with a built-in service registry, so you can call your services by name, without having to worry about IP addresses, ports, or where it's deployed. Your services phone-home to Differential control-plane. 6 | 7 | Your services (whether running in your own compute or Differential cloud) always poll the control-plane to: 8 | 9 | 1. Advertise their presence 10 | 2. Advertise their health 11 | 3. Ask for work 12 | 13 | This allows you to deploy your services anywhere, even across multiple cloud providers and regions, and still have them communicate with each other without any networking configuration. 14 | 15 | It also cuts down the need to secure your services with service to service authentication, as the machines are not directly communicating with each other, or accepting incoming connections. This improves the security posture of your services. 16 | -------------------------------------------------------------------------------- /docs/advanced/index.yml: -------------------------------------------------------------------------------- 1 | icon: plus 2 | expanded: false 3 | order: 980 4 | -------------------------------------------------------------------------------- /docs/assets/image-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/docs/assets/image-1.png -------------------------------------------------------------------------------- /docs/deployments/index.yml: -------------------------------------------------------------------------------- 1 | icon: cloud 2 | expanded: false 3 | order: 980 4 | -------------------------------------------------------------------------------- /docs/getting-started/index.yml: -------------------------------------------------------------------------------- 1 | order: 1000 2 | icon: rocket 3 | expanded: true 4 | -------------------------------------------------------------------------------- /docs/guides/deduplicating.md: -------------------------------------------------------------------------------- 1 | # How to deduplicate function calls 2 | 3 | Consider a scenario where two or more function calls are issued for the same operation. Your code may be racing to complete the operation multiple times, or it may be the network which is repeatedly timing out and causing the same operation to be retried. In such cases, you may want to deduplicate the function calls to ensure that the operation is only executed once. 4 | 5 | ## Execution ID 6 | 7 | Execution ID is a cluster-wide unique identifier that is generated and set for each function call. 8 | 9 | Usually, the control-plane takes care of generating and setting the execution ID. However, you can also set the execution ID manually to deduplicate function calls. 10 | 11 | ## What happens if you call a function with the same Execution ID? 12 | 13 | When two or more function calls happen with the same Execution ID, the subsequent function calls will get the same result as the original function call. 14 | 15 | If the first function call is still in progress, the subsequent function calls will wait for the original function call to complete and return the same result. 16 | 17 | If the first function call has already completed, the subsequent function calls will return the same result as the original function call. 18 | 19 | ## How to deduplicate function calls 20 | 21 | De-duplicating the function call can be done by setting the Execution ID for the function call. Here is an example of how you can set the Execution ID for a function call: 22 | 23 | ```typescript 24 | const result = await client.myFunction("arg1", "arg2", { 25 | $d: { 26 | executionId: "myExecutionId", 27 | }, 28 | }); 29 | 30 | // same function, again 31 | const result2 = await client.myFunction("arg1", "arg2", { 32 | $d: { 33 | executionId: "myExecutionId", 34 | }, 35 | }); 36 | 37 | // another function in a different service 38 | const result3 = await otherClient.myFunction("arg1", "arg2", { 39 | $d: { 40 | executionId: "myExecutionId", 41 | }, 42 | }); 43 | 44 | assert.deepEqual(result, result3); // result and result3 will be the same because Exeuction ID needs to be unique cluster wide 45 | ``` 46 | -------------------------------------------------------------------------------- /docs/guides/end-to-end-encryption.md: -------------------------------------------------------------------------------- 1 | # How to end-to-end encrypt function arguments and return values 2 | 3 | You might wish to encrypt all function arguments and return values, so that the control plane cannot see them. This is possible with Differential, but it requires you to configure your own encryption keys. 4 | 5 | These encryption keys are used to encrypt and decrypt the function arguments and return values. The control plane does not have access to these keys. 6 | 7 | The Typescript example below shows how to configure your own encryption keys. 8 | 9 | ```typescript 10 | const d = new Differential("API_SECRET", { 11 | encryptionKeys: [ 12 | Buffer.from("abcdefghijklmnopqrstuvwxzy123456"), // 32 bytes 13 | ], 14 | }); 15 | ``` 16 | 17 | It accepts an array of encryption keys. This is useful if you want to rotate your encryption keys. Differential will try to decrypt the function arguments and return values with each encryption key until it finds one that works. 18 | 19 | Since this essentially makes function arguments opaque to the control plane, it is important to note that Differential will not be able to provide any of its usual features for these encrypted function arguments, such as predictive retries or predictive alerting. 20 | 21 | Caching is still supported, as it is based on the function arguments and return values as supplied by the client. 22 | -------------------------------------------------------------------------------- /docs/guides/idempotency.md: -------------------------------------------------------------------------------- 1 | # How to ensure idempotency in your functions 2 | 3 | Idempotency is a property of an operation that means the operation can be applied multiple times without changing the result beyond the initial application. In other words, making the same request multiple times has the same effect as making the request once. 4 | 5 | In other words, you'd have to ensure: 6 | 7 | 1. The intended operation is applied at least once. 8 | 2. The intended operation is not applied multiple times. 9 | 10 | ## 1. Making sure the intended operation is applied at least once 11 | 12 | Checking whether the operation is applied successfully is a concern of the developer - in the context of Differential. You can use the [retry policy](https://docs.differential.dev/advanced/calling-functions/customizing-function-calls/#retries) to make sure your functions can recover from machines stalling. 13 | 14 | ```typescript 15 | // service.ts 16 | async function chargeOrder(orderId: string, paymentMethod: PaymantMethod) { 17 | // Check if the order has already been charged 18 | const charge = await getChargeForOrder(orderId); 19 | 20 | if (charge) { 21 | return charge; 22 | } 23 | 24 | // Charge the order 25 | const charge = await chargeOrderWithPaymentMethod(orderId, paymentMethod); 26 | 27 | return charge; 28 | } 29 | 30 | export const orderService = d.service({ 31 | name: "order", 32 | functions: { 33 | chargeOrder, 34 | }, 35 | }); 36 | 37 | // client.ts 38 | function chargeOrder() { 39 | await orderService.chargeOrder(orderId, paymentMethod, { 40 | $d: { 41 | retryCountOnStall: 1, // function will be tried on stalling 42 | }, 43 | }); 44 | } 45 | ``` 46 | 47 | ## Preventing duplicate calls 48 | 49 | At the same time, it's important to make sure that additional clients do not apply the same operation. 50 | 51 | For this purpose, you can use [`$d.executionId`](https://docs.differential.dev/advanced/calling-functions/customizing-function-calls/#execution-id) in the call config that will prevent additional callers from triggering multiple calls. 52 | 53 | Therefore, the function will look like: 54 | 55 | ```typescript 56 | function chargeOrder() { 57 | await orderService.chargeOrder(orderId, paymentMethod, { 58 | $d: { 59 | retryCountOnStall: 1, // function will be tried on stalling 60 | executionId: orderId, // additional clients will not execute `chargeOrder` again for the same key 61 | }, 62 | }); 63 | } 64 | ``` 65 | -------------------------------------------------------------------------------- /docs/guides/index.yml: -------------------------------------------------------------------------------- 1 | order: 990 2 | icon: book 3 | expanded: false 4 | -------------------------------------------------------------------------------- /docs/guides/predictive-alerting.md: -------------------------------------------------------------------------------- 1 | # Predictive Alerting 2 | 3 | Status: **In Development** 4 | 5 | Differential can predict if a function is failing due to an unrecoverable error that requires a code change, and alert you with reproduction steps. 6 | 7 | The control-plane has all the required context on a particular failure, when it happens. It knows the function, some metadata about the source, payload, and the error message. It can use this information to predict if the error is unrecoverable or not. 8 | 9 | If it's predicted to be unrecoverable, Differential will alert you prompting to make the necessary code change. This is especially useful for errors that are hard to reproduce, and require a code change to fix. 10 | 11 | For example, let's consider this that occurs due to a data inconsistency: 12 | 13 | ```typescript 14 | // service.ts 15 | const getCustomerName = async (customerId: string) => { 16 | const customer = await getCustomerFromDatabase(customerId); 17 | return customer.name; 18 | }; 19 | 20 | const customerService = d.service({ 21 | name: "customer", 22 | functions: { 23 | getCustomerName, 24 | }, 25 | }); 26 | 27 | // consumer.ts 28 | const client = d.client("customer"); 29 | 30 | await client.getCustomerName("invalid_id"); // TypeError: Cannot read property 'name' of undefined 31 | ``` 32 | 33 | In this case, Differential can predict that the error is unrecoverable, and prompt you to make a code change. It will provide you with the exact payload that caused the error, and the reproduction steps to reproduce the error locally. 34 | -------------------------------------------------------------------------------- /docs/guides/recovering-from-infrastructure-stalls.md: -------------------------------------------------------------------------------- 1 | # Retrying function calls on infrastructure failures 2 | 3 | Sometimes the functions can stall due to a machine crashing, network issues, or other infrastructure failures. In these cases, you can instruct Differential to retry the function call without writing any custom error recovery logic. 4 | 5 | ## Retry Configuration 6 | 7 | You can configure the number of times a function call should be retried in case of a stall. By default, Differential retries the function call once. You can customize this behavior by setting the `retryCountOnStall` key in the configuration object. 8 | 9 | Imagine you have a function that takes a long time to execute: 10 | 11 | ```typescript 12 | async function myFunction(arg1: string, arg2: string) { 13 | // wait for 1 minute 14 | await new Promise(() => {}, 5 * 60 * 1000); 15 | 16 | return "done"; 17 | } 18 | ``` 19 | 20 | You can configure the number of times the function should be retried in case of a stall: 21 | 22 | ```typescript 23 | const result = await client.myFunction("arg1", "arg2", { 24 | $d: { 25 | retryCountOnStall: 3, // function might be retried 3 times on stalling 26 | timeoutSeconds: 10, // function will timeout after 10 seconds 27 | }, 28 | }); 29 | 30 | const result = await client.myFunction("arg1", "arg2", { 31 | $d: { 32 | retryCountOnStall: 0, // function will not be retried on stalling 33 | timeoutSeconds: 10, // function will timeout after 10 seconds 34 | }, 35 | }); 36 | ``` 37 | 38 | ## Default Behavior 39 | 40 | The default behavior is to retry the function call once, after the function times out. The default timeout is 5 minutes. 41 | 42 | For example, the following function will be retried once after 5 minutes. 43 | 44 | ## Retries vs. Failures 45 | 46 | Note that this retry mechanism is different from the retries that are done when a function call is rejected. When a function call is explicitly rejected becasue of a `throw`, `reject` or an unhandled exception, it is up to the client to retry the function call. 47 | -------------------------------------------------------------------------------- /docs/guides/shared-client-libraries.md: -------------------------------------------------------------------------------- 1 | # Building Shared Client Libraries for Services 2 | 3 | Status: **Private Beta** 4 | 5 | Differential supports publishing a client library for your cluster which can be included in a separate project using `npm`. Client libraries are distributed privately via a `node` package registry managed by Differential. 6 | 7 | The [Differential CLI](https://github.com/differentialhq/differential/tree/main/cli) is used to manage client libraries, you can publish a new client library with the `client publish` command. 8 | 9 | > Differential uses [semantic versioning](https://semver.org), as part of publishing the client library you will be asked to describe the change increment (`patch`, `minor`, `majod`). 10 | 11 | ```sh 12 | differential auth login 13 | differential client publish 14 | ``` 15 | 16 | Once published, the client can be installed in other projects as you would any other `node` package. 17 | 18 | > The `differential auth login` CLI command configures the host's `~/.npmrc` configuration to authenticate with the Differential package registry. If you need to install the package from another host, run `npm auth login`. 19 | 20 | ```sh 21 | npm i @differential.dev/ 22 | ``` 23 | 24 | This will provide a type safe client which can be used to call any service in the cluster. 25 | 26 | ```typescript 27 | import { d } from "../d"; 28 | import type { helloService } from "@differential.dev/"; 29 | 30 | const client = d.client("hello"); 31 | .... 32 | ``` 33 | 34 | Shared client libraries are currently in private beta and will be available soon. To gain early access, please sign up for the waitlist [here](https://forms.fillout.com/t/9M1VhL8Wxyus). 35 | -------------------------------------------------------------------------------- /docs/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/docs/image.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | order: 1500 3 | icon: home 4 | expanded: true 5 | --- 6 | 7 | # Differential 8 | 9 | > Build, Deploy and Refactor fault-tolerant back-end apps without the ceremony. 10 | 11 | !!! 12 | Differential is in the technical preview stage, and is open-source. We are working hard to make our offering generally available. Sign up for the waitlist [here](https://forms.fillout.com/t/9M1VhL8Wxyus). 13 | !!! 14 | 15 | Differential is an open-source application code aware service mesh (control-plane) and a set of adapters (client libraries) which connects your services together with first-class support for Typescript. 16 | ‍ 17 | Differential is a framework that builds on the concepts developers are already familiar with. 18 | 19 | Services are collections of plain old javascript functions which can be deployed in almost any compute. Services ship their own type-safe Typescript clients. 20 | 21 | The control plane takes care of routing data between the functions, and recovering from transient failures, transparently. 22 | 23 | ## Open Source and Self-Hostable 24 | 25 | Differential is released under the Apache 2.0 license, and can be self-hosted. This means you can run your own control-plane, and have full control over your data and infrastructure. Optionally, you can use Differential Cloud, which is a fully managed offering that we are working hard to make generally available. Sign up for the waitlist [here](https://forms.fillout.com/t/9M1VhL8Wxyus). 26 | 27 | ## Quick Start 28 | 29 | Follow the [Quick Start](./getting-started/quick-start.md) guide to get up and running with Differential in under 2 minutes. 30 | 31 | ## See Also: 32 | 33 | - [Get up and running with Differential in under 2 minutes](https://docs.differential.dev/getting-started/quick-start/) 34 | - [Thinking in Differential](https://docs.differential.dev/getting-started/thinking/) 35 | -------------------------------------------------------------------------------- /docs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@differentialhq/docs", 3 | "version": "0.0.36", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "exit 0", 8 | "build": "exit 0" 9 | }, 10 | "dependencies": { 11 | "@differentialhq/core": "3.1.4" 12 | }, 13 | "author": "", 14 | "license": "ISC", 15 | "private": true 16 | } -------------------------------------------------------------------------------- /docs/retype.yml: -------------------------------------------------------------------------------- 1 | input: . 2 | output: .retype 3 | url: https://docs.differential.dev 4 | branding: 5 | title: Differential Docs 6 | logo: /256.png 7 | links: 8 | - text: Home 9 | link: https://www.differential.dev 10 | - text: Differential Cloud 11 | link: https://forms.fillout.com/t/9M1VhL8Wxyus 12 | footer: 13 | copyright: "© Copyright {{ year }}. All rights reserved." 14 | -------------------------------------------------------------------------------- /infrastructure/README.md: -------------------------------------------------------------------------------- 1 | # Differential Infrastructure 2 | 3 | This directory provides infrastructure as code definitions for additional infrastructure resources used by Differential. 4 | 5 | ## AWS CloudFormation 6 | 7 | The `./cfn.yaml` file describes an [AWS CloudFormation](https://aws.amazon.com/cloudformation/) stack providing resources for: 8 | 9 | - Client library registry 10 | - Lambda deployments 11 | 12 | ### Creating stack initially 13 | 14 | The stack can be provisioned by running the following command in your shell: 15 | 16 | ```sh 17 | aws cloudformation create-stack \ 18 | --stack-name $STACK_NAME \ 19 | --template-body file://cfn.yaml \ 20 | --parameters \ 21 | ParameterKey=AssetUploadBucketName,ParameterValue=${ASSET_BUCKET_NAME} \ 22 | ParameterKey=CfnTemplateBucketName,ParameterValue=${CFN_BUCKET_NAME} \ 23 | ParameterKey=CfnSNSWebhook,ParameterValue=${CFN_WEBHOOK} \ 24 | --capabilities CAPABILITY_NAMED_IAM CAPABILITY_IAM 25 | ``` 26 | 27 | ### Updating the stack 28 | 29 | Over time, new resources may be requried, changes can be applied to the stack by running the following command in your shell: 30 | 31 | ```sh 32 | aws cloudformation update-stack \ 33 | --stack-name $STACK_NAME \ 34 | --template-body file://cfn.yaml \ 35 | --parameters ParameterKey=AssetUploadBucketName,ParameterValue=${ASSET_BUCKET_NAME} \ 36 | ParameterKey=CfnTemplateBucketName,ParameterValue=${CFN_BUCKET_NAME} \ 37 | ParameterKey=CloudFormationWebhook,ParameterValue=${CFN_WEBHOOK} \ 38 | --capabilities CAPABILITY_NAMED_IAM CAPABILITY_IAM 39 | ``` 40 | -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "node_modules/lerna/schemas/lerna-schema.json", 3 | "version": "independent" 4 | } 5 | -------------------------------------------------------------------------------- /load-tester/README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 | # Load Tester 6 | 7 | This is a simple load testing apparatus for testing api.differential.dev. 8 | 9 | ## Usage 10 | 11 | Check the [basic example](./src/commands/basic.js) for a simple example of how to write new tests. 12 | 13 | ## Running 14 | 15 | ```sh 16 | npm run load-test 17 | ``` 18 | 19 | The results will be appended to results.jsonl file in the root of the directory. -------------------------------------------------------------------------------- /load-tester/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@differentialhq/load-tester", 3 | "version": "0.0.1", 4 | "description": "Load testing tool for api.differential.dev", 5 | "bin": "setup.sh", 6 | "type": "module", 7 | "scripts": { 8 | "test": "exit 0", 9 | "load-test": "sh run.sh", 10 | "service": "tsx src/services/$npm_config_name.ts --start", 11 | "command": "tsx src/commands/$npm_config_name.ts --start" 12 | }, 13 | "repository": { 14 | "type": "git", 15 | "url": "git+ssh://git@github.com/differentialhq/differential.git" 16 | }, 17 | "author": "differential.dev", 18 | "license": "MIT", 19 | "bugs": { 20 | "url": "https://github.com/differentialhq/differential/issues" 21 | }, 22 | "homepage": "https://github.com/differentialhq/differential#readme", 23 | "dependencies": { 24 | "@differentialhq/core": "^3.12.2", 25 | "tsx": "^4.6.2", 26 | "typescript": "^5.3.3" 27 | }, 28 | "devDependencies": { 29 | "@types/node": "^20.10.4" 30 | } 31 | } -------------------------------------------------------------------------------- /load-tester/run.sh: -------------------------------------------------------------------------------- 1 | npm run service --name=executor& 2 | PID=$! 3 | 4 | # Basic 5 | npm run command --name=basic 6 | npm run command --name=parallel-100 7 | npm run command --name=parallel-1000 8 | # fails. needs connection reuse. 9 | # npm run command --name=parallel-10000 10 | 11 | kill $PID 12 | -------------------------------------------------------------------------------- /load-tester/scripts/setup.sh: -------------------------------------------------------------------------------- 1 | echo "Setting up the project..." 2 | 3 | echo "Installing dependencies..." 4 | npm install 5 | 6 | echo "Obtaining API secret..." 7 | API_SECRET=$(curl https://api.differential.dev/demo/token) 8 | 9 | # replace the API_SECRET with the string REPLACE_ME in src/d.ts file. 10 | sed -i '' "s/REPLACE_ME/$API_SECRET/g" src/d.ts 11 | 12 | echo "Setup complete! 🎉" 13 | 14 | echo "" 15 | echo "To play with the demo, run the following commands in separate terminals:" 16 | echo " 👉 Running the service:" 17 | echo " npm run service --name=hello" 18 | echo " 👉 Running the consumer:" 19 | echo " npm run command --name=greet" 20 | echo "" -------------------------------------------------------------------------------- /load-tester/src/commands/basic.ts: -------------------------------------------------------------------------------- 1 | import { assert } from "console"; 2 | import { d } from "../d"; 3 | import type { executorService } from "../services/executor"; 4 | import { t } from "../t"; 5 | 6 | const executorClient = d.client("executor"); 7 | 8 | t(async function basic() { 9 | const input = Math.random().toString(); 10 | 11 | const expectations = { 12 | object: { type: "object", value: input }, 13 | string: input, 14 | array: [input], 15 | number: 1, 16 | boolean: true, 17 | null: null, 18 | undefined: undefined, 19 | }; 20 | 21 | const result = await Promise.all( 22 | Object.keys(expectations).map((outputType) => 23 | executorClient.exec({ 24 | wait: 0, 25 | error: false, 26 | input, 27 | outputType: outputType as any, 28 | kill: false, 29 | }) 30 | ) 31 | ); 32 | 33 | assert( 34 | JSON.stringify(result) === JSON.stringify(Object.values(expectations)) 35 | ); 36 | }); 37 | -------------------------------------------------------------------------------- /load-tester/src/commands/parallel-100.ts: -------------------------------------------------------------------------------- 1 | import { d } from "../d"; 2 | import type { executorService } from "../services/executor"; 3 | import { t } from "../t"; 4 | import { parallel } from "./parallel"; 5 | 6 | export const executorClient = d.client("executor"); 7 | 8 | t(() => parallel(100)); 9 | -------------------------------------------------------------------------------- /load-tester/src/commands/parallel-1000.ts: -------------------------------------------------------------------------------- 1 | import { t } from "../t"; 2 | import { parallel } from "./parallel"; 3 | 4 | t(() => parallel(1000)); 5 | -------------------------------------------------------------------------------- /load-tester/src/commands/parallel-10000.ts: -------------------------------------------------------------------------------- 1 | import { t } from "../t"; 2 | import { parallel } from "./parallel"; 3 | 4 | t(() => parallel(10000)); 5 | -------------------------------------------------------------------------------- /load-tester/src/commands/parallel.ts: -------------------------------------------------------------------------------- 1 | import { assert } from "console"; 2 | import { executorClient } from "./parallel-100"; 3 | 4 | export async function parallel(n: number) { 5 | const inputs = new Array(n).fill(null).map(() => Math.random().toString()); 6 | 7 | const promises = inputs.map((input) => { 8 | return executorClient.exec({ 9 | wait: 0, 10 | error: false, 11 | input, 12 | outputType: "string", 13 | kill: false, 14 | }); 15 | }); 16 | 17 | const outputs = await Promise.all(promises); 18 | 19 | outputs.forEach((output, i) => { 20 | assert(output === inputs[i]); 21 | }); 22 | } 23 | -------------------------------------------------------------------------------- /load-tester/src/d.ts: -------------------------------------------------------------------------------- 1 | import { Differential } from "@differentialhq/core"; 2 | 3 | if (!process.env.LOAD_TESTER_API_SECRET) { 4 | throw new Error("LOAD_TESTER_API_SECRET not set"); 5 | } 6 | 7 | export const d = new Differential(process.env.LOAD_TESTER_API_SECRET!); 8 | -------------------------------------------------------------------------------- /load-tester/src/services/executor.ts: -------------------------------------------------------------------------------- 1 | import { d } from "../d"; 2 | import process from "process"; 3 | 4 | async function exec(execArgs: { 5 | wait: number; 6 | error: boolean; 7 | input: string; 8 | outputType: 9 | | "object" 10 | | "string" 11 | | "array" 12 | | "number" 13 | | "boolean" 14 | | "null" 15 | | "undefined"; 16 | kill: boolean; 17 | }) { 18 | if (execArgs.wait) { 19 | await new Promise((resolve) => setTimeout(resolve, execArgs.wait)); 20 | } 21 | 22 | if (execArgs.error) { 23 | throw new Error("error"); 24 | } 25 | 26 | if (execArgs.kill) { 27 | process.kill(process.pid, "SIGTERM"); 28 | } 29 | 30 | switch (execArgs.outputType) { 31 | case "object": 32 | return { type: "object", value: execArgs.input }; 33 | case "string": 34 | return execArgs.input; 35 | case "array": 36 | return [execArgs.input]; 37 | case "number": 38 | return 1; 39 | case "boolean": 40 | return true; 41 | case "null": 42 | return null; 43 | case "undefined": 44 | return undefined; 45 | } 46 | } 47 | 48 | export const executorService = d.service({ 49 | name: "executor", 50 | functions: { 51 | exec, 52 | }, 53 | }); 54 | 55 | executorService.start().then(() => { 56 | console.log("executor started"); 57 | }); 58 | -------------------------------------------------------------------------------- /load-tester/src/t.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | 4 | export const t = async (fn: () => Promise) => { 5 | const start = Date.now(); 6 | 7 | await fn(); 8 | 9 | const end = Date.now(); 10 | 11 | const duration = end - start; 12 | const name = fn.name; 13 | const date = new Date().toISOString(); 14 | 15 | const data = { name, duration, date }; 16 | 17 | console.log(`Test ${name} took ${duration}ms`); 18 | 19 | fs.appendFileSync("results.jsonl", JSON.stringify(data) + "\n"); 20 | }; 21 | -------------------------------------------------------------------------------- /load-tester/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2016", 4 | "module": "commonjs", 5 | "esModuleInterop": true, 6 | "forceConsistentCasingInFileNames": true, 7 | "strict": true, 8 | "skipLibCheck": true, 9 | "types": [ 10 | "node" 11 | ] 12 | } 13 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "root", 3 | "private": true, 4 | "workspaces": [ 5 | "sdk", 6 | "control-plane", 7 | "docs", 8 | "admin", 9 | "cli" 10 | ], 11 | "devDependencies": { 12 | "conventional-changelog-cli": "^4.1.0", 13 | "husky": "^8.0.3", 14 | "lerna": "^8.1.2", 15 | "lerna-changelog": "^2.2.0", 16 | "lint-staged": "^15.2.0", 17 | "prettier": "^3.2.2" 18 | }, 19 | "repository": "https://github.com/differentialhq/differential", 20 | "lint-staged": { 21 | "*.{js,css,md,ts,tsx}": "prettier --write" 22 | }, 23 | "scripts": { 24 | "prepare": "npx husky install", 25 | "sync-contracts": "cp control-plane/src/modules/contract.ts ts-core/src/contract.ts && cp control-plane/src/modules/contract.ts ts-core/src/contract.ts && cp control-plane/src/modules/contract.ts admin/client/contract.ts && cp control-plane/src/modules/contract.ts cli/src/client/contract.ts" 26 | } 27 | } -------------------------------------------------------------------------------- /sdk/Makefile: -------------------------------------------------------------------------------- 1 | all: fetch 2 | 3 | update-contract: 4 | mkdir -p src 5 | curl https://api.differential.dev/contract | jq -r '.contract' > src/contract.ts 6 | 7 | clean: 8 | rm -f src/contract.ts 9 | 10 | .PHONY: all fetch clean 11 | -------------------------------------------------------------------------------- /sdk/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/sdk/assets/logo.png -------------------------------------------------------------------------------- /sdk/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | ["@babel/preset-env", { targets: { node: "current" } }], 4 | "@babel/preset-typescript", 5 | ], 6 | }; 7 | -------------------------------------------------------------------------------- /sdk/docs/.nojekyll: -------------------------------------------------------------------------------- 1 | TypeDoc added this file to prevent GitHub Pages from using Jekyll. You can turn off this behavior by setting the `githubPages` option to false. -------------------------------------------------------------------------------- /sdk/docs/modules.md: -------------------------------------------------------------------------------- 1 | # @differentialhq/sdk 2 | -------------------------------------------------------------------------------- /sdk/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@differentialhq/sdk", 3 | "version": "0.0.2", 4 | "description": "Javascript SDK for differential.dev", 5 | "main": "bin/index.js", 6 | "scripts": { 7 | "test": "exit 0", 8 | "test:bak": "jest ./src --runInBand --forceExit --detectOpenHandles", 9 | "test:dev": "jest ./src --watch", 10 | "build": "tsc", 11 | "clean": "rm -rf ./bin", 12 | "docs": "typedoc --plugin typedoc-plugin-markdown --out docs --excludePrivate --hideBreadcrumbs --allReflectionsHaveOwnDocument" 13 | }, 14 | "author": "", 15 | "license": "ISC", 16 | "dependencies": { 17 | "@ts-rest/core": "^3.28.0", 18 | "debug": "^4.3.4", 19 | "google-auth-library": "^8.9.0", 20 | "google-spreadsheet": "^4.1.1", 21 | "msgpackr": "^1.9.7", 22 | "node-os-utils": "^1.3.7", 23 | "zod": "^3.23.5", 24 | "zod-to-json-schema": "^3.23.0" 25 | }, 26 | "devDependencies": { 27 | "@babel/preset-env": "^7.22.10", 28 | "@babel/preset-typescript": "^7.22.11", 29 | "@types/debug": "^4.1.8", 30 | "@types/jest": "^29.5.4", 31 | "@types/node-os-utils": "^1.3.4", 32 | "dotenv": "^16.3.1", 33 | "jest": "^29.6.4", 34 | "typedoc": "^0.25.4", 35 | "typedoc-plugin-markdown": "^3.17.1", 36 | "typescript": "^5.2.2" 37 | }, 38 | "jest": { 39 | "testTimeout": 20000 40 | } 41 | } -------------------------------------------------------------------------------- /sdk/src/Differential.test.ts: -------------------------------------------------------------------------------- 1 | import { Differential } from "./Differential"; 2 | 3 | describe("Differential", () => { 4 | const env = process.env; 5 | beforeEach(() => { 6 | delete process.env.DIFFERENTIAL_API_SECRET; 7 | }); 8 | 9 | afterEach(() => { 10 | process.env = { ...env }; 11 | }); 12 | it("should initialize without optional args", () => { 13 | expect(() => new Differential("test")).not.toThrow(); 14 | }); 15 | 16 | it("should throw if no API secret is provided", () => { 17 | expect(() => new Differential()).toThrow(); 18 | }); 19 | 20 | it("should initialize with API secret in environment", () => { 21 | process.env.DIFFERENTIAL_API_SECRET = "environment_secret"; 22 | expect(() => new Differential()).not.toThrow(); 23 | const d = new Differential(); 24 | expect(d.secretPartial).toBe("envi..."); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /sdk/src/call-config.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Configures the function call with extra options. All options need to be configured under the $d key. 3 | * 4 | * @example 5 | * ```ts 6 | * await client.getUsers({ 7 | * $d: { 8 | * cache: { 9 | * key: "users", 10 | * ttlSeconds: 60, 11 | * }, 12 | * }); 13 | * ``` 14 | */ 15 | export type CallConfig = { 16 | cache?: { 17 | key: string; 18 | ttlSeconds: number; 19 | }; 20 | retryCountOnStall?: number; 21 | predictiveRetriesOnRejection?: boolean; 22 | timeoutSeconds?: number; 23 | executionId?: string; 24 | // TODO: time travel 25 | }; 26 | 27 | export type $d = { 28 | $d: CallConfig; 29 | }; 30 | 31 | type AddParameters = ( 32 | ...args: [Arg, Parameter] 33 | ) => ReturnType; 34 | 35 | export type CallConfiguredFunction any> = 36 | AddParameters< 37 | Parameters[0], 38 | ReturnType, 39 | $d | undefined 40 | >; 41 | 42 | export type CallConfiguredBackgroundFunction< 43 | TFunction extends (...args: any) => any, 44 | > = AddParameters>; 45 | 46 | export const extractCallConfig = ( 47 | args: any[], 48 | ): { 49 | callConfig?: $d["$d"]; 50 | originalArgs: any[]; 51 | } => { 52 | const lastArg = args[args.length - 1]; 53 | 54 | if (typeof lastArg === "object" && lastArg !== null && "$d" in lastArg) { 55 | return { 56 | callConfig: lastArg.$d, 57 | originalArgs: args.slice(0, args.length - 1), 58 | }; 59 | } 60 | 61 | return { 62 | originalArgs: args, 63 | }; 64 | }; 65 | -------------------------------------------------------------------------------- /sdk/src/create-client.ts: -------------------------------------------------------------------------------- 1 | import { initClient, tsRestFetchApi } from "@ts-rest/core"; 2 | import { contract } from "./contract"; 3 | 4 | export const createClient = ({ 5 | baseUrl, 6 | machineId, 7 | deploymentId, 8 | clientAbortController, 9 | }: { 10 | baseUrl: string; 11 | machineId: string; 12 | deploymentId?: string; 13 | clientAbortController?: AbortController; 14 | }) => 15 | initClient(contract, { 16 | baseUrl, 17 | baseHeaders: { 18 | "x-machine-id": machineId, 19 | ...(deploymentId && { "x-deployment-id": deploymentId }), 20 | }, 21 | api: clientAbortController 22 | ? (args) => { 23 | return tsRestFetchApi({ 24 | ...args, 25 | signal: clientAbortController.signal, 26 | }); 27 | } 28 | : undefined, 29 | }); 30 | -------------------------------------------------------------------------------- /sdk/src/errors.ts: -------------------------------------------------------------------------------- 1 | export class DifferentialError extends Error { 2 | static UNAUTHORISED = 3 | "Invalid API Key or API Secret. Make sure you are using the correct API Secret."; 4 | 5 | static UNKNOWN_ENCRYPTION_KEY = 6 | "Encountered an encrypted message with an unknown encryption key. Make sure you are providing all encryption keys to the client."; 7 | 8 | static EXECUTION_DID_NOT_COMPLETE = 9 | "An error occurred while executing the function remotely. Machine may have stalled too many times or the function timed out before reporting a completion status."; 10 | 11 | static TOO_MANY_NETWORK_ERRORS = 12 | "Too many network errors occurred. Make sure the client is connected to the internet."; 13 | 14 | static INVALID_DATA_TYPE = 15 | "Serialization process encountered an invalid data type. The data can not be safely serialized. See: https://docs.differential.dev/advanced/arguments-and-return-values/"; 16 | 17 | static TIMEOUT = 18 | "The function timed out before being able to complete. Make sure the function is not stalling."; 19 | 20 | constructor(message: string, meta?: { [key: string]: unknown }) { 21 | super(message); 22 | this.name = "DifferentialError"; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /sdk/src/events.test.ts: -------------------------------------------------------------------------------- 1 | import * as osu from "node-os-utils"; 2 | import { Events } from "./events"; 3 | 4 | describe("events", () => { 5 | beforeAll(() => { 6 | jest.useFakeTimers(); 7 | }); 8 | 9 | afterAll(() => { 10 | jest.useRealTimers(); 11 | }); 12 | 13 | it("should flush events when bufferFlushSize is reached", () => { 14 | const mockFn = jest.fn(); 15 | const events = new Events(mockFn, { 16 | bufferFlushSize: 2, 17 | }); 18 | expect(mockFn).toHaveBeenCalledTimes(0); 19 | events.push({ type: "functionInvocation", timestamp: new Date() }); 20 | expect(mockFn).toHaveBeenCalledTimes(0); 21 | events.push({ type: "functionInvocation", timestamp: new Date() }); 22 | expect(mockFn).toHaveBeenCalledTimes(1); 23 | events.push({ type: "functionInvocation", timestamp: new Date() }); 24 | expect(mockFn).toHaveBeenCalledTimes(1); 25 | events.push({ type: "functionInvocation", timestamp: new Date() }); 26 | expect(mockFn).toHaveBeenCalledTimes(2); 27 | }); 28 | 29 | it("should schedule a system probe", () => { 30 | const mockFn = jest.fn(); 31 | osu.cpu.usage = mockFn.mockResolvedValue(0.5); 32 | 33 | const events = new Events(mockFn, { 34 | systemProbeInterval: 100, 35 | }); 36 | 37 | events.startResourceProbe(); 38 | expect(mockFn).toHaveBeenCalledTimes(0); 39 | 40 | jest.advanceTimersByTime(100); 41 | expect(mockFn).toHaveBeenCalledTimes(1); 42 | 43 | jest.advanceTimersByTime(100); 44 | expect(mockFn).toHaveBeenCalledTimes(2); 45 | 46 | events.stopResourceProbe(); 47 | jest.advanceTimersByTime(100); 48 | expect(mockFn).toHaveBeenCalledTimes(2); 49 | }); 50 | }); 51 | -------------------------------------------------------------------------------- /sdk/src/events.ts: -------------------------------------------------------------------------------- 1 | import * as osu from "node-os-utils"; 2 | 3 | type EventTypes = "functionInvocation" | "machineResourceProbe"; 4 | type Event = { 5 | timestamp: Date; 6 | type: EventTypes; 7 | tags?: Record; 8 | intFields?: Record; 9 | }; 10 | 11 | type PublishEvents = (events: Event[]) => Promise; 12 | 13 | export class Events { 14 | private metricsInterval?: NodeJS.Timeout; 15 | private eventBuffer: Event[] = []; 16 | private publish: PublishEvents; 17 | private systemProbeInterval: number; 18 | private bufferFlushSize: number; 19 | 20 | constructor( 21 | publish: PublishEvents, 22 | options?: { 23 | systemProbeInterval?: number; 24 | bufferFlushSize?: number; 25 | }, 26 | ) { 27 | this.publish = publish; 28 | this.systemProbeInterval = options?.systemProbeInterval || 5000; 29 | this.bufferFlushSize = options?.bufferFlushSize || 1000; 30 | } 31 | 32 | public startResourceProbe() { 33 | if (this.metricsInterval) { 34 | return; 35 | } 36 | 37 | this.metricsInterval = setInterval(async () => { 38 | const cpuUsage = await osu.cpu.usage(); 39 | const memUsage = await osu.mem.info(); 40 | 41 | this.eventBuffer.push({ 42 | type: "machineResourceProbe", 43 | timestamp: new Date(), 44 | intFields: { 45 | cpuPercentage: cpuUsage, 46 | memPercentage: memUsage.usedMemPercentage, 47 | memUsageByte: process.memoryUsage().rss, 48 | }, 49 | }); 50 | 51 | this.flush(); 52 | }, this.systemProbeInterval); 53 | } 54 | 55 | public stopResourceProbe() { 56 | if (this.metricsInterval) { 57 | clearInterval(this.metricsInterval); 58 | this.metricsInterval = undefined; 59 | } 60 | } 61 | 62 | public push(event: Event) { 63 | this.eventBuffer.push(event); 64 | this.flush(); 65 | } 66 | 67 | public flush() { 68 | if (this.eventBuffer.length < this.bufferFlushSize) { 69 | return; 70 | } 71 | 72 | // Do not await for the publish to finish 73 | this.publish(this.eventBuffer); 74 | 75 | this.eventBuffer = []; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /sdk/src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * # @differentialhq/core 3 | * 4 | * ## Installation 5 | * 6 | * ```bash 7 | * npm install @differentialhq/core 8 | * ``` 9 | * 10 | * ```bash 11 | * yarn add @differentialhq/core 12 | * ``` 13 | * 14 | * ```bash 15 | * pnpm add @differentialhq/core 16 | * ``` 17 | */ 18 | 19 | export { Differential } from "./Differential"; 20 | 21 | export { CallConfig } from "./call-config"; 22 | -------------------------------------------------------------------------------- /sdk/src/serialize.test.ts: -------------------------------------------------------------------------------- 1 | import crypto from "crypto"; 2 | import { pack, unpack } from "./serialize"; 3 | 4 | describe("serailizing and deserializing", () => { 5 | const values = [ 6 | 1, 7 | "string", 8 | true, 9 | false, 10 | null, 11 | undefined, 12 | { hello: "Bob" }, 13 | Buffer.from("hello"), 14 | [1, 2, 3], 15 | new Date(), 16 | ]; 17 | 18 | values.forEach((value) => { 19 | it(`should serialise ${value}`, () => { 20 | expect(unpack(pack(value, true))).toEqual(value); 21 | }); 22 | }); 23 | }); 24 | 25 | describe("encryption", () => { 26 | const cryptoSettings = { 27 | keys: [Buffer.from("abcdefghijklmnopqrstuvwxzy123456")], 28 | }; 29 | 30 | it("should encrypt and decrypt", () => { 31 | const encrypted = pack({ hello: "Bob" }, true, { cryptoSettings }); 32 | const decrypted = unpack(encrypted, { cryptoSettings }); 33 | 34 | expect(decrypted).toEqual({ hello: "Bob" }); 35 | }); 36 | 37 | it("should be able to encrypt and decrypt when keys roll", () => { 38 | const encrypted = pack({ hello: "Bob" }, true, { cryptoSettings }); 39 | 40 | const newCryptoSettings = { 41 | keys: [crypto.randomBytes(32), ...cryptoSettings.keys], 42 | }; 43 | 44 | const decrypted = unpack(encrypted, { cryptoSettings: newCryptoSettings }); 45 | 46 | expect(decrypted).toEqual({ hello: "Bob" }); 47 | }); 48 | }); 49 | -------------------------------------------------------------------------------- /sdk/src/task-queue.ts: -------------------------------------------------------------------------------- 1 | import debug from "debug"; 2 | import { serializeError } from "./serialize-error"; 3 | import { AsyncFunction } from "./types"; 4 | 5 | const log = debug("differential:client"); 6 | 7 | export type Result = { 8 | content: T; 9 | type: "resolution" | "rejection"; 10 | functionExecutionTime?: number; 11 | }; 12 | 13 | export const executeFn = async ( 14 | fn: AsyncFunction["func"], 15 | args: Parameters, 16 | ): Promise => { 17 | const start = Date.now(); 18 | try { 19 | const result = await fn(...args); 20 | 21 | return { 22 | content: result, 23 | type: "resolution", 24 | functionExecutionTime: Date.now() - start, 25 | }; 26 | } catch (e) { 27 | const functionExecutionTime = Date.now() - start; 28 | if (e instanceof Error) { 29 | return { 30 | content: serializeError(e), 31 | type: "rejection", 32 | functionExecutionTime, 33 | }; 34 | } else if (typeof e === "string") { 35 | return { 36 | content: serializeError(new Error(e)), 37 | type: "rejection", 38 | functionExecutionTime, 39 | }; 40 | } else { 41 | return { 42 | content: new Error( 43 | "Differential encountered an unexpected error type. Make sure you are throwing an Error object.", 44 | ), 45 | type: "rejection", 46 | functionExecutionTime, 47 | }; 48 | } 49 | } 50 | }; 51 | 52 | export class TaskQueue { 53 | private tasks: Array<{ 54 | fn: AsyncFunction["func"]; 55 | args: Parameters; 56 | resolve: (value: Result) => void; 57 | }> = []; 58 | 59 | addTask( 60 | fn: AsyncFunction["func"], 61 | args: Parameters, 62 | resolve: (value: Result) => void, 63 | ) { 64 | this.tasks.push({ 65 | fn, 66 | args, 67 | resolve, 68 | }); 69 | 70 | this.run(); 71 | } 72 | 73 | private run() { 74 | const tasks = this.tasks; 75 | this.tasks = []; 76 | 77 | Promise.all( 78 | tasks.map((task) => executeFn(task.fn, task.args).then(task.resolve)), 79 | ); 80 | } 81 | 82 | async quit() { 83 | return new Promise((resolve) => { 84 | if (this.tasks.length === 0) { 85 | resolve(); 86 | } else { 87 | const interval = setInterval(() => { 88 | if (this.tasks.length === 0) { 89 | clearInterval(interval); 90 | resolve(); 91 | } 92 | }, 100); 93 | } 94 | }); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /sdk/src/tests/e2ee/d.ts: -------------------------------------------------------------------------------- 1 | require("dotenv").config(); 2 | 3 | import { Differential } from "../../Differential"; 4 | 5 | if (!process.env.DIFFERENTIAL_API_SECRET) { 6 | throw new Error("Missing env DIFFERENTIAL_API_SECRET"); 7 | } 8 | 9 | export const d = new Differential(process.env.DIFFERENTIAL_API_SECRET, { 10 | encryptionKeys: [Buffer.from("abcdefghijklmnopqrstuvwxzy123456")], 11 | endpoint: process.env.DIFFERENTIAL_API_ENDPOINT_OVERRIDE, 12 | jobPollWaitTime: 5000, 13 | }); 14 | -------------------------------------------------------------------------------- /sdk/src/tests/e2ee/e2ee.test.ts: -------------------------------------------------------------------------------- 1 | import { d } from "./d"; 2 | import { helloService } from "./hello"; 3 | 4 | describe("e2ee", () => { 5 | it("should be able to call a service", async () => { 6 | await helloService.start(); 7 | 8 | const result = await d 9 | .client("hello") 10 | .greet(["Bob", "Alice"]); 11 | 12 | expect(result).toEqual({ 13 | result: "Hello Bob, Alice", 14 | names: ["Bob", "Alice"], 15 | }); 16 | 17 | await helloService.stop(); 18 | }); 19 | }); 20 | -------------------------------------------------------------------------------- /sdk/src/tests/e2ee/hello.ts: -------------------------------------------------------------------------------- 1 | import { d } from "./d"; 2 | 3 | export const greet = async (names: string[]) => { 4 | return { 5 | result: `Hello ${names.join(", ")}`, 6 | names, 7 | }; 8 | }; 9 | 10 | export const helloService = d.service({ 11 | name: "hello", 12 | functions: { 13 | greet, 14 | }, 15 | }); 16 | -------------------------------------------------------------------------------- /sdk/src/tests/errors/animals.ts: -------------------------------------------------------------------------------- 1 | import { d } from "./d"; 2 | 3 | export const getNormalAnimal = async () => { 4 | throw new Error("This is a normal error"); 5 | }; 6 | 7 | export class AnimalError extends Error { 8 | constructor(message: string) { 9 | super(message); 10 | this.name = "AnimalError"; 11 | } 12 | } 13 | 14 | export const getCustomAnimal = async () => { 15 | throw new AnimalError("This is a custom error"); 16 | }; 17 | 18 | export const animalService = d.service({ 19 | name: "animal", 20 | functions: { 21 | getNormalAnimal, 22 | getCustomAnimal, 23 | }, 24 | }); 25 | -------------------------------------------------------------------------------- /sdk/src/tests/errors/d.ts: -------------------------------------------------------------------------------- 1 | require("dotenv").config(); 2 | 3 | import { Differential } from "../../Differential"; 4 | 5 | if (!process.env.DIFFERENTIAL_API_SECRET) { 6 | throw new Error("Missing env DIFFERENTIAL_API_SECRET"); 7 | } 8 | 9 | export const d = new Differential(process.env.DIFFERENTIAL_API_SECRET, { 10 | endpoint: process.env.DIFFERENTIAL_API_ENDPOINT_OVERRIDE, 11 | jobPollWaitTime: 5000, 12 | }); 13 | -------------------------------------------------------------------------------- /sdk/src/tests/errors/errors.test.ts: -------------------------------------------------------------------------------- 1 | import { animalService } from "./animals"; 2 | import { d } from "./d"; 3 | 4 | describe("Errors", () => { 5 | beforeAll(async () => { 6 | await animalService.start(); 7 | }); 8 | 9 | afterAll(async () => { 10 | await animalService.stop(); 11 | }); 12 | 13 | it("should get the normal error", async () => { 14 | const client = d.client("animal"); 15 | 16 | expect(client.getNormalAnimal()).rejects.toThrow("This is a normal error"); 17 | }); 18 | 19 | it("should get the custom error", async () => { 20 | const client = d.client("animal"); 21 | 22 | try { 23 | await client.getCustomAnimal(); 24 | expect(true).toBe(false); 25 | } catch (e: any) { 26 | expect(e.name).toBe("AnimalError"); 27 | expect(e.name).not.toBe("Animal2Error"); 28 | } 29 | }); 30 | }); 31 | -------------------------------------------------------------------------------- /sdk/src/tests/monolith/d.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/sdk/src/tests/monolith/d.ts -------------------------------------------------------------------------------- /sdk/src/tests/monolith/db.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/sdk/src/tests/monolith/db.ts -------------------------------------------------------------------------------- /sdk/src/tests/monolith/differential-398308-5f1deaf67146.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/sdk/src/tests/monolith/differential-398308-5f1deaf67146.json -------------------------------------------------------------------------------- /sdk/src/tests/monolith/expert.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/sdk/src/tests/monolith/expert.ts -------------------------------------------------------------------------------- /sdk/src/tests/monolith/facade.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/sdk/src/tests/monolith/facade.ts -------------------------------------------------------------------------------- /sdk/src/tests/monolith/monolith.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/sdk/src/tests/monolith/monolith.test.ts -------------------------------------------------------------------------------- /sdk/src/tests/monolith/run.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/differentialhq/differential/f521a42b65d2bef33dbf69d8537238ab6465f7e4/sdk/src/tests/monolith/run.ts -------------------------------------------------------------------------------- /sdk/src/tests/utility/caching.test.ts: -------------------------------------------------------------------------------- 1 | import { d } from "./d"; 2 | import { productService } from "./product"; 3 | 4 | describe("Caching", () => { 5 | beforeAll(async () => { 6 | await productService.start(); 7 | }, 10000); 8 | 9 | afterAll(async () => { 10 | await productService.stop(); 11 | }); 12 | 13 | it("should get the cached results when possible", async () => { 14 | const client = d.client("product"); 15 | 16 | const productId = Math.random().toString(); 17 | 18 | const result1 = await client.getProduct(productId, "foo", { 19 | $d: { 20 | cache: { 21 | key: productId, 22 | ttlSeconds: 10, 23 | }, 24 | }, 25 | }); 26 | 27 | const result2 = await client.getProduct(productId, "bar", { 28 | $d: { 29 | cache: { 30 | key: productId, 31 | ttlSeconds: 10, 32 | }, 33 | }, 34 | }); 35 | 36 | expect(result1).toEqual(result2); 37 | }); 38 | 39 | it("should respect cache ttl", async () => { 40 | const client = d.client("product"); 41 | 42 | const productId = Math.random().toString(); 43 | 44 | const result1 = await client.getProduct(productId, "foo", { 45 | $d: { 46 | cache: { 47 | key: productId, 48 | ttlSeconds: 1, 49 | }, 50 | }, 51 | }); 52 | 53 | await new Promise((resolve) => setTimeout(resolve, 2000)); // wait for cache to expire 54 | 55 | const result2 = await client.getProduct(productId, "bar", { 56 | $d: { 57 | cache: { 58 | key: productId, 59 | ttlSeconds: 1, 60 | }, 61 | }, 62 | }); 63 | 64 | expect(result1).not.toEqual(result2); 65 | }); 66 | }); 67 | -------------------------------------------------------------------------------- /sdk/src/tests/utility/d.ts: -------------------------------------------------------------------------------- 1 | require("dotenv").config(); 2 | 3 | import { Differential } from "../../Differential"; 4 | 5 | if (!process.env.DIFFERENTIAL_API_SECRET) { 6 | throw new Error("Missing env DIFFERENTIAL_API_SECRET"); 7 | } 8 | 9 | export const d = new Differential(process.env.DIFFERENTIAL_API_SECRET, { 10 | endpoint: process.env.DIFFERENTIAL_API_ENDPOINT_OVERRIDE, 11 | jobPollWaitTime: 5000, 12 | }); 13 | -------------------------------------------------------------------------------- /sdk/src/tests/utility/product.ts: -------------------------------------------------------------------------------- 1 | import { d } from "./d"; 2 | 3 | const cache = new Map(); 4 | 5 | export const getProduct = async (id: string, random: string) => { 6 | return { 7 | id, 8 | name: `Product ${id}`, 9 | random, 10 | }; 11 | }; 12 | 13 | export const succeedsOnSecondAttempt = async (id: string) => { 14 | if (cache.has(id)) { 15 | return true; 16 | } else { 17 | cache.set(id, true); 18 | // wait 5s and time out 19 | await new Promise((resolve) => setTimeout(resolve, 60000)); 20 | } 21 | }; 22 | 23 | export const productService = d.service({ 24 | name: "product", 25 | functions: { 26 | getProduct, 27 | succeedsOnSecondAttempt, 28 | }, 29 | }); 30 | -------------------------------------------------------------------------------- /sdk/src/tests/utility/retry.test.ts: -------------------------------------------------------------------------------- 1 | import { DifferentialError } from "../../errors"; 2 | import { d } from "./d"; 3 | import { productService } from "./product"; 4 | 5 | describe("retrying", () => { 6 | beforeAll(async () => { 7 | await productService.start(); 8 | }); 9 | 10 | afterAll(async () => { 11 | await productService.stop(); 12 | }); 13 | 14 | it("should not retry a function when attempts is 1", async () => { 15 | const client = d.client("product"); 16 | 17 | const productId = Math.random().toString(); 18 | 19 | await expect( 20 | client.succeedsOnSecondAttempt(productId, { 21 | $d: { 22 | retryCountOnStall: 0, 23 | timeoutSeconds: 5, 24 | }, 25 | }), 26 | ).rejects.toThrow(DifferentialError.EXECUTION_DID_NOT_COMPLETE); 27 | }); 28 | 29 | it("should be able to retry a function", async () => { 30 | const client = d.client("product"); 31 | 32 | const productId = Math.random().toString(); 33 | 34 | const result = await client.succeedsOnSecondAttempt(productId, { 35 | $d: { 36 | retryCountOnStall: 2, 37 | timeoutSeconds: 5, 38 | }, 39 | }); 40 | 41 | expect(result).toBe(true); 42 | }); 43 | }); 44 | -------------------------------------------------------------------------------- /sdk/src/types.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | 3 | export type AsyncFunction = { 4 | input: z.ZodObject; 5 | func: (args: Args) => Promise; 6 | }; 7 | -------------------------------------------------------------------------------- /sdk/src/util.ts: -------------------------------------------------------------------------------- 1 | export const throttle = ( 2 | fn: (...args: A) => R, 3 | delay: number, 4 | ): [(...args: A) => R | undefined, () => void] => { 5 | let wait = false; 6 | let timeout: NodeJS.Timeout; 7 | let cancelled = false; 8 | 9 | return [ 10 | (...args: A) => { 11 | if (cancelled) return undefined; 12 | if (wait) return undefined; 13 | 14 | const val = fn(...args); 15 | 16 | wait = true; 17 | 18 | timeout = setTimeout(() => { 19 | wait = false; 20 | }, delay); 21 | 22 | return val; 23 | }, 24 | () => { 25 | cancelled = true; 26 | clearTimeout(timeout); 27 | }, 28 | ]; 29 | }; 30 | -------------------------------------------------------------------------------- /sdk/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "commonjs", 5 | "outDir": "bin", 6 | "rootDir": "src", 7 | "declaration": true, 8 | "sourceMap": true, 9 | "strict": true, 10 | "esModuleInterop": true, 11 | "allowJs": true 12 | }, 13 | "include": [ 14 | "src/index.ts" 15 | ], 16 | "exclude": [ 17 | "node_modules" 18 | ] 19 | } -------------------------------------------------------------------------------- /sdk/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "kindSortOrder": [ 3 | "Class", 4 | "Reference", 5 | "Project", 6 | "Module", 7 | "Namespace", 8 | "Enum", 9 | "EnumMember", 10 | "Interface", 11 | "TypeAlias", 12 | "Constructor", 13 | "Property", 14 | "Variable", 15 | "Function", 16 | "Accessor", 17 | "Method", 18 | "Parameter", 19 | "TypeParameter", 20 | "TypeLiteral", 21 | "CallSignature", 22 | "ConstructorSignature", 23 | "IndexSignature", 24 | "GetSignature", 25 | "SetSignature", 26 | ] 27 | } --------------------------------------------------------------------------------