├── .dockerignore ├── .eslintrc.js ├── .github ├── copilot-instructions.md └── workflows │ ├── auto-approve-pr.yml │ ├── auto-create-pr.yml │ ├── build-push-deploy.yml │ └── run-tests.yml ├── .gitignore ├── .gitmodules ├── .prettierrc ├── AGENTS.md ├── CLAUDE.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SECURITY.md ├── bun.lock ├── package.json ├── packages ├── backend │ ├── .env.example │ ├── .gitignore │ ├── package.json │ ├── src │ │ ├── access-control │ │ │ └── authorization.ts │ │ ├── api │ │ │ ├── v1 │ │ │ │ ├── alerts.ts │ │ │ │ ├── analytics │ │ │ │ │ ├── index.ts │ │ │ │ │ └── utils.ts │ │ │ │ ├── audit-logs │ │ │ │ │ ├── index.ts │ │ │ │ │ └── utils.ts │ │ │ │ ├── auth │ │ │ │ │ ├── github.ts │ │ │ │ │ ├── google.ts │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── saml.ts │ │ │ │ │ └── utils.ts │ │ │ │ ├── charts.ts │ │ │ │ ├── checklists.ts │ │ │ │ ├── dashboards.ts │ │ │ │ ├── data-warehouse │ │ │ │ │ ├── index.ts │ │ │ │ │ └── utils.ts │ │ │ │ ├── datasets │ │ │ │ │ ├── index.ts │ │ │ │ │ └── utils.ts │ │ │ │ ├── evals │ │ │ │ │ └── index.ts │ │ │ │ ├── evaluations │ │ │ │ │ ├── index.ts │ │ │ │ │ └── utils.ts │ │ │ │ ├── evaluator.ts │ │ │ │ ├── external-users.ts │ │ │ │ ├── filters.ts │ │ │ │ ├── index.ts │ │ │ │ ├── jobs.ts │ │ │ │ ├── models.ts │ │ │ │ ├── openapi.ts │ │ │ │ ├── orgs.ts │ │ │ │ ├── projects │ │ │ │ │ ├── index.ts │ │ │ │ │ └── utils.ts │ │ │ │ ├── prompts.ts │ │ │ │ ├── provider-configs.ts │ │ │ │ ├── redirections.ts │ │ │ │ ├── runs │ │ │ │ │ ├── export.ts │ │ │ │ │ ├── index.ts │ │ │ │ │ ├── ingest.ts │ │ │ │ │ ├── queries.ts │ │ │ │ │ └── utils.ts │ │ │ │ ├── template-versions.ts │ │ │ │ ├── templates.ts │ │ │ │ ├── users.ts │ │ │ │ └── views.ts │ │ │ └── webhooks │ │ │ │ ├── index.ts │ │ │ │ └── stripe.ts │ │ ├── checks │ │ │ ├── ai │ │ │ │ ├── assert.ts │ │ │ │ ├── fact.ts │ │ │ │ ├── sentiment.ts │ │ │ │ └── similarity.ts │ │ │ ├── index.ts │ │ │ ├── runChecks.ts │ │ │ └── utils.ts │ │ ├── create-indexes.ts │ │ ├── emails │ │ │ ├── index.ts │ │ │ ├── sender.ts │ │ │ ├── templates.ts │ │ │ └── utils.ts │ │ ├── evaluators │ │ │ ├── bias.ts │ │ │ ├── bleu.ts │ │ │ ├── cosine.ts │ │ │ ├── fuzzy.ts │ │ │ ├── gleu.ts │ │ │ ├── guidelines.ts │ │ │ ├── index.ts │ │ │ ├── language.ts │ │ │ ├── llm.ts │ │ │ ├── old-assert.ts │ │ │ ├── pii.ts │ │ │ ├── replies.ts │ │ │ ├── rouge.ts │ │ │ ├── sentiment.ts │ │ │ ├── string.ts │ │ │ ├── topics.ts │ │ │ ├── toxicity.ts │ │ │ └── utils │ │ │ │ └── index.ts │ │ ├── index.ts │ │ ├── jobs.ts │ │ ├── jobs │ │ │ ├── alerts.ts │ │ │ ├── data-retention.ts │ │ │ ├── materialized-views.ts │ │ │ ├── realtime-evaluators.ts │ │ │ ├── resetUsage.ts │ │ │ └── stripeMeters.ts │ │ ├── migrate.ts │ │ ├── realtime-evaluators.ts │ │ ├── types │ │ │ ├── database.ts │ │ │ └── index.ts │ │ └── utils │ │ │ ├── alertMetrics.ts │ │ │ ├── authorization.ts │ │ │ ├── cache.ts │ │ │ ├── checks.ts │ │ │ ├── config.ts │ │ │ ├── cors.ts │ │ │ ├── cost.ts │ │ │ ├── countToken.ts │ │ │ ├── cron.ts │ │ │ ├── date.ts │ │ │ ├── db.ts │ │ │ ├── errors.ts │ │ │ ├── ibm.ts │ │ │ ├── ingest.ts │ │ │ ├── instrument.ts │ │ │ ├── jsonLogic.ts │ │ │ ├── koa.ts │ │ │ ├── license.ts │ │ │ ├── middleware.ts │ │ │ ├── misc.ts │ │ │ ├── ml.ts │ │ │ ├── notifications.ts │ │ │ ├── openai.ts │ │ │ ├── playground.ts │ │ │ ├── ratelimit.ts │ │ │ ├── sentry.ts │ │ │ ├── stripe.ts │ │ │ ├── timeout.ts │ │ │ └── tokens │ │ │ ├── google.ts │ │ │ └── openai.ts │ └── tsconfig.json ├── db │ ├── 0001-init.sql │ ├── 0002.sql │ ├── 0003.sql │ ├── 0004.sql │ ├── 0005.sql │ ├── 0006.sql │ ├── 0007.sql │ ├── 0008.sql │ ├── 0009.sql │ ├── 0010.sql │ ├── 0011.sql │ ├── 0012.sql │ ├── 0013.sql │ ├── 0014.sql │ ├── 0015.sql │ ├── 0016.sql │ ├── 0017.sql │ ├── 0018.sql │ ├── 0019.sql │ ├── 0020.sql │ ├── 0021.sql │ ├── 0022.sql │ ├── 0023.sql │ ├── 0024.sql │ ├── 0025.sql │ ├── 0026.sql │ ├── 0027.sql │ ├── 0028.sql │ ├── 0029.sql │ ├── 0030.sql │ ├── 0031.sql │ ├── 0032.sql │ ├── 0033.sql │ ├── 0034.sql │ ├── 0035.sql │ ├── 0036.sql │ ├── 0037.sql │ ├── 0038.sql │ ├── 0039.sql │ ├── 0040.sql │ ├── 0041.sql │ ├── 0042.sql │ ├── 0043.sql │ ├── 0044.sql │ ├── 0045.sql │ ├── 0046.sql │ ├── 0047.sql │ ├── 0048.sql │ ├── 0049.sql │ ├── 0050.sql │ ├── 0051.sql │ ├── 0052.sql │ ├── 0053.sql │ ├── 0054.sql │ ├── 0055.sql │ ├── 0056.sql │ ├── 0057.sql │ ├── 0058.sql │ ├── 0059.sql │ ├── 0060.sql │ ├── 0061.sql │ ├── 0062.sql │ ├── 0063.sql │ ├── 0064.sql │ ├── 0065.sql │ ├── 0066.sql │ ├── 0067.sql │ ├── 0068.sql │ ├── 0069.sql │ ├── 0070.sql │ ├── 0071.sql │ ├── 0072.sql │ ├── 0073.sql │ ├── 0074.sql │ ├── 0075.sql │ ├── 0076.sql │ ├── 0077.sql │ ├── 0078.sql │ ├── 0079.sql │ ├── 0080.sql │ ├── 0081.sql │ ├── 0082.sql │ ├── 0083.sql │ ├── 0084.sql │ ├── 0085.sql │ ├── 0086.sql │ ├── 0087.sql │ ├── 0088.sql │ ├── 0089.sql │ ├── 0090.sql │ ├── 0091.sql │ ├── 0092.sql │ ├── 0093.sql │ ├── 0094.sql │ ├── 0095.sql │ ├── 0096.sql │ ├── 0097.sql │ └── 0098.sql ├── e2e │ ├── .gitignore │ ├── api.spec.ts │ ├── auth.setup.ts │ ├── global.teardown.ts │ ├── login.spec.ts │ ├── logs.spec.ts │ ├── package.json │ ├── playwright.config.ts │ ├── projects.setup.ts │ ├── templates.spec.ts │ ├── users.spec.ts │ └── utils │ │ └── db.ts ├── frontend │ ├── .dockerignore │ ├── .env.example │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ ├── actions │ │ └── openai-actions.ts │ ├── components.json │ ├── components │ │ ├── SmartViewer │ │ │ ├── AudioPlayer.tsx │ │ │ ├── HighlightPii.tsx │ │ │ ├── Message.tsx │ │ │ ├── MessageViewer.tsx │ │ │ ├── RenderJson.tsx │ │ │ ├── index.module.css │ │ │ └── index.tsx │ │ ├── analytics │ │ │ ├── AgentSummary.tsx │ │ │ ├── AnalyticsCard.tsx │ │ │ ├── BarList.tsx │ │ │ ├── ChartCreator.tsx │ │ │ ├── Charts │ │ │ │ ├── AreaChartComponent.tsx │ │ │ │ ├── BarChartComponent.tsx │ │ │ │ ├── ChartComponent.tsx │ │ │ │ ├── Sentiment.tsx │ │ │ │ ├── TopAgents.tsx │ │ │ │ └── TopModels.tsx │ │ │ ├── DashboardModal.tsx │ │ │ ├── DashboardsSidebarButton.tsx │ │ │ ├── DateRangeGranularityPicker.tsx │ │ │ ├── Modals.tsx │ │ │ ├── OldLineChart.tsx │ │ │ ├── TinyPercentChart.tsx │ │ │ ├── TopLanguages.tsx │ │ │ ├── TopTemplates.tsx │ │ │ ├── TopTopics.tsx │ │ │ ├── TopUsers.tsx │ │ │ └── Wrappers.tsx │ │ ├── blocks │ │ │ ├── AppUserAvatar │ │ │ │ ├── index.module.css │ │ │ │ └── index.tsx │ │ │ ├── CopyText.tsx │ │ │ ├── DataTable │ │ │ │ ├── TableBody.tsx │ │ │ │ ├── TableHeader.tsx │ │ │ │ ├── index.module.css │ │ │ │ └── index.tsx │ │ │ ├── DurationBadge.tsx │ │ │ ├── ErrorBoundary.tsx │ │ │ ├── FacetedFilter.tsx │ │ │ ├── Feedbacks │ │ │ │ └── index.tsx │ │ │ ├── HotkeysInfo.tsx │ │ │ ├── IconPicker.tsx │ │ │ ├── Logo.tsx │ │ │ ├── MultiSelectButton.tsx │ │ │ ├── OAuth │ │ │ │ ├── GithubButton.tsx │ │ │ │ └── GoogleButton.tsx │ │ │ ├── OldFeedback.tsx │ │ │ ├── OrgUserBadge.tsx │ │ │ ├── ProtectedText.tsx │ │ │ ├── RenamableField.tsx │ │ │ ├── RingLoader.tsx │ │ │ ├── RunChat.tsx │ │ │ ├── RunInputOutput.tsx │ │ │ ├── SearchBar.tsx │ │ │ ├── SeatAllowanceCard.tsx │ │ │ ├── SettingsCard.tsx │ │ │ ├── SocialProof.tsx │ │ │ ├── StatusBadge.tsx │ │ │ ├── Steps.tsx │ │ │ ├── TokensBadge.tsx │ │ │ └── UserAvatar.tsx │ │ ├── checks │ │ │ ├── AddCheck.tsx │ │ │ ├── ChecksInputs.tsx │ │ │ ├── ChecksModal.tsx │ │ │ ├── ChecksUIData.tsx │ │ │ ├── MultiTextInput.tsx │ │ │ ├── Picker.tsx │ │ │ ├── SmartSelectInput.tsx │ │ │ ├── UserSelectInput.tsx │ │ │ └── index.module.css │ │ ├── evals │ │ │ ├── ResultsMatrix.tsx │ │ │ └── index.module.css │ │ ├── layout │ │ │ ├── Analytics.tsx │ │ │ ├── AuthLayout.tsx │ │ │ ├── Empty.tsx │ │ │ ├── Navbar.tsx │ │ │ ├── Paywall.tsx │ │ │ ├── Sidebar.tsx │ │ │ ├── UpgradeModal.tsx │ │ │ ├── index.tsx │ │ │ └── sidebar.module.css │ │ ├── prompts │ │ │ ├── ModelSelect.module.css │ │ │ ├── ModelSelect.tsx │ │ │ ├── PromptEditor.tsx │ │ │ ├── PromptVariableEditor.tsx │ │ │ ├── Provider.tsx │ │ │ ├── TemplateInputArea.tsx │ │ │ ├── TemplateMenu.tsx │ │ │ └── VariableTextarea.tsx │ │ ├── providers │ │ │ ├── ProviderCard.module.css │ │ │ └── ProviderCard.tsx │ │ ├── settings │ │ │ ├── data-warehouse.tsx │ │ │ └── saml.tsx │ │ └── ui │ │ │ ├── toast.tsx │ │ │ └── toaster.tsx │ ├── hooks │ │ ├── use-keyboard-shortcut.ts │ │ ├── use-notifications.ts │ │ └── use-toast.ts │ ├── instrumentation-client.ts │ ├── lib │ │ └── utils.ts │ ├── load-env.sh │ ├── next-env.d.ts │ ├── next.config.js │ ├── package.json │ ├── pages │ │ ├── _app.tsx │ │ ├── _document.tsx │ │ ├── _error.tsx │ │ ├── alerts │ │ │ ├── [id] │ │ │ │ └── edit.tsx │ │ │ ├── index.tsx │ │ │ └── new.tsx │ │ ├── api │ │ │ ├── app │ │ │ │ └── [id].ts │ │ │ ├── evaluate.ts │ │ │ ├── generate-test-case.ts │ │ │ └── run-prompt.ts │ │ ├── billing │ │ │ ├── index.tsx │ │ │ └── thank-you.tsx │ │ ├── checklists.tsx │ │ ├── dashboards │ │ │ ├── [id].tsx │ │ │ └── index.tsx │ │ ├── data-rules │ │ │ └── index.tsx │ │ ├── datasets │ │ │ ├── [id].tsx │ │ │ └── index.tsx │ │ ├── evaluators │ │ │ ├── index.tsx │ │ │ └── new.tsx │ │ ├── experiments │ │ │ └── index.tsx │ │ ├── guardrails │ │ │ └── index.tsx │ │ ├── index.tsx │ │ ├── insights.tsx │ │ ├── join.tsx │ │ ├── login.tsx │ │ ├── logs │ │ │ ├── [id].tsx │ │ │ └── index.tsx │ │ ├── maintenance.tsx │ │ ├── prompts │ │ │ └── [[...id]].tsx │ │ ├── request-password-reset.tsx │ │ ├── reset-password.tsx │ │ ├── settings │ │ │ ├── data-warehouse │ │ │ │ └── bigquery.tsx │ │ │ ├── index.tsx │ │ │ ├── models.tsx │ │ │ └── providers │ │ │ │ ├── [id].tsx │ │ │ │ └── index.tsx │ │ ├── signup.tsx │ │ ├── team │ │ │ ├── audit-logs.tsx │ │ │ ├── index.tsx │ │ │ └── team.module.css │ │ ├── test.module.css │ │ ├── traces │ │ │ └── [id].tsx │ │ ├── update-password.tsx │ │ ├── users │ │ │ └── [[...id]].tsx │ │ └── verify-email.tsx │ ├── postcss.config.js │ ├── postcss.config.mjs │ ├── public │ │ ├── assets │ │ │ ├── accepted-cards.webp │ │ │ ├── amazon-redshift.svg │ │ │ ├── apple-pay.webp │ │ │ ├── azure-synapse-analytics.svg │ │ │ ├── bigquery.svg │ │ │ ├── databricks.svg │ │ │ ├── github-icon-black.svg │ │ │ ├── github-icon-white.svg │ │ │ ├── google-icon.svg │ │ │ ├── google-pay.webp │ │ │ └── snowflake.svg │ │ └── fonts │ │ │ ├── circular-pro-black.woff2 │ │ │ ├── circular-pro-bold.woff2 │ │ │ ├── circular-pro-book.woff2 │ │ │ └── circular-pro-medium.woff2 │ ├── store │ │ ├── evaluators.ts │ │ └── prompt-versions.ts │ ├── styles │ │ └── globals.css │ ├── tsconfig.json │ ├── types │ │ ├── evaluator-types.ts │ │ └── prompt-types.ts │ └── utils │ │ ├── analytics.ts │ │ ├── auth.tsx │ │ ├── colors.ts │ │ ├── config.ts │ │ ├── context.ts │ │ ├── countries.ts │ │ ├── dataHooks │ │ ├── alerts.ts │ │ ├── analytics.ts │ │ ├── audit-logs.ts │ │ ├── charts.ts │ │ ├── dashboards.ts │ │ ├── data-warehouse.ts │ │ ├── evals.ts │ │ ├── evaluators.ts │ │ ├── external-users.ts │ │ ├── index.ts │ │ ├── jobs.ts │ │ ├── models.ts │ │ ├── prompts.ts │ │ ├── provider-configs.ts │ │ ├── providers.ts │ │ ├── users.ts │ │ └── views.ts │ │ ├── datatable.tsx │ │ ├── enrichment.tsx │ │ ├── errors.tsx │ │ ├── evaluators.ts │ │ ├── features.ts │ │ ├── fetcher.ts │ │ ├── format-validators.ts │ │ ├── format.ts │ │ ├── hooks.ts │ │ ├── pricing.ts │ │ ├── promptsHooks.ts │ │ └── theme.tsx ├── python-sdk │ ├── .gitignore │ ├── README.md │ ├── examples │ │ ├── anthropic │ │ │ ├── async-streaming.py │ │ │ ├── async.py │ │ │ ├── basic.py │ │ │ ├── streaming.py │ │ │ └── tool-call.py │ │ ├── azure-openai │ │ │ ├── async-stream.py │ │ │ ├── async.py │ │ │ ├── basic.py │ │ │ └── stream.py │ │ ├── functions.py │ │ ├── ibm │ │ │ ├── async.py │ │ │ ├── basic.py │ │ │ ├── stream.py │ │ │ ├── tools.py │ │ │ ├── tools2.py │ │ │ └── tools_stream.py │ │ ├── langchain │ │ │ ├── v0.0.1 │ │ │ │ ├── langchain-rag.py │ │ │ │ └── multiple_cb.py │ │ │ └── v0.0.3 │ │ │ │ └── message-reconciliation.py │ │ ├── openai │ │ │ ├── audio.py │ │ │ ├── basic.py │ │ │ └── tools.py │ │ ├── templates.py │ │ └── threads.py │ ├── lunary │ │ ├── __init__.py │ │ ├── agent.py │ │ ├── anthropic_utils.py │ │ ├── config.py │ │ ├── consumer.py │ │ ├── event_queue.py │ │ ├── exceptions.py │ │ ├── ibm_utils.py │ │ ├── openai_utils.py │ │ ├── parent.py │ │ ├── parsers.py │ │ ├── project.py │ │ ├── py.typed │ │ ├── run_manager.py │ │ ├── tags.py │ │ ├── thread.py │ │ ├── users.py │ │ └── utils.py │ ├── poetry.lock │ ├── pyproject.toml │ └── tests │ │ └── __init__.py ├── shared │ ├── .gitignore │ ├── README.md │ ├── access-control │ │ ├── index.ts │ │ └── roles.ts │ ├── audit-logs │ │ └── index.ts │ ├── checks │ │ ├── index.ts │ │ ├── params.ts │ │ ├── serialize.ts │ │ └── types.ts │ ├── dashboards │ │ └── index.ts │ ├── enrichers │ │ └── index.ts │ ├── index.ts │ ├── models.ts │ ├── package.json │ ├── providers │ │ └── index.ts │ ├── schemas │ │ ├── completion.ts │ │ ├── evaluation.ts │ │ ├── index.ts │ │ ├── messages.ts │ │ ├── old-openai.ts │ │ ├── openai.ts │ │ ├── project.ts │ │ ├── prompt.ts │ │ ├── run.ts │ │ └── user.ts │ ├── tsconfig.json │ └── utils │ │ └── date.ts └── tokenizer │ ├── external.ts │ ├── index.ts │ ├── models.ts │ ├── package.json │ ├── tokenizer.ts │ └── types.ts ├── patches └── postgres@3.4.5.patch └── tsconfig.json /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .dockerignore 3 | **.vscode 4 | *npm-debug.log 5 | **.Dockerfile 6 | **.env 7 | **.gitignore 8 | **README.md 9 | **LICENSE 10 | **.swp -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lunary-ai/lunary/27c09c35f982122a2d0b316a9cbbb97bd72711fe/.eslintrc.js -------------------------------------------------------------------------------- /.github/workflows/auto-approve-pr.yml: -------------------------------------------------------------------------------- 1 | name: Auto-approve PRs 2 | on: 3 | pull_request: 4 | types: [opened, reopened, synchronize] 5 | jobs: 6 | auto-approve: 7 | runs-on: ubuntu-latest 8 | if: github.event.pull_request.user.login == 'hughcrt' || github.event.pull_request.user.login == 'vincelwt' 9 | steps: 10 | - name: Checkout code 11 | uses: actions/checkout@v3 12 | - name: Auto-approve PR 13 | continue-on-error: true 14 | uses: hmarr/auto-approve-action@v3 15 | with: 16 | github-token: ${{ secrets.LUNARY_BOT_GH_TOKEN }} 17 | - name: Always succeed 18 | run: exit 0 19 | -------------------------------------------------------------------------------- /.github/workflows/auto-create-pr.yml: -------------------------------------------------------------------------------- 1 | name: Auto Create Pull Request 2 | 3 | on: create 4 | 5 | jobs: 6 | create-pull-request: 7 | if: github.event.ref_type == 'branch' 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Check out code 11 | uses: actions/checkout@v4 12 | with: 13 | fetch-depth: 0 14 | 15 | - name: Create Pull Request 16 | run: | 17 | BRANCH_NAME="${{ github.ref_name }}" 18 | 19 | if [[ $BRANCH_NAME == *"/"* ]]; then 20 | IFS="/" read -ra PARTS <<< "$BRANCH_NAME" 21 | PR_TITLE="${PARTS[0]}: ${PARTS[1]}" 22 | PR_TITLE="${PR_TITLE//-/ }" 23 | else 24 | PR_TITLE="${BRANCH_NAME//-/ }" 25 | fi 26 | 27 | gh pr create \ 28 | -B main \ 29 | -H "$BRANCH_NAME" \ 30 | --title "$PR_TITLE" \ 31 | --body "" 32 | env: 33 | GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} 34 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "ops"] 2 | path = ops 3 | url = git@github.com:lunary-ai/ops.git 4 | branch = main 5 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Development environment setup 2 | 3 | 1. Fork and clone the repository 4 | 2. Setup a PostgreSQL instance (version 15 minimum) 5 | 3. Copy the content of `packages/backend/.env.example` to `packages/backend/.env` and fill the missing values 6 | 4. Copy the content of `packages/frontend/.env.example` to `packages/backend/.env` 7 | 5. Run `bun install` 8 | 6. Run `bun run migrate:db` 9 | 7. Run `bun run dev` 10 | 11 | You can now open the dashboard at `http://localhost:8080`. When using our JS or Python SDK, you need to set the environment variable `LUNARY_API_URL` to `http://localhost:3333`. You can use `LUNARY_VERBOSE=True` to see all the event sent by the SDK 12 | 13 | ## Contributing Guidelines 14 | 15 | We welcome contributions to this project! 16 | 17 | When contributing, please follow these guidelines: 18 | 19 | - Before starting work on a new feature or bug fix, open an issue to discuss the proposed changes. This allows for coordination and avoids duplication of effort. 20 | - Fork the repository and create a new branch for your changes. Use a descriptive branch name that reflects the purpose of your changes. 21 | - Write clear, concise commit messages that describe the purpose of each commit. 22 | - Make sure to update any relevant documentation, including README files and code comments. 23 | - Make sure all tests pass before submitting a pull request. 24 | - When submitting a pull request, provide a detailed description of your changes and reference any related issues. 25 | - Be responsive to feedback and be willing to make changes to your pull request if requested. 26 | 27 | Thank you for your contributions! 28 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | | Version | Supported | 6 | | ------- | ------------------ | 7 | | 1.9.x | :white_check_mark: | 8 | | < 1.9.0 | :x: | 9 | 10 | ## Reporting a Vulnerability 11 | 12 | Email: hugh@lunary.ai 13 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lunary", 3 | "private": "true", 4 | "scripts": { 5 | "start": "bun --filter './packages/frontend' --filter './packages/backend' start", 6 | "start:backend": "bun --filter './packages/backend' start", 7 | "start:frontend": "bun --filter './packages/frontend' start", 8 | "start:tokenizer": "bun --filter './packages/tokenizer' start", 9 | "build:frontend": "bun --filter './packages/frontend' build", 10 | "migrate:db": "bun --filter './packages/backend' migrate:db", 11 | "dev": "bun --elide-lines 0 --filter '*' dev ", 12 | "test": "bun --filter './packages/e2e' test", 13 | "clean": "rm -rf bun.lock & rm -rf node_modules && rm -rf packages/frontend/node_modules && rm -rf packages/backend/node_modules && rm -rf packages/frontend/.next && rm -rf packages/e2e/node_modules" 14 | }, 15 | "workspaces": [ 16 | "packages/*" 17 | ], 18 | "devDependencies": { 19 | "prettier": "^3.5.3" 20 | }, 21 | "patchedDependencies": { 22 | "postgres@3.4.5": "patches/postgres@3.4.5.patch" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /packages/backend/.env.example: -------------------------------------------------------------------------------- 1 | DATABASE_URL="postgresql://postgres:password@your-host:5432/postgres" 2 | JWT_SECRET=yoursupersecret 3 | APP_URL=http://localhost:8080 4 | API_URL=http://localhost:3333 5 | 6 | # Optional (for the playground and evaluators) 7 | OPENAI_API_KEY=sk-... 8 | OPENROUTER_API_KEY=sk-... 9 | PALM_API_KEY=AI... 10 | -------------------------------------------------------------------------------- /packages/backend/src/access-control/authorization.ts: -------------------------------------------------------------------------------- 1 | import { Next } from "koa"; 2 | import Context from "../utils/koa"; 3 | import { roles } from "./roles"; 4 | 5 | type PermissionType = 6 | | "create" 7 | | "read" 8 | | "update" 9 | | "delete" 10 | | "list" 11 | | "export"; 12 | 13 | // TODO: change the whole thing. A user as a field projectRoles, so it's better for modularity later 14 | // TODO: attach to user its projectRoles 15 | export function authorize(resource: string, permission: PermissionType) { 16 | return async (ctx: Context, next: Next) => { 17 | const projectId = ctx.params.projectId; 18 | const projectRole = ctx.state.user.projectRoles.find( 19 | (pr) => pr.projectId === projectId, 20 | ); 21 | 22 | if (!projectRole) { 23 | ctx.throw(403, "Forbidden: No access to the project"); 24 | } 25 | 26 | const hasPermission = projectRole.roles.some((role) => { 27 | const rolePermissions = roles[role]?.permissions; 28 | return rolePermissions && rolePermissions[resource]?.[permission]; 29 | }); 30 | 31 | if (!hasPermission) { 32 | ctx.throw(403, "Forbidden: Insufficient permissions"); 33 | } 34 | 35 | await next(); 36 | }; 37 | } 38 | -------------------------------------------------------------------------------- /packages/backend/src/api/v1/audit-logs/utils.ts: -------------------------------------------------------------------------------- 1 | import sql from "@/src/utils/db"; 2 | import { clearUndefined } from "@/src/utils/ingest"; 3 | import Context from "@/src/utils/koa"; 4 | import { AuditLogAction, AuditLogResourceType, Project, User } from "shared"; 5 | 6 | export async function recordAuditLog( 7 | resourceType: T, 8 | action: AuditLogAction, 9 | ctx: Context, // TODO: attach ipAddress and userAgent to ctx.state instead (+ entier user object), and pass the values directly to the function instead of using ctx 10 | resourceId?: string, 11 | ) { 12 | const { userId, orgId, projectId } = ctx.state; 13 | const ipAddress = ctx.request.ip; 14 | const userAgent = ctx.request.headers["user-agent"] || ""; 15 | 16 | const [user] = await sql`select * from account where id = ${userId}`; 17 | const [project] = await sql< 18 | Project[] 19 | >`select * from project where id = ${projectId}`; 20 | 21 | await sql` 22 | insert into audit_log 23 | ${sql( 24 | clearUndefined({ 25 | orgId, 26 | resourceType, 27 | resourceId, 28 | action, 29 | userId, 30 | userName: user?.name, 31 | userEmail: user?.email, 32 | projectId: project?.id, 33 | projectName: project?.name, 34 | ipAddress, 35 | userAgent, 36 | }), 37 | )} 38 | `; 39 | } 40 | -------------------------------------------------------------------------------- /packages/backend/src/api/v1/data-warehouse/index.ts: -------------------------------------------------------------------------------- 1 | import Context from "@/src/utils/koa"; 2 | import Router from "koa-router"; 3 | import { z } from "zod"; 4 | import { createNewDatastream } from "./utils"; 5 | import sql from "@/src/utils/db"; 6 | import config from "@/src/utils/config"; 7 | 8 | const dataWarehouse = new Router({ 9 | prefix: "/data-warehouse", 10 | }); 11 | 12 | dataWarehouse.get("/bigquery", async (ctx: Context) => { 13 | const { projectId } = ctx.state; 14 | const [connector] = 15 | await sql`select * from _data_warehouse_connector where project_id = ${projectId}`; 16 | 17 | ctx.body = connector; 18 | }); 19 | 20 | dataWarehouse.post("/bigquery", async (ctx: Context) => { 21 | const bodySchema = z.object({ 22 | apiKey: z.string().transform((apiKey) => JSON.parse(apiKey)), 23 | }); 24 | const { apiKey } = bodySchema.parse(ctx.request.body); 25 | const { userId } = ctx.state; 26 | 27 | const [user] = await sql`select * from account where id = ${userId}`; 28 | 29 | if (user.role !== "owner") { 30 | ctx.throw(403, "Forbidden"); 31 | } 32 | 33 | if (config.DATA_WAREHOUSE_EXPORTS_ALLOWED) { 34 | await createNewDatastream(apiKey, process.env.DATABASE_URL!, ctx); 35 | } else { 36 | ctx.throw(403, "Forbidden"); 37 | } 38 | 39 | ctx.body = {}; 40 | }); 41 | 42 | dataWarehouse.patch("/bigquery", async (ctx: Context) => {}); 43 | 44 | export default dataWarehouse; 45 | -------------------------------------------------------------------------------- /packages/backend/src/api/v1/openapi.ts: -------------------------------------------------------------------------------- 1 | import swaggerJsdoc from "swagger-jsdoc"; 2 | import Router from "koa-router"; 3 | 4 | const options: swaggerJsdoc.Options = { 5 | definition: { 6 | openapi: "3.0.0", 7 | info: { 8 | title: "Lunary API", 9 | version: "1.0.0", 10 | }, 11 | servers: [ 12 | { 13 | url: "https://api.lunary.ai", 14 | }, 15 | ], 16 | components: { 17 | securitySchemes: { 18 | BearerAuth: { 19 | type: "http", 20 | scheme: "bearer", 21 | }, 22 | }, 23 | }, 24 | }, 25 | apis: ["./src/api/v1/**/*.ts"], 26 | }; 27 | 28 | const openapiSpecification = swaggerJsdoc(options); 29 | 30 | const router = new Router(); 31 | 32 | router.get("/openapi", async (ctx) => { 33 | ctx.body = openapiSpecification; 34 | }); 35 | 36 | export default router; 37 | -------------------------------------------------------------------------------- /packages/backend/src/api/v1/projects/utils.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lunary-ai/lunary/27c09c35f982122a2d0b316a9cbbb97bd72711fe/packages/backend/src/api/v1/projects/utils.ts -------------------------------------------------------------------------------- /packages/backend/src/api/v1/prompts.ts: -------------------------------------------------------------------------------- 1 | import sql from "@/src/utils/db"; 2 | import Context from "@/src/utils/koa"; 3 | import { unCamelObject } from "@/src/utils/misc"; 4 | import Router from "koa-router"; 5 | import { Prompt, PromptVersion } from "shared/schemas/prompt"; 6 | import { z } from "zod"; 7 | 8 | const prompts = new Router({ 9 | prefix: "/prompts", 10 | }); 11 | 12 | prompts.get("/", async (ctx: Context) => { 13 | const { projectId } = ctx.state; 14 | 15 | const prompts = await sql< 16 | Prompt[] 17 | >`select * from template where project_id = ${projectId} order by created_at desc`; 18 | 19 | ctx.body = prompts; 20 | }); 21 | 22 | prompts.get("/:id/versions", async (ctx: Context) => { 23 | const { id: templateId } = z 24 | .object({ id: z.coerce.number() }) 25 | .parse(ctx.params); 26 | 27 | const promptVersions = await sql< 28 | PromptVersion[] 29 | >`select * from template_version where template_id = ${templateId} order by created_at desc`; 30 | 31 | let i = promptVersions.length; 32 | for (const promptVersion of promptVersions) { 33 | promptVersion.extra = unCamelObject(promptVersion.extra); 34 | promptVersion.version = i--; 35 | } 36 | 37 | ctx.body = promptVersions; 38 | }); 39 | 40 | export default prompts; 41 | -------------------------------------------------------------------------------- /packages/backend/src/api/v1/runs/queries.ts: -------------------------------------------------------------------------------- 1 | import sql from "@/src/utils/db"; 2 | 3 | export async function getRelatedRuns(runId: string, projectId: string) { 4 | const relatedRuns = await sql` 5 | with recursive related_runs as ( 6 | select 7 | r1.* 8 | from 9 | run r1 10 | where 11 | r1.id = ${runId} 12 | and project_id = ${projectId} 13 | 14 | union all 15 | 16 | select 17 | r2.* 18 | from 19 | run r2 20 | inner join related_runs rr on rr.id = r2.parent_run_id 21 | ) 22 | select 23 | rr.created_at, 24 | rr.tags, 25 | rr.project_id, 26 | rr.id, 27 | rr.status, 28 | rr.name, 29 | rr.ended_at, 30 | rr.error, 31 | rr.input, 32 | rr.output, 33 | rr.params, 34 | rr.type, 35 | rr.parent_run_id, 36 | rr.completion_tokens, 37 | rr.prompt_tokens, 38 | coalesce(rr.cost, 0) as cost, 39 | rr.feedback, 40 | rr.metadata 41 | from 42 | related_runs rr; 43 | `; 44 | 45 | return relatedRuns; 46 | } 47 | 48 | export async function getMessages(threadId: string, projectId: string) { 49 | const relatedRuns = await getRelatedRuns(threadId, projectId); 50 | const filteredRuns = relatedRuns.filter((_, i) => i !== 0); 51 | 52 | let messages = []; 53 | for (const run of filteredRuns) { 54 | if (Array.isArray(run.input)) { 55 | messages.push(...run.input); 56 | } 57 | if (Array.isArray(run.output)) { 58 | messages.push(...run.output); 59 | } 60 | } 61 | return messages; 62 | } 63 | -------------------------------------------------------------------------------- /packages/backend/src/api/webhooks/index.ts: -------------------------------------------------------------------------------- 1 | import Router from "koa-router"; 2 | import stripe from "./stripe"; 3 | 4 | const webhooks = new Router({ 5 | prefix: "/webhooks", 6 | }); 7 | 8 | webhooks.use(stripe.routes()); 9 | 10 | export default webhooks; 11 | -------------------------------------------------------------------------------- /packages/backend/src/checks/ai/assert.ts: -------------------------------------------------------------------------------- 1 | import openai from "@/src/utils/openai"; 2 | import { zodTextFormat } from "openai/helpers/zod"; 3 | import { z } from "zod"; 4 | 5 | export default async function aiAssert(sentence: string, assertion: string) { 6 | const assertSchema = z.object({ 7 | passed: z.boolean(), 8 | reason: z.string(), 9 | }); 10 | 11 | const completion = await openai!.responses.parse({ 12 | model: "gpt-4.1", 13 | instructions: ` 14 | You help evaluate if a given response from an AI matches a given assertion. 15 | Return a JSON object with: 16 | passed → true if the assertion is fully satisfied, false otherwise 17 | reason → brief explanation 18 | `, 19 | input: ` 20 | AI Response: 21 | \`${sentence}\` 22 | 23 | Assertion: 24 | \`${assertion}\` 25 | `, 26 | text: { format: zodTextFormat(assertSchema, "assert") }, 27 | }); 28 | 29 | if (completion.output_parsed === null) { 30 | throw new Error("Failed to parse completion"); 31 | } 32 | 33 | const { passed, reason } = completion.output_parsed; 34 | 35 | return { passed, reason }; 36 | } 37 | -------------------------------------------------------------------------------- /packages/backend/src/checks/ai/sentiment.ts: -------------------------------------------------------------------------------- 1 | import { pipeline } from "@xenova/transformers"; 2 | 3 | let nerPipeline: any = null; 4 | let loading = false; 5 | 6 | const THRESHOLD = 0.5; 7 | 8 | type Output = { 9 | label: string; 10 | score: number; 11 | }[]; 12 | 13 | async function aiSentiment(sentence?: string): Promise { 14 | if (!sentence || sentence.length < 10) return 0.5; // neutral 15 | 16 | if (!nerPipeline) { 17 | // this prevents multiple loading of the pipeline simultaneously which causes extreme lag 18 | if (loading) { 19 | await new Promise((resolve) => setTimeout(resolve, 1000)); 20 | return aiSentiment(sentence); 21 | } 22 | 23 | loading = true; 24 | nerPipeline = await pipeline( 25 | "sentiment-analysis", 26 | "Xenova/bert-base-multilingual-uncased-sentiment", 27 | ); 28 | loading = false; 29 | } 30 | 31 | const output: Output = await nerPipeline(sentence); 32 | // [ { label: '1 star', score: 0.6303076148033142 } ] 33 | 34 | const sorted = output.sort((a, b) => b.score - a.score); 35 | 36 | if (sorted[0].score < THRESHOLD) return 0.5; // neutral 37 | 38 | const sentiment = parseInt(output[0].label); 39 | 40 | const score = +(sentiment / 5).toFixed(2); 41 | 42 | return score; 43 | } 44 | 45 | export default aiSentiment; 46 | -------------------------------------------------------------------------------- /packages/backend/src/checks/utils.ts: -------------------------------------------------------------------------------- 1 | import { CheckLogic, LogicElement } from "shared"; 2 | import sql from "../utils/db"; 3 | import CHECK_RUNNERS from "."; 4 | 5 | const and = (arr: any = []) => 6 | arr.reduce((acc: any, x: any) => sql`${acc} AND ${x}`); 7 | const or = (arr: any = []) => 8 | arr.reduce((acc: any, x: any) => sql`(${acc} OR ${x})`); 9 | 10 | export function convertChecksToSQL(filtersData: CheckLogic): any { 11 | const logicToSql = (logicElement: LogicElement): any => { 12 | if (Array.isArray(logicElement)) { 13 | const [logicOperator, ...elements] = logicElement; 14 | 15 | if (logicOperator === "AND") { 16 | return and(elements.map(logicToSql)); 17 | } else if (logicOperator === "OR") { 18 | return or(elements.map(logicToSql)); 19 | } 20 | } else { 21 | const runner = CHECK_RUNNERS.find((f) => f.id === logicElement.id); 22 | 23 | if (!runner || !runner.sql) { 24 | console.warn( 25 | `No SQL method defined for filter with id ${logicElement.id}`, 26 | ); 27 | return sql``; 28 | } 29 | return runner.sql(logicElement.params); 30 | } 31 | }; 32 | const sqlChecks = sql`(${logicToSql(filtersData)})`; 33 | return sqlChecks; 34 | } 35 | -------------------------------------------------------------------------------- /packages/backend/src/emails/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./templates"; 2 | export * from "./sender"; 3 | export * from "./utils"; 4 | -------------------------------------------------------------------------------- /packages/backend/src/emails/utils.ts: -------------------------------------------------------------------------------- 1 | import { signJWT } from "@/src/api/v1/auth/utils"; 2 | import { sendEmail } from "./sender"; 3 | import { CONFIRM_EMAIL } from "./templates"; 4 | import config from "../utils/config"; 5 | 6 | function sanitizeName(name: string): string { 7 | return name.replace(/\s+/g, " ").trim(); 8 | } 9 | 10 | export function extractFirstName(name: string): string { 11 | if (!name) return "there"; 12 | const sanitizedName = sanitizeName(name); 13 | return sanitizedName.split(" ")[0]; 14 | } 15 | 16 | export async function sendVerifyEmail(email: string, name: string = "") { 17 | const token = await signJWT({ email }); 18 | 19 | const confirmLink = `${process.env.APP_URL}/verify-email?token=${token}`; 20 | 21 | if (config.IS_CLOUD) { 22 | await sendEmail(CONFIRM_EMAIL(email, name, confirmLink)); 23 | console.info("[EMAIL] Sent verification email to", email); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /packages/backend/src/evaluators/bias.ts: -------------------------------------------------------------------------------- 1 | import { callML } from "@/src/utils/ml"; 2 | import { Run } from "shared"; 3 | import { z } from "zod"; 4 | import { parseMessages } from "./utils"; 5 | import { cat } from "@xenova/transformers"; 6 | 7 | const biasSchema = z.array( 8 | z.union([z.null(), z.object({ reason: z.string() })]), 9 | ); 10 | type Toxicity = z.infer; 11 | 12 | export interface ToxicityEvaluation { 13 | input: Toxicity; 14 | output: Toxicity; 15 | } 16 | 17 | async function getToxicity(texts: string[]): Promise { 18 | const raw = await callML("toxicity", { texts }); 19 | return toxicitySchema.parse(raw); 20 | } 21 | 22 | export async function evaluate(run: Run): Promise { 23 | const inputTexts = parseMessages(run.input); 24 | const outputTexts = parseMessages(run.output); 25 | 26 | try { 27 | const [input, output] = await Promise.all([ 28 | inputTexts ? getToxicity(inputTexts) : Promise.resolve([null]), 29 | outputTexts ? getToxicity(outputTexts) : Promise.resolve([null]), 30 | ]); 31 | return { input, output }; 32 | } catch (error) { 33 | console.error(error); 34 | return null; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /packages/backend/src/evaluators/bleu.ts: -------------------------------------------------------------------------------- 1 | import { Run } from "shared"; 2 | import { parseMessages } from "./utils"; 3 | 4 | /** 5 | * Simple BLEU evaluator based on unigram precision. 6 | * Params: threshold (0-1) 7 | */ 8 | export async function evaluate( 9 | { input, output }: { input: unknown; output: unknown }, 10 | params: { threshold: number }, 11 | ) { 12 | const refs = parseMessages(input) || []; 13 | const hyps = parseMessages(output) || []; 14 | const score = computeBleu(refs, hyps); 15 | return { score, passed: score >= params.threshold }; 16 | } 17 | 18 | function computeBleu(refs: string[], hyps: string[]): number { 19 | const refTokens = refs.join(" ").split(/\s+/).filter(Boolean); 20 | const hypTokens = hyps.join(" ").split(/\s+/).filter(Boolean); 21 | if (hypTokens.length === 0) return 0; 22 | const refSet = new Set(refTokens); 23 | const matches = hypTokens.filter((w) => refSet.has(w)).length; 24 | return matches / hypTokens.length; 25 | } 26 | -------------------------------------------------------------------------------- /packages/backend/src/evaluators/cosine.ts: -------------------------------------------------------------------------------- 1 | import { Run } from "shared"; 2 | import { parseMessages } from "./utils"; 3 | 4 | /** 5 | * Simple Cosine Similarity evaluator using term-frequency vectors. 6 | * Params: threshold (0-1) 7 | */ 8 | export async function evaluate( 9 | { input, output }: { input: unknown; output: unknown }, 10 | params: { threshold: number }, 11 | ) { 12 | const refs = parseMessages(input) || []; 13 | const hyps = parseMessages(output) || []; 14 | const score = computeCosine(refs.join(" "), hyps.join(" ")); 15 | return { score, passed: score >= params.threshold }; 16 | } 17 | 18 | function computeCosine(refText: string, hypText: string): number { 19 | const refTokens = refText.split(/\s+/).filter(Boolean); 20 | const hypTokens = hypText.split(/\s+/).filter(Boolean); 21 | const freqRef: Record = {}; 22 | const freqHyp: Record = {}; 23 | refTokens.forEach((w) => (freqRef[w] = (freqRef[w] || 0) + 1)); 24 | hypTokens.forEach((w) => (freqHyp[w] = (freqHyp[w] || 0) + 1)); 25 | const allTokens = new Set([...refTokens, ...hypTokens]); 26 | let dot = 0, 27 | magRef = 0, 28 | magHyp = 0; 29 | allTokens.forEach((token) => { 30 | const x = freqRef[token] || 0; 31 | const y = freqHyp[token] || 0; 32 | dot += x * y; 33 | magRef += x * x; 34 | magHyp += y * y; 35 | }); 36 | if (magRef === 0 || magHyp === 0) return 0; 37 | return dot / (Math.sqrt(magRef) * Math.sqrt(magHyp)); 38 | } 39 | -------------------------------------------------------------------------------- /packages/backend/src/evaluators/fuzzy.ts: -------------------------------------------------------------------------------- 1 | import { Run } from "shared"; 2 | import { parseMessages } from "./utils"; 3 | 4 | /** 5 | * Simple Fuzzy Match evaluator using Levenshtein distance ratio. 6 | * Params: threshold (0-1) 7 | */ 8 | export async function evaluate( 9 | { input, output }: { input: unknown; output: unknown }, 10 | params: { threshold: number }, 11 | ) { 12 | const refs = parseMessages(input) || []; 13 | const hyps = parseMessages(output) || []; 14 | const score = computeFuzzy(refs.join(" "), hyps.join(" ")); 15 | return { score, passed: score >= params.threshold }; 16 | } 17 | 18 | function computeFuzzy(refText: string, hypText: string): number { 19 | const a = refText; 20 | const b = hypText; 21 | const lenA = a.length; 22 | const lenB = b.length; 23 | if (lenA === 0 && lenB === 0) return 1; 24 | if (lenA === 0 || lenB === 0) return 0; 25 | // Levenshtein distance 26 | const dp: number[][] = Array(lenA + 1) 27 | .fill(0) 28 | .map(() => Array(lenB + 1).fill(0)); 29 | for (let i = 0; i <= lenA; i++) dp[i][0] = i; 30 | for (let j = 0; j <= lenB; j++) dp[0][j] = j; 31 | for (let i = 1; i <= lenA; i++) { 32 | for (let j = 1; j <= lenB; j++) { 33 | const cost = a[i - 1] === b[j - 1] ? 0 : 1; 34 | dp[i][j] = Math.min( 35 | dp[i - 1][j] + 1, 36 | dp[i][j - 1] + 1, 37 | dp[i - 1][j - 1] + cost, 38 | ); 39 | } 40 | } 41 | const dist = dp[lenA][lenB]; 42 | const maxLen = Math.max(lenA, lenB); 43 | return (maxLen - dist) / maxLen; 44 | } 45 | -------------------------------------------------------------------------------- /packages/backend/src/evaluators/gleu.ts: -------------------------------------------------------------------------------- 1 | import { Run } from "shared"; 2 | import { parseMessages } from "./utils"; 3 | 4 | /** 5 | * Simple GLEU evaluator: geometric mean of unigram precision and recall. 6 | * Params: threshold (0-1) 7 | */ 8 | export async function evaluate( 9 | { input, output }: { input: unknown; output: unknown }, 10 | params: { threshold: number }, 11 | ) { 12 | const refs = parseMessages(input) || []; 13 | const hyps = parseMessages(output) || []; 14 | const score = computeGleu(refs, hyps); 15 | return { score, passed: score >= params.threshold }; 16 | } 17 | 18 | function computeGleu(refs: string[], hyps: string[]): number { 19 | const refTokens = refs.join(" ").split(/\s+/).filter(Boolean); 20 | const hypTokens = hyps.join(" ").split(/\s+/).filter(Boolean); 21 | if (hypTokens.length === 0 || refTokens.length === 0) return 0; 22 | const refSet = new Set(refTokens); 23 | const matches = hypTokens.filter((w) => refSet.has(w)).length; 24 | const precision = matches / hypTokens.length; 25 | const recall = matches / refTokens.length; 26 | return Math.sqrt(precision * recall); 27 | } 28 | -------------------------------------------------------------------------------- /packages/backend/src/evaluators/guidelines.ts: -------------------------------------------------------------------------------- 1 | import { Run } from "shared"; 2 | import { isOpenAIMessage, lastMsg } from "../checks"; 3 | import openai from "@/src/utils/openai"; 4 | import { z } from "zod"; 5 | import { zodTextFormat } from "openai/helpers/zod"; 6 | 7 | export async function evaluate(run: Run) { 8 | if (!Array.isArray(run.input) || !run.input.every(isOpenAIMessage)) return ""; 9 | if (run.input[0].role !== "system") return ""; 10 | 11 | const systemGuidelines = run.input[0].content; 12 | const answer = lastMsg(run.output); 13 | if (!answer) return ""; 14 | 15 | const evalSchema = z.object({ 16 | result: z.boolean(), 17 | reason: z.string(), 18 | }); 19 | 20 | const completion = await openai!.responses.parse({ 21 | model: "gpt-4.1", 22 | instructions: ` 23 | You are judging whether an assistant answer fully complies with the provided GUIDELINES. 24 | 25 | Reply **only** with JSON matching the schema: 26 | { 27 | "result": boolean, // true if every guideline is followed 28 | "reason": string // short justification 29 | } 30 | `, 31 | input: ` 32 | GUIDELINES: 33 | ${systemGuidelines} 34 | 35 | ANSWER: 36 | ${answer} 37 | `, 38 | text: { 39 | format: zodTextFormat(evalSchema, "evaluation"), 40 | }, 41 | }); 42 | 43 | if (completion.output_parsed === null) 44 | throw new Error("Failed to parse completion"); 45 | 46 | const { result, reason } = completion.output_parsed; 47 | 48 | return { 49 | result, 50 | reason, 51 | }; 52 | } 53 | -------------------------------------------------------------------------------- /packages/backend/src/evaluators/index.ts: -------------------------------------------------------------------------------- 1 | import * as pii from "./pii"; 2 | import * as language from "./language"; 3 | import * as llm from "./llm"; 4 | import * as topics from "./topics"; 5 | import * as toxicity from "./toxicity"; 6 | import * as sentiment from "./sentiment"; 7 | import * as guidelines from "./guidelines"; 8 | import * as replies from "./replies"; 9 | import * as bias from "./bias"; 10 | import * as bleu from "./bleu"; 11 | import * as gleu from "./gleu"; 12 | import * as rouge from "./rouge"; 13 | import * as cosine from "./cosine"; 14 | import * as fuzzy from "./fuzzy"; 15 | 16 | const evaluators = { 17 | pii, 18 | language, 19 | llm, 20 | topics, 21 | toxicity, 22 | sentiment, 23 | guidelines, 24 | replies, 25 | bias, 26 | bleu, 27 | gleu, 28 | rouge, 29 | cosine, 30 | fuzzy, 31 | }; 32 | 33 | export default evaluators; 34 | -------------------------------------------------------------------------------- /packages/backend/src/evaluators/llm.ts: -------------------------------------------------------------------------------- 1 | import { callML } from "@/src/utils/ml"; 2 | import { Run } from "shared"; 3 | 4 | interface Params { 5 | prompt: string; 6 | model: string; 7 | } 8 | 9 | export async function evaluate(run: Run, params: Params) { 10 | try { 11 | const { prompt } = params; 12 | const result = await callML("assertion", { 13 | input: run.input, 14 | output: run.output, 15 | instructions: prompt, 16 | // model, 17 | }); 18 | return result as { pass: boolean; reason: string }; 19 | } catch (error) { 20 | console.error(error); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /packages/backend/src/evaluators/old-assert.ts: -------------------------------------------------------------------------------- 1 | import openai from "@/src/utils/openai"; 2 | import { z } from "zod"; 3 | import { zodTextFormat } from "openai/helpers/zod"; 4 | import { Run } from "shared"; 5 | import { lastMsg } from "../checks"; 6 | 7 | interface AssertParams { 8 | conditions: string[]; 9 | } 10 | 11 | export async function evaluate(run: Run, params: AssertParams) { 12 | const { conditions } = params; 13 | const conditionList = conditions.map((c) => `- ${c}`).join("\n"); 14 | 15 | const assertSchema = z.object({ 16 | result: z.boolean(), 17 | reason: z.string(), 18 | }); 19 | 20 | const completion = await openai!.responses.parse({ 21 | model: "gpt-4.1", 22 | instructions: ` 23 | You help evaluate if a given interaction from an AI matches one or more assertions. 24 | Return a JSON object with: 25 | result → true if all assertions are satisfied, false otherwise 26 | reason → brief explanation 27 | `, 28 | input: ` 29 | User Input: 30 | \`${lastMsg(run.input)}\` 31 | 32 | AI Response/answer: 33 | \`${lastMsg(run.output)}\` 34 | 35 | Assertions: 36 | ${conditionList} 37 | `, 38 | text: { format: zodTextFormat(assertSchema, "assert") }, 39 | }); 40 | 41 | if (completion.output_parsed === null) throw new Error("Failed to parse"); 42 | 43 | const { result, reason } = completion.output_parsed; 44 | return { result, reason }; 45 | } 46 | -------------------------------------------------------------------------------- /packages/backend/src/evaluators/replies.ts: -------------------------------------------------------------------------------- 1 | import { Run } from "shared"; 2 | import { lastMsg } from "../checks"; 3 | import openai from "@/src/utils/openai"; 4 | import { z } from "zod"; 5 | import { zodTextFormat } from "openai/helpers/zod"; 6 | 7 | export async function evaluate(run: Run) { 8 | if (!run.input || !run.output) return null; 9 | 10 | const question = lastMsg(run.input); 11 | const answer = lastMsg(run.output); 12 | if (!question || !answer) return null; 13 | 14 | const evalSchema = z.object({ 15 | result: z.enum(["YES", "NO"]), 16 | reason: z.string().optional().default(""), 17 | }); 18 | 19 | const completion = await openai!.responses.parse({ 20 | model: "gpt-4.1", 21 | instructions: ` 22 | You judge whether the ANSWER actually addresses the QUESTION / instruction. 23 | 24 | • Return "YES" if it does. 25 | • Return "NO" if it does not. 26 | • If the prompt isn’t really a question (e.g.\ a statement), treat it as answered and return "YES". 27 | 28 | Reply *only* with JSON matching the schema. 29 | `, 30 | input: ` 31 | QUESTION: 32 | ${question} 33 | 34 | ANSWER: 35 | ${answer} 36 | `, 37 | text: { 38 | format: zodTextFormat(evalSchema, "evaluation"), 39 | }, 40 | }); 41 | 42 | if (completion.output_parsed === null) 43 | throw new Error("Failed to parse completion"); 44 | 45 | const { result, reason } = completion.output_parsed; 46 | 47 | return { 48 | result: result === "YES", 49 | reason, 50 | }; 51 | } 52 | -------------------------------------------------------------------------------- /packages/backend/src/evaluators/rouge.ts: -------------------------------------------------------------------------------- 1 | import { Run } from "shared"; 2 | import { parseMessages } from "./utils"; 3 | 4 | /** 5 | * Simple ROUGE evaluator: unigram recall. 6 | * Params: threshold (0-1) 7 | */ 8 | export async function evaluate( 9 | { input, output }: { input: unknown; output: unknown }, 10 | params: { threshold: number }, 11 | ) { 12 | const refs = parseMessages(input) || []; 13 | const hyps = parseMessages(output) || []; 14 | const score = computeRouge(refs, hyps); 15 | return { score, passed: score >= params.threshold }; 16 | } 17 | 18 | function computeRouge(refs: string[], hyps: string[]): number { 19 | const refTokens = refs.join(" ").split(/\s+/).filter(Boolean); 20 | const hypTokens = hyps.join(" ").split(/\s+/).filter(Boolean); 21 | if (refTokens.length === 0) return 0; 22 | const refSet = new Set(refTokens); 23 | const matches = hypTokens.filter((w) => refSet.has(w)).length; 24 | return matches / refTokens.length; 25 | } 26 | -------------------------------------------------------------------------------- /packages/backend/src/evaluators/string.ts: -------------------------------------------------------------------------------- 1 | import { Run } from "shared"; 2 | import { parseMessages } from "./utils"; 3 | 4 | /** 5 | * String Comparator evaluator: compares output text to a target string. 6 | * Params: comparator (equals|not_equals|contains|contains_ignore_case), target string 7 | */ 8 | export async function evaluate( 9 | { input, output }: { input: unknown; output: unknown }, 10 | params: { comparator: string; target: string }, 11 | ) { 12 | const hyps = parseMessages(output) || []; 13 | const text = hyps.join(" "); 14 | const { comparator, target } = params; 15 | let passed = false; 16 | 17 | switch (comparator) { 18 | case "equals": 19 | passed = text === target; 20 | break; 21 | case "not_equals": 22 | passed = text !== target; 23 | break; 24 | case "contains": 25 | passed = text.includes(target); 26 | break; 27 | case "contains_ignore_case": 28 | passed = text.toLowerCase().includes(target.toLowerCase()); 29 | break; 30 | default: 31 | passed = false; 32 | } 33 | 34 | return { passed }; 35 | } 36 | -------------------------------------------------------------------------------- /packages/backend/src/evaluators/utils/index.ts: -------------------------------------------------------------------------------- 1 | import { MessageSchema } from "shared/schemas/openai"; 2 | import { z } from "zod"; 3 | 4 | export function parseMessages(data: unknown): string[] | undefined { 5 | const parsed = z.array(MessageSchema).safeParse(data); 6 | if (!parsed.success) return; 7 | return parsed.data.map((m) => JSON.stringify(m)); 8 | } 9 | -------------------------------------------------------------------------------- /packages/backend/src/jobs/materialized-views.ts: -------------------------------------------------------------------------------- 1 | import sql from "../utils/db"; 2 | import { sleep } from "../utils/misc"; 3 | 4 | export async function startMaterializedViewRefreshJob() { 5 | // TODO: locks 6 | if (process.env.DISABLE_MATERIALIZED_VIEW_REFRESH) { 7 | return; 8 | } 9 | try { 10 | const views = ["metadata_cache"]; 11 | 12 | while (true) { 13 | for (const view of views) { 14 | await sql`refresh materialized view concurrently ${sql(view)};`.catch( 15 | (error) => { 16 | console.error(`Error refreshing materialized view: ${view}`); 17 | console.error(error); 18 | }, 19 | ); 20 | } 21 | 22 | await sleep(5 * 60 * 1000); 23 | } 24 | } catch (error) { 25 | console.error(error); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /packages/backend/src/realtime-evaluators.ts: -------------------------------------------------------------------------------- 1 | import runEvaluatorsJob from "./jobs/realtime-evaluators"; 2 | 3 | runEvaluatorsJob(); 4 | -------------------------------------------------------------------------------- /packages/backend/src/types/database.ts: -------------------------------------------------------------------------------- 1 | export interface Account { 2 | id: string; 3 | createdAt: Date; 4 | email: string | null; 5 | passwordHash: string | null; 6 | recoveryToken: string | null; 7 | name: string | null; 8 | orgId: string | null; 9 | role: "owner" | "admin" | "member" | "viewer" | "prompt_editor" | "billing"; 10 | verified: boolean; 11 | avatarUrl: string | null; 12 | lastLoginAt: Date | null; 13 | singleUseToken: string | null; 14 | } 15 | -------------------------------------------------------------------------------- /packages/backend/src/types/index.ts: -------------------------------------------------------------------------------- 1 | export * as Db from "./database"; 2 | -------------------------------------------------------------------------------- /packages/backend/src/utils/authorization.ts: -------------------------------------------------------------------------------- 1 | import { Next } from "koa"; 2 | import sql from "./db"; 3 | import Context from "./koa"; 4 | import { Action, ResourceName, hasAccess } from "shared"; 5 | 6 | export async function checkProjectAccess(projectId: string, userId: string) { 7 | const [{ exists: hasAccess }] = await sql` 8 | select exists ( 9 | select 1 10 | from account_project ap 11 | where ap.project_id = ${projectId} and ap.account_id = ${userId} 12 | ) 13 | `; 14 | return hasAccess; 15 | } 16 | 17 | export function checkAccess(resourceName: ResourceName, action: Action) { 18 | return async (ctx: Context, next: Next) => { 19 | if (ctx.state.privateKey) { 20 | // give all rights to private key 21 | await next(); 22 | return; 23 | } 24 | 25 | const [user] = 26 | await sql`select * from account where id = ${ctx.state.userId}`; 27 | 28 | const hasAccessToResource = hasAccess(user.role, resourceName, action); 29 | 30 | if (hasAccessToResource) { 31 | await next(); 32 | } else { 33 | ctx.status = 403; 34 | ctx.body = { 35 | error: "Forbidden", 36 | message: "You don't have access to this resource", 37 | }; 38 | } 39 | }; 40 | } 41 | -------------------------------------------------------------------------------- /packages/backend/src/utils/cache.ts: -------------------------------------------------------------------------------- 1 | import { Next } from "koa"; 2 | import { LRUCache } from "lru-cache"; 3 | import config from "./config"; 4 | import Context from "./koa"; 5 | 6 | const ONE_DAY = 1000 * 60 * 60 * 24; 7 | const ONE_MB = 1024 * 1024; 8 | const ONE_HUNDREd_MB = ONE_MB * 100; 9 | const TWO_GB = ONE_MB * 2000; 10 | 11 | 12 | const cache = new LRUCache({ 13 | maxSize: config.IS_SELF_HOSTED ? ONE_HUNDREd_MB : TWO_GB, 14 | ttl: ONE_DAY, 15 | sizeCalculation: (value, key) => 16 | Buffer.byteLength(key, "utf8") + 17 | Buffer.byteLength(JSON.stringify(value), "utf8"), 18 | }); 19 | 20 | export async function cacheAnalyticsMiddleware(ctx: Context, next: Next) { 21 | if (ctx.method !== "GET" || !ctx.path.startsWith("/v1/analytics")) { 22 | return await next(); 23 | } 24 | 25 | const key = `${ctx.path}?${ctx.querystring}::${ctx.state.projectId}`; 26 | 27 | if (cache.has(key)) { 28 | ctx.body = cache.get(key)!; 29 | return; 30 | } 31 | 32 | await next(); 33 | 34 | if (ctx.status === 200 && ctx.body !== undefined) { 35 | cache.set(key, ctx.body); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /packages/backend/src/utils/checks.ts: -------------------------------------------------------------------------------- 1 | import { CheckLogic, LogicElement } from "shared"; 2 | import sql from "./db"; 3 | import CHECK_RUNNERS from "../checks"; 4 | 5 | export const and = (arr: any = []) => 6 | arr.reduce((acc: any, x: any) => sql`${acc} AND ${x}`); 7 | export const or = (arr: any = []) => 8 | arr.reduce((acc: any, x: any) => sql`(${acc} OR ${x})`); 9 | 10 | // TODO: unit tests 11 | export function convertChecksToSQL(filtersData: CheckLogic): any { 12 | const logicToSql = (logicElement: LogicElement): any => { 13 | if (Array.isArray(logicElement)) { 14 | const [logicOperator, ...elements] = logicElement; 15 | 16 | if (logicOperator === "AND") { 17 | return and(elements.map(logicToSql)); 18 | } else if (logicOperator === "OR") { 19 | return or(elements.map(logicToSql)); 20 | } 21 | } else { 22 | const runner = CHECK_RUNNERS.find((f) => f.id === logicElement.id); 23 | 24 | if (!runner || !runner.sql) { 25 | console.warn( 26 | `No SQL method defined for filter with id ${logicElement.id}`, 27 | ); 28 | return sql``; 29 | } 30 | return runner.sql(logicElement.params); 31 | } 32 | }; 33 | const sqlChecks = sql`(${logicToSql(filtersData)})`; 34 | return sqlChecks; 35 | } 36 | -------------------------------------------------------------------------------- /packages/backend/src/utils/config.ts: -------------------------------------------------------------------------------- 1 | // TODO: use zod 2 | const IS_SELF_HOSTED = process.env.IS_SELF_HOSTED === "true" ? true : false; 3 | 4 | const config = { 5 | IS_SELF_HOSTED, 6 | IS_CLOUD: 7 | process.env.NEXT_PUBLIC_IS_SELF_HOSTED !== "true" && 8 | process.env.NODE_ENV === "production", 9 | SKIP_EMAIL_VERIFY: process.env.SKIP_EMAIL_VERIFY === "true" ? true : false, 10 | GENERIC_SENDER_ADDRESS: IS_SELF_HOSTED 11 | ? process.env.EMAIL_SENDER_ADDRESS 12 | : process.env.GENERIC_SENDER_ADDRESS, 13 | PERSONAL_SENDER_ADDRESS: process.env.PERSONAL_SENDER_ADDRESS, 14 | SMTP_HOST: process.env.SMTP_HOST, 15 | SMTP_PORT: Number.parseInt(process.env.SMTP_PORT || "465"), 16 | SMTP_USER: process.env.SMTP_USER, 17 | SMTP_PASSWORD: process.env.SMTP_PASSWORD, 18 | DATA_WAREHOUSE_EXPORTS_ALLOWED: process.env.ALLOW_DATA_WAREHOUSE_EXPORTS, // WARNING: this should only enabled for deployments with one organization, because all the database will be exported 19 | RECAPTCHA_SECRET_KEY: process.env.RECAPTCHA_SECRET_KEY, 20 | }; 21 | 22 | export default config; 23 | -------------------------------------------------------------------------------- /packages/backend/src/utils/cors.ts: -------------------------------------------------------------------------------- 1 | import cors from "@koa/cors"; 2 | import { Context, Next } from "koa"; 3 | import { createMiddleware } from "./middleware"; 4 | 5 | async function patchedCors(ctx: Context, next: Next) { 6 | if (ctx.method === "options") { 7 | ctx.set("Access-Control-Allow-Origin", ctx.get("Origin") || "*"); 8 | ctx.set( 9 | "Access-Control-Allow-Methods", 10 | "GET, POST, PATCH, OPTIONS, DELETE", 11 | ); 12 | ctx.set("Access-Control-Allow-Credentials", "true"); 13 | ctx.set( 14 | "Access-Control-Allow-Headers", 15 | "Origin, X-Requested-With, Content-Type, Accept, fdi-version, rid, st-auth-mode, Authorization", 16 | ); 17 | ctx.status = 204; 18 | return; 19 | } 20 | await cors({ 21 | origin(ctx) { 22 | return ctx.get("Origin") || "*"; 23 | }, 24 | credentials: true, 25 | allowMethods: ["GET", "POST", "PATCH", "DELETE", "OPTIONS"], 26 | allowHeaders: ["Content-Type", "Authorization", "Accept"], 27 | })(ctx, next); 28 | } 29 | 30 | export const corsMiddleware = createMiddleware(patchedCors); 31 | -------------------------------------------------------------------------------- /packages/backend/src/utils/cron.ts: -------------------------------------------------------------------------------- 1 | import resetUsage from "@/src/jobs/resetUsage"; 2 | import cron from "node-cron"; 3 | import purgeRuns from "../jobs/data-retention"; 4 | import stripeCounters from "../jobs/stripeMeters"; 5 | import { checkAlerts } from "@/src/jobs/alerts"; 6 | 7 | const EVERY_HOUR = "0 * * * *"; 8 | const EVERY_DAY_AT_4AM = "0 4 * * *"; 9 | const EVERY_DAY_AT_10AM = "0 10 * * *"; 10 | const EVERY_MINUTE = "* * * * *"; 11 | 12 | export function setupCronJobs() { 13 | cron.schedule(EVERY_DAY_AT_10AM, resetUsage, { 14 | name: "reset usage", 15 | }); 16 | 17 | cron.schedule(EVERY_HOUR, stripeCounters, { 18 | name: "stripe meters", 19 | }); 20 | 21 | cron.schedule(EVERY_DAY_AT_4AM, purgeRuns, { 22 | name: "purge runs", 23 | }); 24 | 25 | cron.schedule(EVERY_MINUTE, checkAlerts, { 26 | name: "check alerts", 27 | }); 28 | } 29 | -------------------------------------------------------------------------------- /packages/backend/src/utils/date.ts: -------------------------------------------------------------------------------- 1 | export function getReadableDateTime(date = new Date()) { 2 | const formatter = new Intl.DateTimeFormat("en-GB", { 3 | weekday: "long", 4 | year: "numeric", 5 | month: "long", 6 | day: "numeric", 7 | hour: "2-digit", 8 | minute: "2-digit", 9 | second: "2-digit", 10 | hour12: true, 11 | }); 12 | const readableDateTime = formatter.format(date); 13 | return readableDateTime; 14 | } 15 | -------------------------------------------------------------------------------- /packages/backend/src/utils/ibm.ts: -------------------------------------------------------------------------------- 1 | import { WatsonXAI } from "@ibm-cloud/watsonx-ai"; 2 | 3 | function getClient() { 4 | if ( 5 | !process.env.WATSONX_AI_AUTH_TYPE || 6 | !process.env.WATSONX_AI_APIKEY || 7 | !process.env.WATSONX_AI_PROJECT_ID 8 | ) { 9 | return null; 10 | } 11 | return WatsonXAI.newInstance({ 12 | version: "2024-03-14", 13 | serviceUrl: "https://us-south.ml.cloud.ibm.com", 14 | }); 15 | } 16 | 17 | const watsonxAi = getClient(); 18 | 19 | export default watsonxAi; 20 | -------------------------------------------------------------------------------- /packages/backend/src/utils/instrument.ts: -------------------------------------------------------------------------------- 1 | import * as Sentry from "@sentry/bun"; 2 | 3 | export function initSentry() { 4 | // Sentry will only be active if the DSN valid, and do nothing if not 5 | Sentry.init({ 6 | dsn: Bun.env.SENTRY_DSN, 7 | normalizeDepth: 10, 8 | }); 9 | } 10 | -------------------------------------------------------------------------------- /packages/backend/src/utils/jsonLogic.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * ["AND", { id: 'tag', paramsData: { tag: 'foo' } }, { id: 'tag', paramsData: { tag: 'bar' } }, ["OR", { id: 'tag', paramsData: { tag: 'baz' } }, { id: 'tag', paramsData: { tag: 'qux' } }] 3 | */ 4 | -------------------------------------------------------------------------------- /packages/backend/src/utils/koa.ts: -------------------------------------------------------------------------------- 1 | import Koa from "koa"; 2 | 3 | type CustomContext< 4 | StateT = Koa.DefaultState, 5 | ContextT = Koa.DefaultContext, 6 | > = Koa.ParameterizedContext; 7 | 8 | type AuthenticatedContext = Koa.Context & { 9 | state: { userId: string; orgId: string; projectId: string }; 10 | body: Object; 11 | }; 12 | type Context = AuthenticatedContext; 13 | 14 | export default Context; 15 | -------------------------------------------------------------------------------- /packages/backend/src/utils/license.ts: -------------------------------------------------------------------------------- 1 | import { Next } from "koa"; 2 | import Context from "./koa"; 3 | 4 | const TWO_HOURS = 2 * 60 * 60 * 1000; 5 | const cache = { 6 | license: { 7 | // Everything is set to true by default in case there's a problem connecting to the license server 8 | evalEnabled: true, 9 | samlEnabled: true, 10 | accessControlEnabled: true, 11 | }, 12 | lastFetch: TWO_HOURS + 200, 13 | }; 14 | 15 | async function licenseMiddleware(ctx: Context, next: Next) { 16 | const { LICENSE_KEY } = process.env; 17 | if (!LICENSE_KEY) { 18 | console.error("Please set the `LICENSE_KEY` environment variable."); 19 | process.exit(0); 20 | } 21 | 22 | try { 23 | if (Date.now() - cache.lastFetch > TWO_HOURS) { 24 | const licenseData = await fetch( 25 | `https://license.lunary.ai/v1/licenses/${LICENSE_KEY}`, 26 | ).then((res) => res.json()); 27 | 28 | cache.license = licenseData; 29 | cache.lastFetch = Date.now(); 30 | } 31 | } catch (error) { 32 | console.error(error); 33 | } finally { 34 | ctx.state.license = cache.license; 35 | await next(); 36 | } 37 | } 38 | 39 | export default licenseMiddleware; 40 | -------------------------------------------------------------------------------- /packages/backend/src/utils/middleware.ts: -------------------------------------------------------------------------------- 1 | import { Context, Next } from "koa"; 2 | 3 | export function createMiddleware( 4 | handler: (ctx: Context, next: Next) => Promise, 5 | ) { 6 | return async (ctx: Context, next: Next) => { 7 | await handler(ctx, next); 8 | }; 9 | } 10 | -------------------------------------------------------------------------------- /packages/backend/src/utils/ml.ts: -------------------------------------------------------------------------------- 1 | export async function callML( 2 | method: string, 3 | data: any, 4 | baseUrl: string = process.env.ML_URL!, 5 | ) { 6 | const response = await fetch(`${baseUrl}/${method}`, { 7 | method: "POST", 8 | headers: { 9 | "Content-Type": "application/json", 10 | }, 11 | body: JSON.stringify(data), 12 | }); 13 | 14 | return response.json(); 15 | } 16 | -------------------------------------------------------------------------------- /packages/backend/src/utils/notifications.ts: -------------------------------------------------------------------------------- 1 | const channels = { 2 | billing: process.env.SLACK_BILLING_CHANNEL, 3 | users: process.env.SLACK_USERS_CHANNEL, 4 | feedback: process.env.SLACK_FEEDBACK_CHANNEL, 5 | }; 6 | 7 | export const sendSlackMessage = async ( 8 | msg: string, 9 | thread: "billing" | "users" | "feedback", 10 | ) => { 11 | if (!process.env.SLACK_BOT_TOKEN) return; 12 | 13 | if (msg.includes("test@lunary.ai")) return; // ignore test runner emails 14 | 15 | try { 16 | const channelId = channels[thread] || null; 17 | 18 | if (!channelId) { 19 | console.error("No channel found for", thread); 20 | return; 21 | } 22 | 23 | await fetch(`https://hooks.slack.com/services/${channelId}`, { 24 | method: "POST", 25 | headers: { 26 | "Content-Type": "application/json", 27 | }, 28 | body: JSON.stringify({ 29 | text: msg, 30 | }), 31 | }); 32 | } catch (e) { 33 | console.error(e); 34 | } 35 | }; 36 | -------------------------------------------------------------------------------- /packages/backend/src/utils/openai.ts: -------------------------------------------------------------------------------- 1 | import OpenAI from "openai"; 2 | 3 | export function getOpenAIParams() { 4 | if (process.env.OPENAI_API_KEY) { 5 | return { 6 | apiKey: process.env.OPENAI_API_KEY, 7 | }; 8 | } else { 9 | return null; 10 | } 11 | } 12 | const clientParams = getOpenAIParams(); 13 | 14 | export default clientParams ? new OpenAI(clientParams) : null; 15 | -------------------------------------------------------------------------------- /packages/backend/src/utils/ratelimit.ts: -------------------------------------------------------------------------------- 1 | import ratelimit from "koa-ratelimit"; 2 | 3 | const db = new Map(); 4 | 5 | const MAX_REQUESTS_PER_MINUTE = parseInt( 6 | process.env.MAX_REQUESTS_PER_MINUTE || "150", 7 | 10, 8 | ); 9 | 10 | export default ratelimit({ 11 | driver: "memory", 12 | db: db, 13 | duration: 60000, 14 | errorMessage: "Sometimes You Just Have to Slow Down.", 15 | id: (ctx) => ctx.request.ip || ctx.ip || ctx.state.userId, 16 | headers: { 17 | remaining: "Rate-Limit-Remaining", 18 | reset: "Rate-Limit-Reset", 19 | total: "Rate-Limit-Total", 20 | }, 21 | max: MAX_REQUESTS_PER_MINUTE, 22 | disableHeader: false, 23 | whitelist: (ctx) => { 24 | // don't limit logged in users 25 | if (ctx.state.userId) return true; 26 | return false; 27 | }, 28 | }); 29 | 30 | export const aggressiveRatelimit = ratelimit({ 31 | driver: "memory", 32 | db: db, 33 | duration: 60000, 34 | errorMessage: "Sometimes You Just Have to Slow Down.", 35 | id: (ctx) => ctx.request.ip || ctx.ip || ctx.state.userId, 36 | headers: { 37 | remaining: "Rate-Limit-Remaining", 38 | reset: "Rate-Limit-Reset", 39 | total: "Rate-Limit-Total", 40 | }, 41 | max: 10, 42 | disableHeader: false, 43 | }); 44 | -------------------------------------------------------------------------------- /packages/backend/src/utils/sentry.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lunary-ai/lunary/27c09c35f982122a2d0b316a9cbbb97bd72711fe/packages/backend/src/utils/sentry.ts -------------------------------------------------------------------------------- /packages/backend/src/utils/stripe.ts: -------------------------------------------------------------------------------- 1 | import Stripe from "stripe"; 2 | 3 | const stripe = new Stripe(process.env.STRIPE_SECRET_KEY!, { 4 | apiVersion: "2024-04-10", 5 | }); 6 | 7 | export default stripe; 8 | -------------------------------------------------------------------------------- /packages/backend/src/utils/timeout.ts: -------------------------------------------------------------------------------- 1 | import { setTimeout } from "node:timers/promises"; 2 | import { TimeoutError } from "./errors"; 3 | 4 | export async function withTimeout( 5 | asyncFn: (...args: any[]) => Promise, 6 | ms: number, 7 | errorMessage: string = "Function execution timeout", 8 | ) { 9 | const timeoutPromise = setTimeout(ms).then(() => { 10 | throw new TimeoutError(errorMessage); 11 | }); 12 | 13 | return Promise.race([asyncFn(), timeoutPromise]); 14 | } 15 | -------------------------------------------------------------------------------- /packages/backend/src/utils/tokens/google.ts: -------------------------------------------------------------------------------- 1 | const GOOGLE_MODELS = [ 2 | "chat-bison-001", 3 | "code-bison-001", 4 | "text-bison-001", 5 | "codechat-bison-001", 6 | ] as const; 7 | 8 | type GoogleModel = (typeof GOOGLE_MODELS)[number]; 9 | 10 | export function isGoogleModel(modelName: string): boolean { 11 | return Boolean(GOOGLE_MODELS.find((model) => modelName.includes(model))); 12 | } 13 | 14 | export async function countGoogleTokens( 15 | modelName: GoogleModel, 16 | input: unknown, 17 | ): Promise { 18 | function prepareData(input: unknown) { 19 | const messages = Array.isArray(input) ? input : [input]; 20 | 21 | return messages.map((message) => { 22 | return { 23 | content: typeof message === "string" ? message : message.text, 24 | }; 25 | }); 26 | } 27 | 28 | try { 29 | const options = { 30 | method: "POST", 31 | headers: { 32 | "Content-Type": "application/json", 33 | }, 34 | body: JSON.stringify({ 35 | prompt: { 36 | messages: prepareData(input), 37 | }, 38 | }), 39 | }; 40 | 41 | const res = await fetch( 42 | `https://generativelanguage.googleapis.com/v1beta3/models/${modelName}:countMessageTokens?key=${process.env.PALM_API_KEY}`, 43 | options, 44 | ); 45 | const data = await res.json(); 46 | 47 | return data.tokenCount as number; 48 | } catch (e) { 49 | console.error("Error while counting tokens with Google API", e); 50 | return null; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /packages/backend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["ESNext"], 4 | "module": "esnext", 5 | "target": "ESNext", 6 | "moduleResolution": "bundler", 7 | "moduleDetection": "force", 8 | "allowImportingTsExtensions": true, 9 | "noEmit": true, 10 | "composite": true, 11 | "strict": true, 12 | "downlevelIteration": true, 13 | "skipLibCheck": true, 14 | "jsx": "react-jsx", 15 | "allowSyntheticDefaultImports": true, 16 | "forceConsistentCasingInFileNames": true, 17 | "allowJs": true, 18 | "baseUrl": ".", 19 | "rootDir": "src", 20 | "paths": { 21 | "@/*": ["./*"], 22 | "@/utils/*": ["src/utils/*"], 23 | "@/checks/*": ["src/checks/*"] 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /packages/db/0002.sql: -------------------------------------------------------------------------------- 1 | drop table provider cascade; 2 | 3 | alter table evaluation add column providers jsonb; 4 | 5 | drop index evaluation_result_evaluation_id_prompt_id_variation_id_mode_idx; 6 | 7 | alter table evaluation_result add column provider jsonb; 8 | 9 | alter table evaluation_result add column status text default 'success'; 10 | alter table evaluation_result alter column status set not null; 11 | alter table evaluation_result add column error text; 12 | alter table evaluation_result alter column results drop not null; 13 | alter table evaluation_result alter column output drop not null; 14 | alter table evaluation_result alter column model drop not null; -------------------------------------------------------------------------------- /packages/db/0003.sql: -------------------------------------------------------------------------------- 1 | alter table account add column avatar_url text; 2 | alter table account add column last_login_at timestamp with time zone; 3 | 4 | alter table org add column saml_idp_xml text; 5 | alter table org add column saml_enabled boolean default false; 6 | -------------------------------------------------------------------------------- /packages/db/0004.sql: -------------------------------------------------------------------------------- 1 | alter type user_role 2 | add value 'owner'; 3 | 4 | alter type user_role 5 | add value 'viewer'; 6 | 7 | alter type user_role 8 | add value 'prompt_editor'; 9 | 10 | alter type user_role 11 | add value 'billing'; 12 | 13 | create type user_role_new as enum ( 14 | 'owner', 15 | 'admin', 16 | 'member', 17 | 'viewer', 18 | 'prompt_editor', 19 | 'billing' 20 | ); 21 | 22 | alter table account alter column role type user_role_new 23 | using role::text::user_role_new; 24 | 25 | drop type user_role; 26 | 27 | alter type user_role_new rename to user_role; 28 | 29 | update 30 | account 31 | set 32 | role = 'owner' 33 | where 34 | role = 'admin'; 35 | -------------------------------------------------------------------------------- /packages/db/0005.sql: -------------------------------------------------------------------------------- 1 | alter table org add column if not exists radar_allowance integer default 500; 2 | alter table org add column if not exists eval_allowance integer default 500; 3 | 4 | -- Update to runs to save metadatas 5 | alter table run add column if not exists metadata jsonb; 6 | create index if not exists run_metadata_idx on run using gin (metadata); 7 | 8 | -- Migration of radars with PII 9 | 10 | create index if not exists radar_result_radar_id_run_id_idx on radar_result (radar_id, run_id); 11 | 12 | alter table radar add column if not exists created_at timestamp with time zone default now() not null; 13 | alter table radar add column if not exists updated_at timestamp with time zone default now() not null; 14 | 15 | alter table radar alter column project_id set not null; 16 | alter table radar alter column checks set not null; 17 | 18 | update radar 19 | set checks = '[ 20 | "AND", 21 | { 22 | "id": "pii", 23 | "params": { 24 | "field": "input", 25 | "type": "contains", 26 | "entities": ["person", "location", "email", "cc", "phone", "ssn"] 27 | } 28 | } 29 | ]'::jsonb 30 | where description like '%rompt contains PII%'; 31 | 32 | update radar 33 | set checks = '[ 34 | "AND", 35 | { 36 | "id": "pii", 37 | "params": { 38 | "field": "output", 39 | "type": "contains", 40 | "entities": ["person", "location", "email", "cc", "phone", "ssn"] 41 | } 42 | } 43 | ]'::jsonb 44 | where description like '%nswer contains PII%'; 45 | -------------------------------------------------------------------------------- /packages/db/0006.sql: -------------------------------------------------------------------------------- 1 | alter table account add column if not exists single_use_token text; 2 | 3 | create table if not exists account_project ( 4 | account_id uuid references account(id) on delete cascade, 5 | project_id uuid references project(id) on delete cascade, 6 | primary key (account_id, project_id) 7 | ); 8 | 9 | insert into account_project (account_id, project_id) 10 | select a.id as account_id, p.id as project_id 11 | from account a 12 | join project p on a.org_id = p.org_id 13 | on conflict (account_id, project_id) do nothing; -------------------------------------------------------------------------------- /packages/db/0009.sql: -------------------------------------------------------------------------------- 1 | alter table evaluation add column if not exists checklist_id uuid; 2 | alter table evaluation DROP CONSTRAINT IF EXISTS evaluation_checklist_id_fkey; 3 | alter table evaluation add constraint evaluation_checklist_id_fkey foreign key (checklist_id) references checklist(id) on delete set null; 4 | 5 | drop table if exists evaluation_prompt cascade; 6 | drop table if exists evaluation_prompt_variation cascade; 7 | 8 | alter table evaluation_result add constraint "fk_evaluation_result_prompt_id" foreign key (prompt_id) references dataset_prompt(id) on delete cascade; 9 | -------------------------------------------------------------------------------- /packages/db/0010.sql: -------------------------------------------------------------------------------- 1 | alter table account alter column role set not null; -------------------------------------------------------------------------------- /packages/db/0011.sql: -------------------------------------------------------------------------------- 1 | create index idx_run_feedback_thumb on run using gin ((feedback -> 'thumb') jsonb_path_ops); 2 | create index idx_run_feedback_thumbs on run using gin ((feedback -> 'thumbs') jsonb_path_ops); -------------------------------------------------------------------------------- /packages/db/0012.sql: -------------------------------------------------------------------------------- 1 | 2 | alter type org_plan add value if not exists 'team' after 'pro'; -------------------------------------------------------------------------------- /packages/db/0013.sql: -------------------------------------------------------------------------------- 1 | alter table dataset_prompt_variation drop column context; -------------------------------------------------------------------------------- /packages/db/0014.sql: -------------------------------------------------------------------------------- 1 | lock radar_result; 2 | delete from radar_result where run_id is null; 3 | alter table radar_result alter column run_id set not null; 4 | -------------------------------------------------------------------------------- /packages/db/0015.sql: -------------------------------------------------------------------------------- 1 | alter table template_version add column if not exists notes text; 2 | -------------------------------------------------------------------------------- /packages/db/0016.sql: -------------------------------------------------------------------------------- 1 | create table _email_block_list ( 2 | email text primary key not null 3 | ) -------------------------------------------------------------------------------- /packages/db/0017.sql: -------------------------------------------------------------------------------- 1 | create index on run((error is not null)); -------------------------------------------------------------------------------- /packages/db/0018.sql: -------------------------------------------------------------------------------- 1 | create table evaluator ( 2 | id uuid not null default uuid_generate_v4(), 3 | created_at timestamptz default now(), 4 | updated_at timestamptz default now(), 5 | project_id uuid not null, 6 | owner_id uuid, 7 | name varchar not null, 8 | slug varchar not null, 9 | type varchar not null, 10 | mode varchar, 11 | description text, 12 | params jsonb, 13 | filters jsonb, 14 | constraint evaluator_owner_id_fkey foreign key (owner_id) references account(id) on delete set null, 15 | constraint evaluator_project_id_fkey foreign key (project_id) references project(id) on delete cascade, 16 | primary key (id) 17 | ); 18 | 19 | create table evaluation_result_v2 ( 20 | run_id uuid not null, 21 | evaluator_id uuid not null, 22 | created_at timestamptz default now(), 23 | updated_at timestamptz default now(), 24 | result jsonb NOT NULL, 25 | constraint evaluation_result_evaluator_id_fkey foreign key (evaluator_id) references evaluator(id) on delete cascade, 26 | constraint evaluation_result_run_id_fkey foreign key (run_id) references run(id) on delete cascade, 27 | primary key(run_id, evaluator_id) 28 | ); -------------------------------------------------------------------------------- /packages/db/0019.sql: -------------------------------------------------------------------------------- 1 | lock table run; 2 | alter table run drop constraint run_external_user_id_fkey; 3 | alter table run add constraint run_external_user_id_fkey foreign key (external_user_id) references external_user(id) on delete cascade on update cascade; 4 | -------------------------------------------------------------------------------- /packages/db/0020.sql: -------------------------------------------------------------------------------- 1 | lock table run in exclusive mode; 2 | 3 | update run 4 | set template_version_id = null 5 | where template_version_id is not null 6 | and template_version_id not in (select id from template_version); 7 | 8 | alter table run add constraint run_template_version_id_fkey foreign key (template_version_id) references template_version(id) on delete set null; -------------------------------------------------------------------------------- /packages/db/0021.sql: -------------------------------------------------------------------------------- 1 | create index on run using gin (input_text gin_trgm_ops); 2 | create index on run using gin (output_text gin_trgm_ops); 3 | -------------------------------------------------------------------------------- /packages/db/0022.sql: -------------------------------------------------------------------------------- 1 | create table ingestion_rule ( 2 | id uuid not null default uuid_generate_v4(), 3 | created_at timestamptz default now(), 4 | updated_at timestamptz default now(), 5 | project_id uuid not null, 6 | type bpchar, 7 | filters jsonb, 8 | constraint ingestion_rule_project_id_fkey foreign key (project_id) REFERENCES project(id) on delete cascade, 9 | primary key (id) 10 | ); -------------------------------------------------------------------------------- /packages/db/0023.sql: -------------------------------------------------------------------------------- 1 | alter table ingestion_rule add constraint unique_project_type unique (project_id, type); -------------------------------------------------------------------------------- /packages/db/0025.sql: -------------------------------------------------------------------------------- 1 | alter table org add column seat_allowance int4 default null; -------------------------------------------------------------------------------- /packages/db/0026.sql: -------------------------------------------------------------------------------- 1 | alter table view add column type text not null default 'llm'; -------------------------------------------------------------------------------- /packages/db/0027.sql: -------------------------------------------------------------------------------- 1 | create index on run (created_at desc nulls last); 2 | create index on run (created_at asc nulls last); 3 | create index on run (duration desc nulls last); 4 | create index on run (duration asc nulls last); 5 | create index on run ((prompt_tokens + completion_tokens) desc nulls last); 6 | create index on run ((prompt_tokens + completion_tokens) asc nulls last); 7 | create index on run (cost desc nulls last); 8 | create index on run (cost asc nulls last); 9 | create index on external_user (created_at desc nulls last); 10 | create index on external_user (created_at asc nulls last); 11 | create index on external_user (last_seen desc nulls last); 12 | create index on external_user (last_seen asc nulls last); -------------------------------------------------------------------------------- /packages/db/0028.sql: -------------------------------------------------------------------------------- 1 | insert into model_mapping (name, pattern, unit, input_cost, output_cost, tokenizer, start_date) values 2 | ('gpt-4o-mini', '^(gpt-4o)$', 'TOKENS', 0.15, 0.6, 'openai', null), 3 | ('gpt-4o-mini-2024-07-18', '^(gpt-4o-mini-2024-07-18)$', 'TOKENS', 0.15, 0.6, 'openai', null); -------------------------------------------------------------------------------- /packages/db/0030.sql: -------------------------------------------------------------------------------- 1 | drop index if exists run_created_at_idx1; 2 | drop index if exists run_created_at_idx3; 3 | drop index if exists run_cost_idx; 4 | drop index if exists run_cost_idx1; 5 | drop index if exists run_created_at_project_id_idx; 6 | drop index if exists run_expr_idx1; 7 | drop index if exists run_expr_idx2; 8 | drop index if exists run_type_external_user_id_idx; 9 | drop index if exists run_type_idx; 10 | drop index if exists run_type_parent_run_id_idx; 11 | 12 | create index on run (project_id, created_at); 13 | create index on run (project_id, cost); 14 | create index on run (project_id, external_user_id); 15 | create index on run (project_id, (error is not null)); 16 | 17 | create index on run (project_id, created_at desc nulls last); 18 | create index on run (project_id, created_at asc nulls last); 19 | create index on run (project_id, duration desc nulls last); 20 | create index on run (project_id, duration asc nulls last); 21 | create index on run (project_id, (prompt_tokens + completion_tokens) desc nulls last); 22 | create index on run (project_id, (prompt_tokens + completion_tokens) asc nulls last); 23 | create index on run (project_id, cost desc nulls last); 24 | create index on run (project_id, cost asc nulls last); 25 | 26 | 27 | analyze; -------------------------------------------------------------------------------- /packages/db/0031.sql: -------------------------------------------------------------------------------- 1 | drop index if exists external_user_project_id_last_seen_idx; 2 | drop index if exists external_user_last_seen_idx; 3 | drop index if exists external_user_created_at_idx; 4 | 5 | create index on external_user (project_id, created_at desc nulls last); 6 | create index on external_user (project_id, created_at asc nulls last); 7 | create index on external_user (project_id, last_seen desc nulls last); 8 | create index on external_user (project_id, last_seen asc nulls last); 9 | 10 | analyze; -------------------------------------------------------------------------------- /packages/db/0032.sql: -------------------------------------------------------------------------------- 1 | create unique index on feedback_cache (project_id, feedback); 2 | create index on run (project_id, type, name); 3 | drop materialized view if exists model_name_cache; 4 | analyze run; -------------------------------------------------------------------------------- /packages/db/0033.sql: -------------------------------------------------------------------------------- 1 | drop materialized view if exists tag_cache; -------------------------------------------------------------------------------- /packages/db/0034.sql: -------------------------------------------------------------------------------- 1 | drop index if exists run_project_id_external_user_id_idx1; 2 | create index on run using gin (metadata); 3 | analyze run; 4 | drop materialized view if exists metadata_cache; 5 | -------------------------------------------------------------------------------- /packages/db/0035.sql: -------------------------------------------------------------------------------- 1 | drop materialized view if exists feedback_cache; -------------------------------------------------------------------------------- /packages/db/0036.sql: -------------------------------------------------------------------------------- 1 | alter table org add column data_retention_days int4 default null; 2 | alter table run drop column sibling_run_id; 3 | create index on radar_result(run_id); 4 | -------------------------------------------------------------------------------- /packages/db/0037.sql: -------------------------------------------------------------------------------- 1 | create index on run using gin ((input::text) gin_trgm_ops); 2 | create index on run using gin ((output::text) gin_trgm_ops); 3 | create index on run using gin ((error::text) gin_trgm_ops); -------------------------------------------------------------------------------- /packages/db/0038.sql: -------------------------------------------------------------------------------- 1 | alter table run drop column input_text; 2 | alter table run drop column output_text; 3 | alter table run drop column error_text; 4 | -------------------------------------------------------------------------------- /packages/db/0039.sql: -------------------------------------------------------------------------------- 1 | insert into model_mapping (name, pattern, unit, input_cost, output_cost, tokenizer, start_date) values 2 | ('gpt-4o-2024-08-06', '^(gpt-4o-2024-08-06)$', 'TOKENS', 2.5, 10, 'openai', null) -------------------------------------------------------------------------------- /packages/db/0040.sql: -------------------------------------------------------------------------------- 1 | alter type user_role add value 'analytics'; -------------------------------------------------------------------------------- /packages/db/0041.sql: -------------------------------------------------------------------------------- 1 | alter table view alter column owner_id drop not null; 2 | -------------------------------------------------------------------------------- /packages/db/0042.sql: -------------------------------------------------------------------------------- 1 | update model_mapping set pattern = '^(gpt-4o-mini)$' where name = 'gpt-4o-mini' and org_id is null; -------------------------------------------------------------------------------- /packages/db/0043.sql: -------------------------------------------------------------------------------- 1 | alter table org drop column radar_allowance; 2 | drop table radar cascade; 3 | drop table radar_result cascade; -------------------------------------------------------------------------------- /packages/db/0044.sql: -------------------------------------------------------------------------------- 1 | alter table template_version add column published_at timestamp with time zone default null; 2 | -------------------------------------------------------------------------------- /packages/db/0045.sql: -------------------------------------------------------------------------------- 1 | create table _data_warehouse_connector( 2 | id uuid default uuid_generate_v4() primary key, 3 | created_at timestamp with time zone default now(), 4 | updated_at timestamp with time zone default now(), 5 | project_id uuid not null, 6 | type text, 7 | status text, 8 | constraint fk_checklist_project_id foreign key (project_id) references project(id) on delete cascade 9 | ); 10 | create unique index on _data_warehouse_connector(project_id); -------------------------------------------------------------------------------- /packages/db/0046.sql: -------------------------------------------------------------------------------- 1 | insert into model_mapping (name, pattern, unit, input_cost, output_cost, tokenizer, start_date) values 2 | ('gpt-4o', '^(gpt-4o)$', 'TOKENS', 2.5, 10, 'openai', '2023-10-02'), 3 | ('o1-preview', '^(o1-preview)$', 'TOKENS', 15, 60, 'openai', null), 4 | ('o1-preview-2024-09-12', '^(o1-preview-2024-09-12)$', 'TOKENS', 15, 60, 'openai', null), 5 | ('o1-mini', '^(o1-mini)$', 'TOKENS', 3, 12, 'openai', null), 6 | ('o1-mini-2024-09-12', '^(o1-mini-2024-09-12)$', 'TOKENS', 3, 12, 'openai', null); 7 | -------------------------------------------------------------------------------- /packages/db/0047.sql: -------------------------------------------------------------------------------- 1 | alter table run drop constraint "run_parent_run_id_fkey"; 2 | alter table run add constraint "run_parent_run_id_fkey" foreign key (parent_run_id) references run (id) on update cascade on delete cascade; -------------------------------------------------------------------------------- /packages/db/0048.sql: -------------------------------------------------------------------------------- 1 | insert into model_mapping (name, pattern, unit, input_cost, output_cost, tokenizer, start_date) values 2 | ('gemini-1.5-pro', '^(gemini-1.5-pro)(@[a-zA-Z0-9]+)?$', 'CHARACTERS', 1.25, 5, 'google', '2024-10-01'), 3 | ('gemini-1.5-pro-002', '^(gemini-1.5-pro-002)(@[a-zA-Z0-9]+)?$', 'CHARACTERS', 1.25, 5, 'google', '2024-10-01'), 4 | ('gemini-1.5-flash', '^(gemini-1.5-flash)(@[a-zA-Z0-9]+)?$', 'CHARACTERS', 0.075, 0.3, 'google', '2024-10-01'), 5 | ('gemini-1.5-flash-8b', '^(gemini-1.5-flash-8b)(@[a-zA-Z0-9]+)?$', 'CHARACTERS', 0.075, 0.3, 'google', '2024-10-01'); -------------------------------------------------------------------------------- /packages/db/0049.sql: -------------------------------------------------------------------------------- 1 | alter type user_role add value 'collaborator' after 'viewer'; -------------------------------------------------------------------------------- /packages/db/0050.sql: -------------------------------------------------------------------------------- 1 | create table run_score ( 2 | run_id uuid not null, 3 | label text not null, 4 | created_at timestamptz default now(), 5 | updated_at timestamptz default now(), 6 | value jsonb not null, 7 | comment text, 8 | primary key ("run_id", "label"), 9 | constraint value_type_check 10 | check ( 11 | jsonb_typeof(value) in ('number', 'string', 'boolean') 12 | ) 13 | ); 14 | -------------------------------------------------------------------------------- /packages/db/0051.sql: -------------------------------------------------------------------------------- 1 | alter table evaluator add constraint evaluator_project_id_slug_unique unique (project_id, slug); -------------------------------------------------------------------------------- /packages/db/0052.sql: -------------------------------------------------------------------------------- 1 | alter table account add column if not exists export_single_use_token text null default null; -------------------------------------------------------------------------------- /packages/db/0054.sql: -------------------------------------------------------------------------------- 1 | create index idx_evaluator_type on evaluator (type); 2 | alter table org add column custom_dashboards_enabled boolean not null default false; 3 | 4 | -------------------------------------------------------------------------------- /packages/db/0055.sql: -------------------------------------------------------------------------------- 1 | alter table evaluation_result_v2 2 | drop constraint evaluation_result_run_id_fkey, 3 | add constraint evaluation_result_run_id_fkey foreign key (run_id) references public.run(id) on delete cascade on update cascade; -------------------------------------------------------------------------------- /packages/db/0056.sql: -------------------------------------------------------------------------------- 1 | drop table if exists log; 2 | create extension if not exists btree_gin; 3 | 4 | -- !!!WARNING!!! statement below moved to the new index creation system in 0059.sql, but it's kept as a reference for users who have already run this migration 5 | 6 | -- drop index if exists run_input_idx; 7 | -- drop index if exists run_output_idx; 8 | -- drop index if exists run_error_idx; 9 | -- create index run_project_id_input_idx on run using gin (project_id, ((input)::text) gin_trgm_ops); 10 | -- create index run_project_id_output_idx on run using gin (project_id, ((output)::text) gin_trgm_ops); -------------------------------------------------------------------------------- /packages/db/0058.sql: -------------------------------------------------------------------------------- 1 | create table _db_migration_index ( 2 | id serial primary key, 3 | name text not null, 4 | statement text not null, 5 | operation text not null default 'create', 6 | status text not null default 'pending' -- "pending", "in-progress", "done", "failed" 7 | ); 8 | 9 | -------------------------------------------------------------------------------- /packages/db/0059.sql: -------------------------------------------------------------------------------- 1 | insert into _db_migration_index (name, operation, statement) values 2 | ('run_input_idx', 'drop', 'drop index concurrently if exists run_input_idx'), 3 | ('run_output_idx', 'drop', 'drop index concurrently if exists run_output_idx'), 4 | ('run_error_idx', 'drop', 'drop index concurrently if exists run_error_idx'), 5 | ('run_project_id_input_idx', 'create', 'create index concurrently if not exists run_project_id_input_idx on run using gin (project_id, ((input)::text) gin_trgm_ops)'), 6 | ('run_project_id_output_idx', 'create', 'create index concurrently if not exists run_project_id_output_idx on run using gin (project_id, ((output)::text) gin_trgm_ops)'); 7 | -------------------------------------------------------------------------------- /packages/db/0060.sql: -------------------------------------------------------------------------------- 1 | alter table _db_migration_index rename to _db_migration_async; 2 | 3 | insert into _db_migration_async (name, operation, statement) values 4 | ('metadata_cache', 'create-materialized-view', 5 | 'create materialized view metadata_cache as 6 | select 7 | run.project_id, 8 | run.type, 9 | jsonb_object_keys(run.metadata) as key, 10 | now() as refreshed_at 11 | from 12 | run 13 | group by 14 | run.project_id, 15 | run.type, 16 | jsonb_object_keys(run.metadata);' 17 | ), 18 | ('metadata_cache_project_id_idx', 'create', 'create index concurrently if not exists metadata_cache_project_id_idx on metadata_cache(project_id)'), 19 | ('metadata_cache_project_id_type_key_idx', 'create', 'create unique index concurrently if not exists metadata_cache_project_id_type_key_idx on metadata_cache (project_id, type, key)'); 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /packages/db/0061.sql: -------------------------------------------------------------------------------- 1 | insert into model_mapping (name, pattern, unit, input_cost, output_cost, tokenizer, start_date) values 2 | ('claude-3-5-sonnet', '(claude-3.5-sonnet|claude-3-5-sonnet)', 'TOKENS', 3, 15, 'anthropic', null), 3 | ('claude-3-5-haiku', '(claude-3.5-haiku|claude-3-5-haiku)', 'TOKENS', 0.8, 4, 'anthropic', null); -------------------------------------------------------------------------------- /packages/db/0062.sql: -------------------------------------------------------------------------------- 1 | insert into model_mapping (name, pattern, unit, input_cost, output_cost, tokenizer, start_date) values 2 | ('o1', 'o1', 'TOKENS', 15, 60, 'openai', null); -------------------------------------------------------------------------------- /packages/db/0063.sql: -------------------------------------------------------------------------------- 1 | drop materialized view if exists run_parent_feedback_cache; -------------------------------------------------------------------------------- /packages/db/0064.sql: -------------------------------------------------------------------------------- 1 | insert into _db_migration_async (name, operation, statement) values 2 | ('run', 'alter-table', 'alter table run alter column type set not null'); -------------------------------------------------------------------------------- /packages/db/0065.sql: -------------------------------------------------------------------------------- 1 | insert into _db_migration_async (name, operation, statement) values 2 | ('run', 'alter-table', 3 | 'update run 4 | set 5 | feedback = (feedback - ''thumbs'') || jsonb_build_object(''thumb'', feedback ->> ''thumbs'') 6 | where 7 | feedback ->> ''thumbs'' is not null;'), 8 | ('run_project_id_type_feedback_idx', 'drop', 'drop index concurrently if exists run_project_id_type_feedback_idx'), 9 | ('run_project_id_type_feedback_idx', 'create', 'create index concurrently if not exists run_project_id_type_feedback_idx on run (project_id, type, (feedback->>''thumb''))'); -------------------------------------------------------------------------------- /packages/db/0066.sql: -------------------------------------------------------------------------------- 1 | insert into _db_migration_async (name, operation, statement) values 2 | ('run_project_id_type_idx1', 'create', 'create index concurrently run_project_id_type_idx1 on run(project_id, type) where type in (''chain'', ''agent'') and parent_run_id is null'); -------------------------------------------------------------------------------- /packages/db/0067.sql: -------------------------------------------------------------------------------- 1 | alter table org rename column custom_dashboards_enabled to custom_charts_enabled; -------------------------------------------------------------------------------- /packages/db/0068.sql: -------------------------------------------------------------------------------- 1 | create type provider_name as enum ( 2 | 'openai', 3 | 'azure_openai', 4 | 'amazon_bedrock', 5 | 'google_ai_studio', 6 | 'google_vertex', 7 | 'anthropic', 8 | 'x_ai' 9 | ); 10 | 11 | 12 | create table provider_config ( 13 | id uuid default uuid_generate_v4() primary key, 14 | project_id uuid not null, 15 | created_at timestamptz not null default now(), 16 | updated_at timestamptz not null default now(), 17 | provider_name provider_name not null, 18 | api_key text not null, 19 | extra_config jsonb, 20 | constraint fk_project_id foreign key (project_id) references project (id) 21 | ); 22 | 23 | create table provider_config_model ( 24 | id uuid default uuid_generate_v4() primary key, 25 | created_at timestamptz not null default now(), 26 | updated_at timestamptz not null default now(), 27 | provider_config_id uuid not null, 28 | name varchar(100) not null, 29 | display_name varchar(100), 30 | constraint fk_provider_config_id foreign key (provider_config_id) references provider_config(id) 31 | ); 32 | create unique index on provider_config_model(provider_config_id, name) 33 | -------------------------------------------------------------------------------- /packages/db/0069.sql: -------------------------------------------------------------------------------- 1 | create table audit_log ( 2 | id uuid default uuid_generate_v4() primary key, 3 | created_at timestamptz not null default now(), 4 | updated_at timestamptz not null default now(), 5 | org_id uuid not null, 6 | resource_type text not null, 7 | resource_id text, 8 | action text not null, 9 | user_id uuid not null, 10 | user_name text, 11 | user_email text, 12 | project_id uuid, 13 | project_name text, 14 | ip_address text, 15 | user_agent text, 16 | constraint fk_audit_log_org_id foreign key (org_id) references org(id) on delete cascade 17 | ); 18 | 19 | create index audit_log_org_id_created_at_idx on audit_log(org_id, created_at desc); -------------------------------------------------------------------------------- /packages/db/0070.sql: -------------------------------------------------------------------------------- 1 | alter table run add column if not exists is_deleted boolean default false; 2 | alter table project add column if not exists data_retention_days integer default null; 3 | insert into _db_migration_async (name, operation, statement) values 4 | ('run_project_id_not_deleted_idx', 'create', 'create index concurrently run_project_id_not_deleted_idx on run (project_id) where is_deleted = false'); -------------------------------------------------------------------------------- /packages/db/0071.sql: -------------------------------------------------------------------------------- 1 | create table _whitelisted_domain ( 2 | id uuid default uuid_generate_v4() primary key, 3 | created_at timestamp with time zone default now(), 4 | updated_at timestamp with time zone default now(), 5 | domain text not null 6 | ); -------------------------------------------------------------------------------- /packages/db/0072.sql: -------------------------------------------------------------------------------- 1 | insert into model_mapping (name, pattern, unit, input_cost, output_cost, tokenizer, start_date, provider) values 2 | ('gpt-4.5-preview', '^(gpt-4.5-preview)$', 'TOKENS', 75, 150, 'openai', null, 'openai'), 3 | ('gpt-4o-realtime-preview', '^(gpt-4o-realtime-preview)$', 'TOKENS', 40, 80, 'openai', null, 'openai'); -------------------------------------------------------------------------------- /packages/db/0073.sql: -------------------------------------------------------------------------------- 1 | alter table model_mapping add column input_caching_cost_reduction smallint not null default 0; 2 | 3 | update model_mapping 4 | set 5 | input_caching_cost_reduction = 0.5 6 | where 7 | name in ( 8 | 'gpt-4.5-preview', 9 | 'gpt-4o', 10 | 'gpt-4o-mini', 11 | 'gpt-4o-realtime-preview', 12 | 'o1-preview', 13 | 'o1-mini' 14 | ) 15 | and org_id is null; 16 | 17 | alter table run add column cached_prompt_tokens int4 not null default 0; -------------------------------------------------------------------------------- /packages/db/0074.sql: -------------------------------------------------------------------------------- 1 | drop extension if exists bree_gin; 2 | 3 | insert into model_mapping (name, pattern, unit, input_cost, output_cost, tokenizer, provider, input_caching_cost_reduction) values 4 | ('gpt-4.1', '^(gpt-4.1)$', 'TOKENS', 2, 8, 'openai', 'openai', 0.25), 5 | ('gpt-4.1-mini', '^(gpt-4.1-mini)$', 'TOKENS', 0.4, 1.60, 'openai', 'openai', 0.25), 6 | ('gpt-4.1-nano', '^(gpt-4.1-nano)$', 'TOKENS', 0.15, 0.4, 'openai', 'openai', 0.25); -------------------------------------------------------------------------------- /packages/db/0075.sql: -------------------------------------------------------------------------------- 1 | insert into _db_migration_async (name, operation, statement) values 2 | ('run_metadata_idx', 'create', 'create index concurrently run_metadata_idx on run using gin (metadata jsonb_path_ops)'); -------------------------------------------------------------------------------- /packages/db/0076.sql: -------------------------------------------------------------------------------- 1 | insert into _db_migration_async (name, operation, statement) values 2 | ('run_project_id_with_template_version', 'create', 'create index concurrently run_project_id_with_template_version on run (project_id) where template_version is not null'), 3 | ('run_project_id_with_error', 'create', 'create index concurrently run_project_id_with_error on run (project_id) where error is not null'); -------------------------------------------------------------------------------- /packages/db/0077.sql: -------------------------------------------------------------------------------- 1 | insert into model_mapping (name, pattern, unit, input_cost, output_cost, tokenizer, provider, input_caching_cost_reduction) values 2 | ('o4-mini', '^(o4-mini)$', 'TOKENS', 1.10, 4.4, 'openai', 'openai', 4), 3 | ('o3', '^(o3)$', 'TOKENS', 10, 40, 'openai', 'openai', 4), 4 | ('o3-mini', '^(o3-mini)$', 'TOKENS', 1.10, 4.40, 'openai', 'openai', 4); -------------------------------------------------------------------------------- /packages/db/0078.sql: -------------------------------------------------------------------------------- 1 | update model_mapping set start_date = '2020-01-01 00:00:00+00' where start_date is null; 2 | alter table model_mapping alter column start_date set not null; 3 | alter table model_mapping alter column start_date set default '2020-01-01 00:00:00+00'; 4 | 5 | 6 | 7 | alter table model_mapping alter column input_caching_cost_reduction drop not null; 8 | insert into model_mapping (name, pattern, unit, input_cost, output_cost, tokenizer, provider) values 9 | ('claude-3-7-sonnet', '^(claude-3-7-sonnet-.*)$', 'TOKENS', 3, 15, 'anthropic', 'anthropic'); -------------------------------------------------------------------------------- /packages/db/0079.sql: -------------------------------------------------------------------------------- 1 | insert into model_mapping (name, pattern, unit, input_cost, output_cost, tokenizer, provider) values 2 | ('gemini-2.0-flash-lite', '^(gemini-2.0-flash-lite-.*)$', 'CHARACTERS', 0.075, 0.30, 'google', 'google'), 3 | ('gemini-2.0-flash', '^(gemini-2.0-flash-.*)$', 'CHARACTERS', 0.10, 0.40, 'google', 'google'), 4 | ('gemini-2.5-pro-preview', '^(gemini-2.5-pro-preview.*)$', 'CHARACTERS', 1.25, 10, 'google', 'google'), 5 | ('gemini-2.5-flash-preview', '^(gemini-2.5-flash-preview.*)$', 'CHARACTERS', 0.15, 3.50, 'google', 'google'), 6 | ('gemini-2.5-pro-exp', '^(gemini-2.5-pro-exp-.*)$', 'CHARACTERS', 1.25, 2.50, 'google', 'google'); -------------------------------------------------------------------------------- /packages/db/0080.sql: -------------------------------------------------------------------------------- 1 | delete from _db_migration_async where name = 'run_project_id_with_template_version'; 2 | 3 | insert into _db_migration_async (name, operation, statement) values 4 | ('run_project_id_with_template_version', 'create', 'create index concurrently run_project_id_with_template_version on run (project_id) where template_version_id is not null'); -------------------------------------------------------------------------------- /packages/db/0081.sql: -------------------------------------------------------------------------------- 1 | create index template_version_template_idx on template_version (template_id); 2 | -------------------------------------------------------------------------------- /packages/db/0082.sql: -------------------------------------------------------------------------------- 1 | create table _job ( 2 | id uuid primary key default gen_random_uuid(), 3 | created_at timestamptz not null default now(), 4 | ended_at timestamptz, 5 | org_id uuid not null, 6 | type text not null, 7 | status text not null check (status in ('pending', 'running', 'done', 'failed')), 8 | error text, 9 | progress float not null default 0, 10 | constraint job_org_id_fkey foreign key (org_id) references org(id) on delete cascade 11 | ); 12 | 13 | create unique index one_active_job_per_org_and_type 14 | on _job (org_id, type) 15 | where status in ('pending','running'); -------------------------------------------------------------------------------- /packages/db/0083.sql: -------------------------------------------------------------------------------- 1 | alter table provider_config 2 | drop constraint fk_project_id, 3 | add constraint fk_project_id foreign key (project_id) references project (id) on delete cascade; -------------------------------------------------------------------------------- /packages/db/0084.sql: -------------------------------------------------------------------------------- 1 | create table org_invitation ( 2 | id uuid primary key default gen_random_uuid(), 3 | created_at timestamptz not null default now(), 4 | updated_at timestamptz not null default now(), 5 | email text not null, 6 | org_id uuid not null, 7 | role user_role not null, 8 | token text not null, 9 | email_verified boolean, 10 | constraint org_invitation_account_id_fkey foreign key (org_id) references org(id) on delete cascade 11 | ); -------------------------------------------------------------------------------- /packages/db/0085.sql: -------------------------------------------------------------------------------- 1 | drop index if exists idx_run_project_id_duration; 2 | drop index if exists run_project_id_duration_idx; 3 | 4 | insert into _db_migration_async (name, operation, statement) values 5 | ('run_project_id_duration_desc_idx', 'create', 'create index concurrently if not exists run_project_id_duration_desc_idx on run (project_id, duration desc nulls last)'), 6 | ('run_project_id_duration_asc_idx', 'create', 'create index concurrently if not exists run_project_id_duration_asc_idx on run (project_id, duration)'), 7 | ('run_project_total_tokens_desc_idx', 'create', 'create index concurrently if not exists run_project_total_tokens_desc_idx on run (project_id, (prompt_tokens + completion_tokens) desc nulls last)'), 8 | ('run_project_total_tokens_asc_idx', 'create', 'create index concurrently if not exists run_project_total_tokens_asc_idx on run (project_id, (prompt_tokens + completion_tokens) asc)'), 9 | ('run_project_cost_desc_idx', 'create', 'create index concurrently if not exists run_project_cost_desc_idx on run (project_id, cost desc nulls last)'), 10 | ('run_project_cost_asc_idx', 'create', 'create index concurrently if not exists run_project_cost_asc_idx on run (project_id, cost asc)'); -------------------------------------------------------------------------------- /packages/db/0086.sql: -------------------------------------------------------------------------------- 1 | drop index if exists run_project_id_cost_idx; 2 | drop index if exists run_project_id_type_feedback_idx; 3 | 4 | insert into _db_migration_async (name, operation, statement) values 5 | ('run_project_id_template_version_id_idx', 'create', 'create index concurrently if not exists run_project_id_template_version_id_idx on run (project_id, template_version_id)'), 6 | ('run_project_name_created_at_desc_idx', 'create', 'create index concurrently if not exists run_project_name_created_at_desc_idx on run (project_id, name, created_at desc nulls last)'); 7 | -------------------------------------------------------------------------------- /packages/db/0087.sql: -------------------------------------------------------------------------------- 1 | alter table dataset_prompt add column ground_truth text; 2 | 3 | create table evaluation_v2 ( 4 | id uuid primary key default gen_random_uuid(), 5 | created_at timestamp with time zone default now(), 6 | updated_at timestamp with time zone default now(), 7 | name text not null, 8 | project_id uuid not null, 9 | dataset_id uuid not null, 10 | foreign key (dataset_id) references dataset (id) on delete cascade, 11 | foreign key (project_id) references project (id) on delete cascade 12 | ); 13 | 14 | 15 | create table evaluation_evaluator ( 16 | evaluation_id uuid not null, 17 | evaluator_id uuid not null, 18 | "order" integer, 19 | weight numeric, 20 | created_at timestamptz default now(), 21 | primary key (evaluation_id, evaluator_id), 22 | foreign key (evaluation_id) references evaluation_v2 (id) on delete cascade, 23 | foreign key (evaluator_id) references evaluator (id) on delete cascade 24 | ); -------------------------------------------------------------------------------- /packages/db/0088.sql: -------------------------------------------------------------------------------- 1 | alter table org add column beta boolean not null default false; -------------------------------------------------------------------------------- /packages/db/0089.sql: -------------------------------------------------------------------------------- 1 | alter table org add column data_filtering_enabled boolean not null default false; 2 | alter table org add column data_warehouse_enabled boolean not null default false; -------------------------------------------------------------------------------- /packages/db/0090.sql: -------------------------------------------------------------------------------- 1 | insert into _db_migration_async (name, operation, statement) values 2 | ('run_name_is_deleted_idx', 'create', 'create index concurrently if not exists run_name_is_deleted_idx on run (name, is_deleted)'); -------------------------------------------------------------------------------- /packages/db/0091.sql: -------------------------------------------------------------------------------- 1 | -- create alert and alert_history tables 2 | 3 | create table alert ( 4 | id uuid default uuid_generate_v4() primary key, 5 | created_at timestamptz default now() not null, 6 | updated_at timestamptz default now() not null, 7 | project_id uuid not null, 8 | owner_id uuid not null, 9 | name text not null, 10 | status text not null default 'healthy', 11 | threshold float not null, 12 | metric text not null, 13 | time_frame_minutes integer not null, 14 | email text, 15 | webhook_url text, 16 | constraint fk_alert_project_id foreign key (project_id) references project(id) on delete cascade, 17 | constraint fk_alert_owner_id foreign key (owner_id) references account(id) on delete set null 18 | ); 19 | 20 | create index idx_alert_project_id on alert (project_id); 21 | 22 | create table alert_history ( 23 | id uuid default uuid_generate_v4() primary key, 24 | alert_id uuid not null, 25 | start_time timestamptz not null, 26 | end_time timestamptz not null, 27 | trigger float not null, 28 | status text not null, 29 | constraint fk_alert_history_alert foreign key (alert_id) references alert(id) on delete set null 30 | ); 31 | 32 | create index idx_alert_history_alert_id on alert_history (alert_id); 33 | -------------------------------------------------------------------------------- /packages/db/0092.sql: -------------------------------------------------------------------------------- 1 | insert into _db_migration_async (name, operation, statement) values 2 | ( 3 | 'run_llm_error_created_at_idx', 4 | 'create', 5 | $$create index concurrently if not exists run_llm_error_created_at_idx on run (created_at) where (type = 'llm' and is_deleted = false and status = 'error')$$ 6 | ); -------------------------------------------------------------------------------- /packages/db/0093.sql: -------------------------------------------------------------------------------- 1 | alter table evaluator drop constraint evaluator_project_id_slug_unique; -------------------------------------------------------------------------------- /packages/db/0094.sql: -------------------------------------------------------------------------------- 1 | create table run_toxicity ( 2 | run_id uuid primary key references run(id) on delete cascade, 3 | toxic_input boolean not null, 4 | toxic_output boolean not null, 5 | input_labels text[] not null, 6 | output_labels text[] not null, 7 | messages jsonb not null 8 | ); 9 | 10 | create index on run_toxicity(toxic_input); 11 | create index on run_toxicity(toxic_output); -------------------------------------------------------------------------------- /packages/db/0095.sql: -------------------------------------------------------------------------------- 1 | alter table template drop column if exists name; 2 | alter table template drop column if exists "group"; 3 | alter table template alter column slug set not null; -------------------------------------------------------------------------------- /packages/db/0096.sql: -------------------------------------------------------------------------------- 1 | create index external_user_props_gin on external_user using gin (props jsonb_path_ops); -------------------------------------------------------------------------------- /packages/db/0097.sql: -------------------------------------------------------------------------------- 1 | insert into model_mapping (name, pattern, unit, input_cost, output_cost, tokenizer, provider) values 2 | ('claude-opus-4', '^(.*claude-opus-4.*)$', 'TOKENS', 15, 75, 'anthropic', 'anthropic'), 3 | ('claude-sonnet-4', '^(.*claude-sonnet-4.*)$', 'TOKENS', 3, 15, 'anthropic', 'anthropic'); -------------------------------------------------------------------------------- /packages/db/0098.sql: -------------------------------------------------------------------------------- 1 | alter table provider_config_model drop constraint fk_provider_config_id; 2 | alter table provider_config_model add constraint fk_provider_config_id foreign key (provider_config_id) references provider_config (id) on delete cascade; 3 | -------------------------------------------------------------------------------- /packages/e2e/.gitignore: -------------------------------------------------------------------------------- 1 | test-results/ 2 | playwright-report/ 3 | blob-report/ 4 | .cache/ 5 | .auth/ -------------------------------------------------------------------------------- /packages/e2e/auth.setup.ts: -------------------------------------------------------------------------------- 1 | import { test } from "@playwright/test"; 2 | 3 | import { deleteOrg, populateLogs } from "./utils/db"; 4 | 5 | const authFile = ".auth/user.json"; 6 | 7 | test.beforeAll(async () => { 8 | // ensure the database is clean before we run the tests 9 | await deleteOrg(); 10 | }); 11 | 12 | test("signup flow", async ({ page }) => { 13 | await page.goto("/"); 14 | 15 | await page.getByRole("link", { name: "Sign Up" }).click(); 16 | 17 | await page.waitForURL("**/signup"); 18 | 19 | await page.getByPlaceholder("Your email").click(); 20 | await page.getByPlaceholder("Your email").fill("test@lunary.ai"); 21 | 22 | await page.getByPlaceholder("Your full name").click(); 23 | await page.getByPlaceholder("Your full name").fill("test test"); 24 | 25 | await page.getByPlaceholder("Pick a password").click(); 26 | await page.getByPlaceholder("Pick a password").fill("testtest"); 27 | 28 | await page.getByTestId("continue-button").click(); 29 | 30 | await page.waitForURL("**/dashboards*"); 31 | 32 | await page.context().storageState({ path: authFile }); 33 | 34 | await populateLogs(); 35 | }); 36 | -------------------------------------------------------------------------------- /packages/e2e/global.teardown.ts: -------------------------------------------------------------------------------- 1 | import { test } from "@playwright/test"; 2 | import { deleteOrg } from "./utils/db"; 3 | 4 | test("clean up database", async ({}) => { 5 | await deleteOrg(); 6 | }); 7 | -------------------------------------------------------------------------------- /packages/e2e/login.spec.ts: -------------------------------------------------------------------------------- 1 | import { test } from "@playwright/test"; 2 | 3 | test("logout and back in login", async ({ page }) => { 4 | await page.goto("/"); 5 | 6 | await page.waitForLoadState("networkidle"); 7 | 8 | await page.getByTestId("account-sidebar-item").click(); 9 | await page.getByTestId("logout-button").click(); 10 | 11 | await page.waitForURL("**/login*"); 12 | 13 | await page.getByPlaceholder("Your email").click(); 14 | await page.getByPlaceholder("Your email").fill("test@lunary.ai"); 15 | 16 | await page.getByPlaceholder("Your password").click(); 17 | await page.getByPlaceholder("Your password").fill("testtest"); 18 | 19 | await page.getByRole("button", { name: "Login" }).click(); 20 | 21 | await page.waitForURL("**/dashboards*"); 22 | }); 23 | -------------------------------------------------------------------------------- /packages/e2e/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@lunary/e2e", 3 | "scripts": { 4 | "test": "playwright test" 5 | }, 6 | "dependencies": { 7 | "@langchain/core": "^0.3.49", 8 | "@langchain/openai": "^0.5.7", 9 | "@playwright/test": "^1.52.0", 10 | "backend": "*", 11 | "dotenv": "^16.5.0", 12 | "lunary": "^0.8.8" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /packages/e2e/projects.setup.ts: -------------------------------------------------------------------------------- 1 | // import { expect, test } from "@playwright/test"; 2 | 3 | // test("create new project, rename it and delete it", async ({ page }) => { 4 | // await page.goto("/"); 5 | 6 | // await page.waitForLoadState("networkidle"); 7 | 8 | // await page.getByRole("button", { name: "Project #1" }).click(); 9 | 10 | // await page.getByTestId("new-project").click(); 11 | 12 | // await page.getByRole("heading", { name: "Project #" }).click(); 13 | // await page.getByTestId("rename-input").fill("Project #12"); 14 | // await page.getByTestId("rename-input").press("Enter"); 15 | 16 | // await expect( 17 | // page.getByRole("heading", { name: "Project #12" }), 18 | // ).toBeVisible(); 19 | 20 | // await page.getByRole("button", { name: "Project #12" }).click(); 21 | // await page.goto("/settings"); 22 | 23 | // await page.getByTestId("delete-project-button").click(); 24 | // await page.getByTestId("delete-project-popover-button").click(); 25 | 26 | // await page.waitForURL("**/dashboards/**"); 27 | // }); 28 | -------------------------------------------------------------------------------- /packages/e2e/users.spec.ts: -------------------------------------------------------------------------------- 1 | import test, { expect } from "@playwright/test" 2 | 3 | test("view external users list", async ({ page }) => { 4 | await page.goto("/users") 5 | 6 | const table = page.locator("table") 7 | await expect(table.getByText("Salut-123")).toBeVisible() 8 | }) 9 | -------------------------------------------------------------------------------- /packages/frontend/.dockerignore: -------------------------------------------------------------------------------- 1 | */node_modules 2 | *npm-debug.log 3 | *.Dockerfile 4 | docker-compose* 5 | *.env 6 | .dockerignore 7 | .git -------------------------------------------------------------------------------- /packages/frontend/.env.example: -------------------------------------------------------------------------------- 1 | API_URL=http://localhost:3333 2 | NEXT_PUBLIC_API_URL=http://localhost:3333 3 | GOOGLE_CLIENT_ID=your-google-client-id 4 | NEXT_PUBLIC_GOOGLE_CLIENT_ID=your-google-client-id 5 | -------------------------------------------------------------------------------- /packages/frontend/.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .env.local 3 | 4 | # Logs 5 | *.log 6 | npm-debug.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | 13 | # Directory for instrumented libs generated by jscoverage/JSCover 14 | lib-cov 15 | 16 | # Coverage directory used by tools like istanbul 17 | coverage 18 | 19 | # nyc test coverage 20 | .nyc_output 21 | 22 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 23 | .grunt 24 | 25 | # node-waf configuration 26 | .lock-wscript 27 | 28 | # Compiled binary addons (http://nodejs.org/api/addons.html) 29 | build/Release 30 | 31 | # Dependency directories 32 | node_modules 33 | jspm_packages 34 | 35 | # Optional npm cache directory 36 | .npm 37 | 38 | # Optional REPL history 39 | .node_repl_history 40 | .next 41 | .swc 42 | .vercel 43 | .vscode/settings.json 44 | 45 | *.DS_STORE 46 | 47 | 48 | public/config.js 49 | # Sentry Config File 50 | .env.sentry-build-plugin 51 | -------------------------------------------------------------------------------- /packages/frontend/README.md: -------------------------------------------------------------------------------- 1 | ## Next.js 2 | -------------------------------------------------------------------------------- /packages/frontend/actions/openai-actions.ts: -------------------------------------------------------------------------------- 1 | "use server" 2 | 3 | import type { PromptVersion } from "@/types/prompt-types" 4 | 5 | export async function runSinglePrompt(version: PromptVersion, userMessage: string, context: string): Promise { 6 | try { 7 | // Use fetch to call our API route instead of initializing OpenAI directly 8 | const response = await fetch(`${process.env.NEXT_PUBLIC_BASE_URL || ""}/api/run-prompt`, { 9 | method: "POST", 10 | headers: { 11 | "Content-Type": "application/json", 12 | }, 13 | body: JSON.stringify({ 14 | version, 15 | userMessage, 16 | context, 17 | }), 18 | }) 19 | 20 | if (!response.ok) { 21 | throw new Error(`API returned status ${response.status}`) 22 | } 23 | 24 | const data = await response.json() 25 | return data.response || "No response generated" 26 | } catch (error) { 27 | console.error("Error running prompt:", error) 28 | throw new Error("Failed to run prompt") 29 | } 30 | } 31 | 32 | export async function runAllPrompts() { 33 | // This function is intentionally left blank as it's not used in the provided code. 34 | // It's included to satisfy the missing export requirement. 35 | } 36 | -------------------------------------------------------------------------------- /packages/frontend/components.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://ui.shadcn.com/schema.json", 3 | "style": "new-york", 4 | "rsc": false, 5 | "tsx": true, 6 | "tailwind": { 7 | "config": "", 8 | "css": "styles/globals.css", 9 | "baseColor": "neutral", 10 | "cssVariables": true, 11 | "prefix": "" 12 | }, 13 | "aliases": { 14 | "components": "@/components", 15 | "utils": "@/lib/utils", 16 | "ui": "@/components/ui", 17 | "lib": "@/lib", 18 | "hooks": "@/hooks" 19 | }, 20 | "iconLibrary": "lucide" 21 | } -------------------------------------------------------------------------------- /packages/frontend/components/SmartViewer/MessageViewer.tsx: -------------------------------------------------------------------------------- 1 | import { ChatMessage } from "@/components/SmartViewer/Message"; 2 | import { Stack } from "@mantine/core"; 3 | import classes from "./index.module.css"; 4 | 5 | function getLastMessage(messages) { 6 | if (Array.isArray(messages)) { 7 | return messages[messages.length - 1]; 8 | } 9 | 10 | return messages; 11 | } 12 | 13 | export default function MessageViewer({ data, compact, piiDetection }) { 14 | const obj = Array.isArray(data) ? data : [data]; 15 | 16 | return compact ? ( 17 | 18 | ) : ( 19 |
20 | 21 | {obj.map((message, i) => ( 22 | 23 | ))} 24 | 25 |
26 | ); 27 | } 28 | -------------------------------------------------------------------------------- /packages/frontend/components/analytics/AgentSummary.tsx: -------------------------------------------------------------------------------- 1 | import AnalyticsCard from "./AnalyticsCard"; 2 | import BarList from "./BarList"; 3 | 4 | export default function AgentSummary({ usage }) { 5 | return ( 6 | 7 | u.type === "agent") 10 | .map((model) => ({ 11 | value: model.name, 12 | runs: model.success + model.errors, 13 | errors: model.errors, 14 | barSections: [ 15 | { 16 | value: "Success", 17 | tooltip: "Successful Runs", 18 | count: model.success, 19 | color: "green.3", 20 | }, 21 | { 22 | value: "Errors", 23 | tooltip: "Errors", 24 | count: model.errors, 25 | color: "red.4", 26 | }, 27 | ], 28 | }))} 29 | columns={[ 30 | { 31 | name: "Agent", 32 | bar: true, 33 | }, 34 | { 35 | name: "Runs", 36 | key: "runs", 37 | main: true, 38 | }, 39 | { 40 | name: "Errors", 41 | key: "errors", 42 | }, 43 | ]} 44 | /> 45 | 46 | ); 47 | } 48 | -------------------------------------------------------------------------------- /packages/frontend/components/analytics/TinyPercentChart.tsx: -------------------------------------------------------------------------------- 1 | /* 2 | * TODO everything, this is just a mockup 3 | */ 4 | 5 | import { AreaChart, Area, CartesianGrid, ResponsiveContainer } from "recharts"; 6 | 7 | export default function TinyPercentChart({ height, width, data, negative }) { 8 | return ( 9 | 10 | 11 | 12 | 13 | 20 | 27 | 28 | 29 | ); 30 | } 31 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/AppUserAvatar/index.module.css: -------------------------------------------------------------------------------- 1 | .anchor { 2 | max-width: min(100%, 300px); 3 | white-space: nowrap; 4 | overflow: hidden; 5 | text-overflow: ellipsis; 6 | font-size: 14px; 7 | } 8 | 9 | .anchor:hover { 10 | text-decoration: none; 11 | } 12 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/DurationBadge.tsx: -------------------------------------------------------------------------------- 1 | import { Badge, ThemeIcon } from "@mantine/core"; 2 | import { IconBolt, IconClock } from "@tabler/icons-react"; 3 | 4 | export default function DurationBadge({ 5 | cached = false, 6 | createdAt, 7 | endedAt, 8 | minimal = false, 9 | type, 10 | }) { 11 | const duration = endedAt 12 | ? new Date(endedAt).getTime() - new Date(createdAt).getTime() 13 | : NaN; 14 | 15 | if (type === "llm" && (cached || duration < 0.01 * 1000)) { 16 | return ( 17 | 24 | 25 | 26 | } 27 | > 28 | Cached ({(duration / 1000).toFixed(2)}s) 29 | 30 | ); 31 | } 32 | 33 | return ( 34 | 41 | 42 | 43 | } 44 | > 45 | {(duration / 1000).toFixed(2)}s 46 | 47 | ); 48 | } 49 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/ErrorBoundary.tsx: -------------------------------------------------------------------------------- 1 | import React, { ReactNode } from "react"; 2 | import * as Sentry from "@sentry/nextjs"; 3 | 4 | interface ErrorBoundaryState { 5 | hasError: boolean; 6 | } 7 | 8 | interface ErrorBoundaryProps { 9 | children: ReactNode; 10 | } 11 | 12 | class ErrorBoundary extends React.Component< 13 | ErrorBoundaryProps, 14 | ErrorBoundaryState 15 | > { 16 | constructor(props: ErrorBoundaryProps) { 17 | super(props); 18 | this.state = { hasError: false }; 19 | } 20 | 21 | static getDerivedStateFromError(error: Error): ErrorBoundaryState { 22 | return { hasError: true }; 23 | } 24 | 25 | componentDidCatch(error: Error, errorInfo: React.ErrorInfo) { 26 | Sentry.captureException(error); 27 | console.error(error, errorInfo); 28 | } 29 | 30 | render() { 31 | if (this.state.hasError) { 32 | return

Error rendering this component

; 33 | } 34 | 35 | return this.props.children; 36 | } 37 | } 38 | 39 | export default ErrorBoundary; 40 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/HotkeysInfo.tsx: -------------------------------------------------------------------------------- 1 | import { Kbd, Text } from "@mantine/core"; 2 | 3 | export default function HotkeysInfo({ hot, size, style = {} }) { 4 | const fz = size === "xs" ? 10 : 14; 5 | 6 | return ( 7 | 8 | 9 | ⌘ 10 | 11 | 12 | {` + `} 13 | 14 | 15 | {hot?.replace("Enter", "⏎")} 16 | 17 | 18 | ); 19 | } 20 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/Logo.tsx: -------------------------------------------------------------------------------- 1 | import { Anchor, Group, Text, useComputedColorScheme } from "@mantine/core"; 2 | 3 | export default function Logo() { 4 | const scheme = useComputedColorScheme(); 5 | 6 | return ( 7 | 12 | 13 | 18 | lunary 19 | 20 | 21 | ); 22 | } 23 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/OrgUserBadge.tsx: -------------------------------------------------------------------------------- 1 | import { useOrgUser } from "@/utils/dataHooks"; 2 | import { Badge } from "@mantine/core"; 3 | import UserAvatar from "./UserAvatar"; 4 | 5 | export default function OrgUserBadge({ userId }) { 6 | const { user, loading } = useOrgUser(userId); 7 | 8 | if (!userId || loading) { 9 | return null; 10 | } 11 | 12 | return ( 13 | } 21 | > 22 | {user?.name || "Unknown"} 23 | 24 | ); 25 | } 26 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/ProtectedText.tsx: -------------------------------------------------------------------------------- 1 | import { useOrg } from "@/utils/dataHooks"; 2 | import { Children, cloneElement } from "react"; 3 | 4 | export default function ProtectedText({ children }) { 5 | const { org } = useOrg(); 6 | const limited = org?.limited; 7 | 8 | const replaceText = (child) => { 9 | if (child && typeof child.props.children === "string") { 10 | return cloneElement(child, { 11 | children: child.props.children.replace(/\S/g, "X"), 12 | }); 13 | } 14 | if (child && Array.isArray(child.props.children)) { 15 | return cloneElement(child, { 16 | children: Children.map(child.props.children, replaceText), 17 | }); 18 | } 19 | return child; 20 | }; 21 | 22 | if (typeof children !== "string") { 23 | if (limited) { 24 | return Children.map(children, replaceText); 25 | } 26 | return children; 27 | } 28 | 29 | // create a string of fake characters same length and keep new lines 30 | const fakeChars = children.replace(/\S/g, "X"); 31 | 32 | return limited ? {fakeChars} : children; 33 | } 34 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/RenamableField.tsx: -------------------------------------------------------------------------------- 1 | import { FocusTrap, Group, TextInput, Title } from "@mantine/core"; 2 | import { IconPencil } from "@tabler/icons-react"; 3 | import { useState } from "react"; 4 | 5 | function RenamableField({ 6 | defaultValue, 7 | onRename, 8 | hidePencil = false, 9 | order = 4, 10 | ...props 11 | }: { 12 | defaultValue: string; 13 | onRename: (newName: string) => void; 14 | [key: string]: any; 15 | }) { 16 | const [focused, setFocused] = useState(false); 17 | 18 | const doRename = (e) => { 19 | setFocused(false); 20 | onRename(e.target.value); 21 | }; 22 | 23 | return focused ? ( 24 | 25 | { 32 | if (e.key === "Enter") doRename(e); 33 | }} 34 | onBlur={(e) => doRename(e)} 35 | /> 36 | 37 | ) : ( 38 | setFocused(true)} 41 | style={{ cursor: "pointer" }} 42 | {...props} 43 | > 44 | <Group> 45 | {defaultValue} 46 | {!hidePencil && <IconPencil size="14" />} 47 | </Group> 48 | 49 | ); 50 | } 51 | 52 | export default RenamableField; 53 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/SearchBar.tsx: -------------------------------------------------------------------------------- 1 | import { Input } from "@mantine/core"; 2 | import { useFocusWithin } from "@mantine/hooks"; 3 | import { IconSearch, IconX } from "@tabler/icons-react"; 4 | import HotkeysInfo from "./HotkeysInfo"; 5 | 6 | import { useGlobalShortcut } from "@/utils/hooks"; 7 | 8 | export default function SearchBar({ query, setQuery, ...props }) { 9 | const { ref, focused } = useFocusWithin(); 10 | 11 | useGlobalShortcut([ 12 | [ 13 | "mod+K", 14 | () => { 15 | if (ref.current?.focus) ref.current.focus(); 16 | }, 17 | ], 18 | ]); 19 | 20 | const showCross = query && query.length > 0; 21 | 22 | const clearInput = () => { 23 | setQuery(""); 24 | ref.current.value = ""; 25 | }; 26 | 27 | return ( 28 | } 30 | maw={400} 31 | w="30%" 32 | type="search" 33 | size="sm" 34 | ref={ref} 35 | id="search" 36 | rightSectionWidth={showCross ? 40 : 80} 37 | rightSectionPointerEvents="auto" 38 | rightSection={ 39 | showCross ? ( 40 | 41 | ) : !focused ? ( 42 | 47 | ) : null 48 | } 49 | placeholder="Type to filter" 50 | defaultValue={query} 51 | onChange={(e) => setQuery(e.currentTarget.value)} 52 | {...props} 53 | /> 54 | ); 55 | } 56 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/SeatAllowanceCard.tsx: -------------------------------------------------------------------------------- 1 | import { useOrg } from "@/utils/dataHooks"; 2 | import { SEAT_ALLOWANCE } from "@/utils/pricing"; 3 | import { Card, Progress, Stack, Text } from "@mantine/core"; 4 | import { SettingsCard } from "./SettingsCard"; 5 | 6 | export default function SeatAllowanceCard() { 7 | const { org } = useOrg(); 8 | 9 | if (!org?.plan) { 10 | return null; 11 | } 12 | 13 | const seatAllowance = org?.seatAllowance || SEAT_ALLOWANCE[org?.plan]; 14 | 15 | return ( 16 | 17 | 18 | {org?.users?.length} / {seatAllowance} users 19 | 20 | 26 | 27 | ); 28 | } 29 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/SettingsCard.tsx: -------------------------------------------------------------------------------- 1 | import { Card, Stack, Title } from "@mantine/core"; 2 | import Paywall, { PaywallConfig } from "../layout/Paywall"; 3 | 4 | export function SettingsCard({ 5 | title, 6 | children, 7 | align, 8 | paywallConfig, 9 | gap = "lg", 10 | }: { 11 | title; 12 | children: React.ReactNode; 13 | paywallConfig?: PaywallConfig; 14 | align?: string; 15 | gap?: string; 16 | }) { 17 | if (paywallConfig?.enabled) { 18 | return ( 19 | 20 | 21 | {title} 22 | 31 | {children} 32 | 33 | 34 | 35 | ); 36 | } 37 | 38 | return ( 39 | 40 | 41 | {title} 42 | {children} 43 | 44 | 45 | ); 46 | } 47 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/SocialProof.tsx: -------------------------------------------------------------------------------- 1 | import { Group, Avatar, Stack, Rating, Text } from "@mantine/core"; 2 | 3 | export default function SocialProof() { 4 | return ( 5 | 6 | 7 | {[ 8 | "https://lunary.ai/users/1.png", 9 | "https://lunary.ai/users/2.jpeg", 10 | "https://lunary.ai/users/3.jpeg", 11 | "https://lunary.ai/users/4.jpeg", 12 | ].map((src) => ( 13 | 14 | ))} 15 | 16 | 17 | 18 | 19 | 25 | 5000+ 26 | {" "} 27 | GenAI devs build better apps 28 | 29 | 30 | 31 | ); 32 | } 33 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/StatusBadge.tsx: -------------------------------------------------------------------------------- 1 | import { Badge, ThemeIcon } from "@mantine/core"; 2 | import { IconCheck, IconShieldBolt, IconX } from "@tabler/icons-react"; 3 | import ProtectedText from "./ProtectedText"; 4 | 5 | function getColor(status) { 6 | if (status === "success") { 7 | return "green"; 8 | } else if (status === "error") { 9 | return "red"; 10 | } else if (status === "filtered") { 11 | return "gray"; 12 | } else { 13 | return "blue"; 14 | } 15 | } 16 | 17 | function Icon({ status }) { 18 | if (status === "success") { 19 | return ; 20 | } else if (status === "filtered") { 21 | return ; 22 | } else { 23 | return ; 24 | } 25 | } 26 | 27 | export default function StatusBadge({ status, minimal = false }) { 28 | const color = getColor(status); 29 | 30 | if (minimal) 31 | return ( 32 | 33 | 34 | 35 | ); 36 | 37 | return ( 38 | 39 | {status} 40 | 41 | ); 42 | } 43 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/Steps.tsx: -------------------------------------------------------------------------------- 1 | import { Box, Group, Text, Title } from "@mantine/core"; 2 | 3 | function Steps({ children, ...props }) { 4 | return ( 5 | 11 | {children} 12 | 13 | ); 14 | } 15 | 16 | Steps.Step = ({ label, n, children }) => ( 17 | 18 | 19 |
37 | {n} 38 |
39 |
40 | 41 | {label} 42 | 43 | {children} 44 |
45 | 46 | 47 | ); 48 | 49 | export default Steps; 50 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/TokensBadge.tsx: -------------------------------------------------------------------------------- 1 | import { Badge, ThemeIcon } from "@mantine/core"; 2 | import { IconCashBanknote } from "@tabler/icons-react"; 3 | 4 | export default function TokensBadge({ 5 | tokens, 6 | cachedTokens, 7 | }: { 8 | tokens: number; 9 | cachedTokens?: number; 10 | }) { 11 | if (!tokens) return null; 12 | 13 | return ( 14 | 21 | 22 | 23 | } 24 | > 25 | {tokens} tokens {cachedTokens ? `(${cachedTokens} cached)` : ""} 26 | 27 | ); 28 | } 29 | -------------------------------------------------------------------------------- /packages/frontend/components/blocks/UserAvatar.tsx: -------------------------------------------------------------------------------- 1 | import { capitalize } from "@/utils/format"; 2 | import { Avatar, Text } from "@mantine/core"; 3 | import { memo } from "react"; 4 | 5 | function UserAvatar({ 6 | profile: user, 7 | size = "md", 8 | }: { 9 | profile: any; 10 | size?: string | number; 11 | }) { 12 | const text = user?.name || user?.email; 13 | return ( 14 | 23 | {text && ( 24 | 25 | {capitalize(text) 26 | ?.split(" ") 27 | .map((n) => n[0]) 28 | .slice(0, 2) 29 | .join("")} 30 | 31 | )} 32 | 33 | ); 34 | } 35 | 36 | export default memo(UserAvatar); 37 | -------------------------------------------------------------------------------- /packages/frontend/components/checks/MultiTextInput.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lunary-ai/lunary/27c09c35f982122a2d0b316a9cbbb97bd72711fe/packages/frontend/components/checks/MultiTextInput.tsx -------------------------------------------------------------------------------- /packages/frontend/components/checks/index.module.css: -------------------------------------------------------------------------------- 1 | .custom-input { 2 | display: flex; 3 | gap: 8px; 4 | 5 | background: var(--mantine-color-body); 6 | 7 | align-items: center; 8 | 9 | &.minimal { 10 | border: 1px solid var(--mantine-color-default-border); 11 | border-radius: var(--mantine-radius-default); 12 | padding: 0px 10px; 13 | > *:not(:last-of-type) { 14 | border-right: 1px solid var(--mantine-color-default-border); 15 | height: 100%; 16 | } 17 | } 18 | 19 | &:not(.minimal) { 20 | display: flex; 21 | 22 | flex-direction: column; 23 | align-items: flex-start; 24 | 25 | > *:not(:last-of-type) { 26 | font-size: 16px !important; 27 | } 28 | } 29 | } 30 | 31 | .input-label { 32 | padding-right: 8px; 33 | font-weight: 600; 34 | } 35 | 36 | .modal-content { 37 | padding: 24px; 38 | background-color: #f5f6f8; 39 | } 40 | 41 | .item { 42 | display: flex; 43 | flex-direction: column; 44 | align-items: center; 45 | justify-content: center; 46 | text-align: center; 47 | border-radius: 8px; 48 | height: 120px; 49 | background-color: light-dark( 50 | var(--mantine-color-white), 51 | var(--mantine-color-dark-7) 52 | ); 53 | transition: 54 | box-shadow 150ms ease, 55 | transform 100ms ease; 56 | } 57 | 58 | .footer-container { 59 | height: 70px; 60 | position: sticky; 61 | bottom: 0; 62 | left: 0; 63 | right: 0; 64 | z-index: 1; 65 | padding: 24px; 66 | 67 | justify-content: end; 68 | } 69 | -------------------------------------------------------------------------------- /packages/frontend/components/evals/index.module.css: -------------------------------------------------------------------------------- 1 | .matrix-container { 2 | overflow-x: scroll; 3 | } 4 | 5 | .matrix-table { 6 | border-collapse: collapse; 7 | border-spacing: 0; 8 | border: 1px solid var(--mantine-color-default-border); 9 | vertical-align: middle; 10 | 11 | th { 12 | background: var(--mantine-color-body); 13 | } 14 | 15 | th, 16 | td { 17 | border: 1px solid var(--mantine-color-default-border); 18 | padding: 16px; 19 | text-align: center; 20 | vertical-align: middle; 21 | } 22 | 23 | tr { 24 | height: 1px; 25 | } 26 | 27 | /* first col excluding nested cells */ 28 | > tbody > tr > td:first-of-type { 29 | min-width: 400px; 30 | } 31 | 32 | td.output-cell { 33 | width: 700px; 34 | min-width: 500px; 35 | 36 | text-align: left !important; 37 | } 38 | 39 | td.nested-cell { 40 | padding: 0; 41 | 42 | height: 1px; 43 | } 44 | 45 | td > table { 46 | height: 100%; 47 | width: 100%; 48 | table-layout: fixed; 49 | border-collapse: collapse; 50 | border: none; 51 | vertical-align: middle; 52 | 53 | td { 54 | border-top: none; 55 | border-bottom: none; 56 | } 57 | 58 | tr td:first-child { 59 | border-left: none; 60 | } 61 | 62 | tr td:last-child { 63 | border-right: none; 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /packages/frontend/components/layout/Analytics.tsx: -------------------------------------------------------------------------------- 1 | import { useRouter } from "next/router"; 2 | import { useEffect } from "react"; 3 | 4 | import { PostHogProvider } from "posthog-js/react"; 5 | 6 | import posthog from "posthog-js"; 7 | 8 | import analytics from "@/utils/analytics"; 9 | 10 | export default function AnalyticsWrapper({ children }) { 11 | const router = useRouter(); 12 | 13 | useEffect(() => { 14 | analytics.handleRouteChange(); 15 | 16 | router.events.on("routeChangeComplete", analytics.handleRouteChange); 17 | return () => { 18 | router.events.off("routeChangeComplete", analytics.handleRouteChange); 19 | }; 20 | }, []); 21 | 22 | return process.env.NEXT_PUBLIC_POSTHOG_KEY ? ( 23 | {children} 24 | ) : ( 25 | children 26 | ); 27 | } 28 | -------------------------------------------------------------------------------- /packages/frontend/components/layout/AuthLayout.tsx: -------------------------------------------------------------------------------- 1 | import { Box, Center, Container, useComputedColorScheme } from "@mantine/core"; 2 | import { IconAnalyze } from "@tabler/icons-react"; 3 | 4 | export default function AuthLayout({ 5 | children, 6 | }: { 7 | children: React.ReactNode; 8 | }) { 9 | const scheme = useComputedColorScheme(); 10 | 11 | return ( 12 | 21 | 22 | 30 | 31 |
32 | {children} 33 |
34 | 35 | ); 36 | } 37 | -------------------------------------------------------------------------------- /packages/frontend/components/layout/sidebar.module.css: -------------------------------------------------------------------------------- 1 | .closeButton { 2 | position: absolute; 3 | right: 8px; 4 | top: 8px; 5 | } -------------------------------------------------------------------------------- /packages/frontend/components/prompts/ModelSelect.module.css: -------------------------------------------------------------------------------- 1 | .group-label:first-child { 2 | padding-top: 0.5rem; 3 | } 4 | 5 | .group-label { 6 | font-size: 0.8rem; 7 | font-weight: 500; 8 | padding-top: 1rem; 9 | padding-bottom: 0.2rem; 10 | 11 | &::after { 12 | content: ""; 13 | display: none !important; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /packages/frontend/components/prompts/TemplateInputArea.tsx: -------------------------------------------------------------------------------- 1 | import { Text, ScrollArea, Group, Box } from "@mantine/core"; 2 | import SmartViewer from "@/components/SmartViewer"; 3 | import TokensBadge from "../blocks/TokensBadge"; 4 | import { PromptEditor } from "./PromptEditor"; 5 | 6 | function TemplateInputArea({ 7 | template, 8 | setTemplate, 9 | saveTemplate, 10 | setHasChanges, 11 | output, 12 | outputTokens, 13 | error, 14 | }: { 15 | template: any; 16 | setTemplate: (template: any) => void; 17 | saveTemplate: () => void; 18 | setHasChanges: (hasChanges: boolean) => void; 19 | output: any; 20 | outputTokens: number; 21 | error: any; 22 | }) { 23 | const isText = typeof template?.content === "string"; 24 | const handleContentChange = (newContent: any) => { 25 | setTemplate({ ...template, content: newContent }); 26 | setHasChanges(true); 27 | }; 28 | 29 | return ( 30 | 31 | 36 | {(output || error) && ( 37 | <> 38 | 39 | 40 | {error ? "Error" : "Output"} 41 | 42 | {outputTokens && } 43 | 44 | 45 | 46 | )} 47 | 48 | ); 49 | } 50 | 51 | export default TemplateInputArea; 52 | -------------------------------------------------------------------------------- /packages/frontend/components/providers/ProviderCard.module.css: -------------------------------------------------------------------------------- 1 | .card { 2 | transition: all 300ms cubic-bezier(0.4, 0, 0.2, 1); 3 | 4 | &:hover { 5 | transform: translateY(-0.25rem); 6 | box-shadow: 7 | 0 4px 6px -1px rgb(0 0 0 / 0.1), 8 | 0 2px 4px -2px rgb(0 0 0 / 0.1); 9 | } 10 | } 11 | 12 | .button { 13 | &:disabled, 14 | &[data-disabled] { 15 | background-color: black; 16 | color: white; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /packages/frontend/components/ui/toaster.tsx: -------------------------------------------------------------------------------- 1 | import { useToast } from "@/hooks/use-toast" 2 | import { 3 | Toast, 4 | ToastClose, 5 | ToastDescription, 6 | ToastProvider, 7 | ToastTitle, 8 | ToastViewport, 9 | } from "@/components/ui/toast" 10 | 11 | export function Toaster() { 12 | const { toasts } = useToast() 13 | 14 | return ( 15 | 16 | {toasts.map(function ({ id, title, description, action, ...props }) { 17 | return ( 18 | 19 |
20 | {title && {title}} 21 | {description && ( 22 | {description} 23 | )} 24 |
25 | {action} 26 | 27 | 28 | ) 29 | })} 30 | 31 | 32 | ) 33 | } 34 | -------------------------------------------------------------------------------- /packages/frontend/hooks/use-keyboard-shortcut.ts: -------------------------------------------------------------------------------- 1 | "use client" 2 | 3 | import { useEffect } from "react" 4 | 5 | export function useKeyboardShortcut(keys: string[], callback: () => void, node = globalThis) { 6 | useEffect(() => { 7 | const handler = (e: KeyboardEvent) => { 8 | if ( 9 | keys.every((key) => { 10 | if (key === "meta") return e.metaKey 11 | if (key === "ctrl") return e.ctrlKey 12 | if (key === "alt") return e.altKey 13 | if (key === "shift") return e.shiftKey 14 | return e.key.toLowerCase() === key.toLowerCase() 15 | }) 16 | ) { 17 | e.preventDefault() 18 | callback() 19 | } 20 | } 21 | 22 | node.addEventListener("keydown", handler) 23 | return () => node.removeEventListener("keydown", handler) 24 | }, [callback, keys, node]) 25 | } 26 | -------------------------------------------------------------------------------- /packages/frontend/hooks/use-notifications.ts: -------------------------------------------------------------------------------- 1 | "use client" 2 | 3 | import { notifications } from "@mantine/notifications" 4 | 5 | type NotificationProps = { 6 | title?: string 7 | description?: string 8 | variant?: "default" | "destructive" 9 | } 10 | 11 | export function useNotifications() { 12 | const notify = ({ title, description, variant = "default" }: NotificationProps) => { 13 | notifications.show({ 14 | title, 15 | message: description, 16 | color: variant === "destructive" ? "red" : "blue", 17 | }) 18 | } 19 | 20 | return { 21 | notify, 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /packages/frontend/instrumentation-client.ts: -------------------------------------------------------------------------------- 1 | // This file configures the initialization of Sentry on the client. 2 | // The added config here will be used whenever a users loads a page in their browser. 3 | // https://docs.sentry.io/platforms/javascript/guides/nextjs/ 4 | 5 | import * as Sentry from "@sentry/nextjs"; 6 | 7 | Sentry.init({ 8 | dsn: process.env.NEXT_PUBLIC_SENTRY_DSN, 9 | }); 10 | 11 | export const onRouterTransitionStart = Sentry.captureRouterTransitionStart; 12 | export const onRequestError = Sentry.captureRequestError; 13 | -------------------------------------------------------------------------------- /packages/frontend/lib/utils.ts: -------------------------------------------------------------------------------- 1 | import { clsx, type ClassValue } from "clsx" 2 | import { twMerge } from "tailwind-merge" 3 | 4 | export function cn(...inputs: ClassValue[]) { 5 | return twMerge(clsx(inputs)) 6 | } 7 | -------------------------------------------------------------------------------- /packages/frontend/load-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script is necessary because we need to be able to inject API_URL after build time, and Next does not provide and easy way to do that. 3 | # This should work find both with npm run dev, locally and with docker 4 | 5 | # Loads .env 6 | if [ -f .env ]; then 7 | set -o allexport 8 | source .env 9 | set +o allexport 10 | fi 11 | 12 | 13 | if [ -z "$API_URL" ]; then 14 | echo "Error: API_URL not set. Please set the API_URL environment variables." 15 | exit 1 16 | fi 17 | 18 | LC_ALL=C find .next -type f -exec perl -pi -e "s|xyzPLACEHOLDERxyz|${API_URL}|g" {} + 19 | 20 | if [ -n "$GOOGLE_CLIENT_ID" ]; then 21 | LC_ALL=C find .next -type f -exec perl -pi -e "s|xyzGOOGLECLIENTIDxyz|${GOOGLE_CLIENT_ID}|g" {} + 22 | fi 23 | -------------------------------------------------------------------------------- /packages/frontend/next-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | /// 3 | /// 4 | 5 | // NOTE: This file should not be edited 6 | // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. 7 | -------------------------------------------------------------------------------- /packages/frontend/pages/_document.tsx: -------------------------------------------------------------------------------- 1 | import { Html, Head, Main, NextScript } from "next/document"; 2 | import { ColorSchemeScript } from "@mantine/core"; 3 | 4 | export default function Document() { 5 | return ( 6 | 7 | 8 | 9 | 10 | 11 |
12 | 13 | 14 | 15 | ); 16 | } 17 | -------------------------------------------------------------------------------- /packages/frontend/pages/_error.tsx: -------------------------------------------------------------------------------- 1 | import * as Sentry from "@sentry/nextjs"; 2 | import Error from "next/error"; 3 | 4 | const CustomErrorComponent = (props) => { 5 | return ; 6 | }; 7 | 8 | CustomErrorComponent.getInitialProps = async (contextData) => { 9 | // In case this is running in a serverless function, await this in order to give Sentry 10 | // time to send the error before the lambda exits 11 | await Sentry.captureUnderscoreErrorException(contextData); 12 | 13 | // This will contain the status code of the response 14 | return Error.getInitialProps(contextData); 15 | }; 16 | 17 | export default CustomErrorComponent; 18 | -------------------------------------------------------------------------------- /packages/frontend/pages/alerts/[id]/edit.tsx: -------------------------------------------------------------------------------- 1 | import { useEffect } from "react"; 2 | import { useRouter } from "next/router"; 3 | 4 | const EditAlertPage = () => { 5 | const router = useRouter(); 6 | const { id } = router.query; 7 | useEffect(() => { 8 | if (typeof id === "string") { 9 | router.replace(`/alerts?modal=edit&id=${id}`, undefined, { 10 | shallow: true, 11 | }); 12 | } 13 | }, [id, router]); 14 | return null; 15 | }; 16 | 17 | export default EditAlertPage; 18 | -------------------------------------------------------------------------------- /packages/frontend/pages/alerts/new.tsx: -------------------------------------------------------------------------------- 1 | import { useEffect } from "react"; 2 | import { useRouter } from "next/router"; 3 | 4 | const NewAlertPage = () => { 5 | const router = useRouter(); 6 | useEffect(() => { 7 | router.replace("/alerts?modal=create", undefined, { shallow: true }); 8 | }, [router]); 9 | return null; 10 | }; 11 | 12 | export default NewAlertPage; 13 | -------------------------------------------------------------------------------- /packages/frontend/pages/api/app/[id].ts: -------------------------------------------------------------------------------- 1 | import { NextApiRequest, NextApiResponse } from "next"; 2 | 3 | // TODO: put this on the backend (used for legacy LLMonitorCallbackHandler python) 4 | export default async function handler( 5 | req: NextApiRequest, 6 | res: NextApiResponse, 7 | ) { 8 | return res.status(200).json("ok"); 9 | } 10 | -------------------------------------------------------------------------------- /packages/frontend/pages/api/run-prompt.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next" 2 | import { OpenAI } from "openai" 3 | 4 | export default async function handler(req: NextApiRequest, res: NextApiResponse) { 5 | if (req.method !== "POST") { 6 | res.setHeader("Allow", ["POST"]) 7 | return res.status(405).end(`Method ${req.method} Not Allowed`) 8 | } 9 | 10 | try { 11 | const { version, userMessage, context } = req.body 12 | 13 | if (!version || !userMessage) { 14 | return res.status(400).json({ error: "Missing required parameters" }) 15 | } 16 | 17 | // Initialize the OpenAI client inside the API route handler 18 | const openai = new OpenAI({ 19 | apiKey: process.env.OPENAI_API_KEY, 20 | }) 21 | 22 | const prompt = `${userMessage}\n\nContext: ${context || ""}` 23 | 24 | const response = await openai.chat.completions.create({ 25 | model: version.model, 26 | messages: [ 27 | { 28 | role: "system", 29 | content: version.systemPrompt, 30 | }, 31 | { 32 | role: "user", 33 | content: prompt, 34 | }, 35 | ], 36 | temperature: version.temperature, 37 | max_tokens: version.max_tokens, 38 | top_p: version.top_p, 39 | }) 40 | 41 | return res.status(200).json({ response: response.choices[0]?.message?.content || "No response generated" }) 42 | } catch (error) { 43 | console.error("Error running prompt:", error) 44 | return res.status(500).json({ error: "Failed to run prompt" }) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /packages/frontend/pages/billing/thank-you.tsx: -------------------------------------------------------------------------------- 1 | import { Anchor, Center, Container, Stack, Text, Title } from "@mantine/core"; 2 | import { NextSeo } from "next-seo"; 3 | import Confetti from "react-confetti"; 4 | 5 | export default function ThankYou() { 6 | return ( 7 | 8 | 9 | {typeof window !== "undefined" && ( 10 | 17 | )} 18 | 19 |
20 | 21 | 🎉 22 | 23 | {`You're all set.`} 24 | 25 | Thank you for your upgrade. You will receive an email shortly with 26 | your receipt. 27 | 28 | 29 | 30 | 31 | Schedule a call 32 | {" "} 33 | with us at any time. 34 | 35 | ← Back to my projects 36 | 37 |
38 | 39 | ); 40 | } 41 | -------------------------------------------------------------------------------- /packages/frontend/pages/dashboards/index.tsx: -------------------------------------------------------------------------------- 1 | import { useProject } from "@/utils/dataHooks"; 2 | import { useDashboards } from "@/utils/dataHooks/dashboards"; 3 | import { Flex, Loader } from "@mantine/core"; 4 | import { useRouter } from "next/router"; 5 | import { useEffect } from "react"; 6 | 7 | export default function Dashboards() { 8 | const { dashboards, isLoading } = useDashboards(); 9 | const { project } = useProject(); 10 | const router = useRouter(); 11 | 12 | useEffect(() => { 13 | if (project?.homeDashboardId) { 14 | router.push(`/dashboards/${project.homeDashboardId}`); 15 | return; 16 | } 17 | if (dashboards.length && !isLoading) { 18 | const homeDashboard = dashboards.find((dashboard) => dashboard.isHome); 19 | if (!homeDashboard) { 20 | router.push(`/dashboards/${dashboards[0].id}`); 21 | return; 22 | } else { 23 | router.push(`/dashboards/${homeDashboard.id}`); 24 | return; 25 | } 26 | } 27 | }, [project, dashboards, isLoading]); 28 | 29 | return ( 30 | 31 | 32 | 33 | ); 34 | } 35 | -------------------------------------------------------------------------------- /packages/frontend/pages/index.tsx: -------------------------------------------------------------------------------- 1 | import { useAuth } from "@/utils/auth"; 2 | import { useUser } from "@/utils/dataHooks"; 3 | import { Center, Loader } from "@mantine/core"; 4 | import { useRouter } from "next/router"; 5 | import { useEffect } from "react"; 6 | import { hasAccess } from "shared"; 7 | 8 | function IndexPage() { 9 | const router = useRouter(); 10 | const { isSignedIn } = useAuth(); 11 | const { user } = useUser(); 12 | 13 | useEffect(() => { 14 | if (!router.isReady) { 15 | return; 16 | } 17 | 18 | if (!isSignedIn) { 19 | router.replace("/login"); 20 | return; 21 | } 22 | 23 | if (hasAccess(user.role, "analytics", "read")) { 24 | router.replace("/dashboards"); 25 | } else { 26 | router.replace("/prompts"); 27 | } 28 | }, [user, router.isReady]); 29 | 30 | return ( 31 |
32 | 33 |
34 | ); 35 | } 36 | 37 | export default IndexPage; 38 | -------------------------------------------------------------------------------- /packages/frontend/pages/logs/[id].tsx: -------------------------------------------------------------------------------- 1 | import { Button, Container, Group, Loader, Stack, Text } from "@mantine/core"; 2 | 3 | import RunInputOutput from "@/components/blocks/RunInputOutput"; 4 | import Logo from "@/components/blocks/Logo"; 5 | 6 | import useSWR from "swr"; 7 | import { useRouter } from "next/router"; 8 | import { useAuth } from "@/utils/auth"; 9 | import Link from "next/link"; 10 | 11 | export default function PublicRun() { 12 | const router = useRouter(); 13 | const id = router.query?.id as string; 14 | 15 | const { isSignedIn } = useAuth(); 16 | 17 | const { data, isLoading, error } = useSWR( 18 | id && `/runs/${id}${isSignedIn ? "" : `/public`}`, 19 | ); 20 | 21 | return ( 22 | 23 | 24 | 25 | 26 | {isSignedIn ? ( 27 | 30 | ) : ( 31 | The developer toolkit for LLM apps. 32 | )} 33 | 34 | {isLoading && } 35 | {error && Could not get this log, it might not be public.} 36 | {data && ( 37 | 43 | )} 44 | 45 | 46 | ); 47 | } 48 | -------------------------------------------------------------------------------- /packages/frontend/pages/maintenance.tsx: -------------------------------------------------------------------------------- 1 | // pages/maintenance.tsx 2 | import { useRouter } from "next/router"; 3 | import React, { useEffect } from "react"; 4 | 5 | export default function Maintenance() { 6 | const router = useRouter(); 7 | useEffect(() => { 8 | if (process.env.NEXT_PUBLIC_MAINTENANCE_MODE !== "on") { 9 | router.push("/"); 10 | } 11 | }, [router]); 12 | 13 | return ( 14 |
15 |

We're currently undergoing maintenance

16 |

17 | Rest assured, all your new events are safely stored and will be 18 | accessible once we're back. 19 |

20 |
21 | ); 22 | } 23 | -------------------------------------------------------------------------------- /packages/frontend/pages/settings/providers/index.tsx: -------------------------------------------------------------------------------- 1 | import { ProviderCard } from "@/components/providers/ProviderCard"; 2 | import { useProviderConfigs } from "@/utils/dataHooks/provider-configs"; 3 | import { Container, SimpleGrid, Text, Title } from "@mantine/core"; 4 | 5 | export default function ProvidersManager() { 6 | const { configuredProviders, isLoading } = useProviderConfigs(); 7 | 8 | return ( 9 | 10 | LLM Providers 11 | 12 | Set up and manage your own LLM providers for use in the Prompt 13 | Playground. 14 | 15 | 16 | 17 | {configuredProviders.map((configuredProvider) => ( 18 | 22 | ))} 23 | 24 | 25 | ); 26 | } 27 | -------------------------------------------------------------------------------- /packages/frontend/pages/team/team.module.css: -------------------------------------------------------------------------------- 1 | .pillsList { 2 | flex-wrap: nowrap; 3 | overflow: hidden; 4 | text-overflow: ellipsis; 5 | white-space: nowrap; 6 | } 7 | -------------------------------------------------------------------------------- /packages/frontend/pages/test.module.css: -------------------------------------------------------------------------------- 1 | .gridContainer { 2 | display: flex; 3 | flex-wrap: wrap; 4 | gap: var(--mantine-spacing-sm); 5 | } 6 | 7 | .item { 8 | display: flex; 9 | align-items: center; 10 | border-radius: var(--mantine-radius-md); 11 | border: 1px solid var(--mantine-color-gray-2); 12 | padding: var(--mantine-spacing-sm) var(--mantine-spacing-xl); 13 | background-color: var(--mantine-color-white); 14 | width: calc(33.333% - var(--mantine-spacing-sm * 2)); 15 | box-sizing: border-box; 16 | } 17 | 18 | .itemDragging { 19 | box-shadow: var(--mantine-shadow-sm); 20 | } 21 | 22 | .symbol { 23 | font-size: 30px; 24 | font-weight: 700; 25 | width: 60px; 26 | } -------------------------------------------------------------------------------- /packages/frontend/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | "@tailwindcss/postcss": {}, 4 | "postcss-preset-mantine": {}, 5 | "postcss-simple-vars": { 6 | variables: { 7 | "mantine-breakpoint-xs": "36em", 8 | "mantine-breakpoint-sm": "48em", 9 | "mantine-breakpoint-md": "62em", 10 | "mantine-breakpoint-lg": "75em", 11 | "mantine-breakpoint-xl": "88em", 12 | }, 13 | }, 14 | }, 15 | }; 16 | -------------------------------------------------------------------------------- /packages/frontend/postcss.config.mjs: -------------------------------------------------------------------------------- 1 | const config = { 2 | plugins: { 3 | 'postcss-preset-mantine': {}, 4 | }, 5 | }; 6 | 7 | export default config; 8 | -------------------------------------------------------------------------------- /packages/frontend/public/assets/accepted-cards.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lunary-ai/lunary/27c09c35f982122a2d0b316a9cbbb97bd72711fe/packages/frontend/public/assets/accepted-cards.webp -------------------------------------------------------------------------------- /packages/frontend/public/assets/amazon-redshift.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /packages/frontend/public/assets/apple-pay.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lunary-ai/lunary/27c09c35f982122a2d0b316a9cbbb97bd72711fe/packages/frontend/public/assets/apple-pay.webp -------------------------------------------------------------------------------- /packages/frontend/public/assets/bigquery.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/frontend/public/assets/databricks.svg: -------------------------------------------------------------------------------- 1 | 2 | download (2)-svg 3 | 6 | 7 | -------------------------------------------------------------------------------- /packages/frontend/public/assets/github-icon-black.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/frontend/public/assets/github-icon-white.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/frontend/public/assets/google-icon.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /packages/frontend/public/assets/google-pay.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lunary-ai/lunary/27c09c35f982122a2d0b316a9cbbb97bd72711fe/packages/frontend/public/assets/google-pay.webp -------------------------------------------------------------------------------- /packages/frontend/public/fonts/circular-pro-black.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lunary-ai/lunary/27c09c35f982122a2d0b316a9cbbb97bd72711fe/packages/frontend/public/fonts/circular-pro-black.woff2 -------------------------------------------------------------------------------- /packages/frontend/public/fonts/circular-pro-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lunary-ai/lunary/27c09c35f982122a2d0b316a9cbbb97bd72711fe/packages/frontend/public/fonts/circular-pro-bold.woff2 -------------------------------------------------------------------------------- /packages/frontend/public/fonts/circular-pro-book.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lunary-ai/lunary/27c09c35f982122a2d0b316a9cbbb97bd72711fe/packages/frontend/public/fonts/circular-pro-book.woff2 -------------------------------------------------------------------------------- /packages/frontend/public/fonts/circular-pro-medium.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lunary-ai/lunary/27c09c35f982122a2d0b316a9cbbb97bd72711fe/packages/frontend/public/fonts/circular-pro-medium.woff2 -------------------------------------------------------------------------------- /packages/frontend/store/prompt-versions.ts: -------------------------------------------------------------------------------- 1 | import type { PromptVersion } from "@/types/prompt-types" 2 | 3 | // Predefined prompt versions (simulating DB storage) 4 | export const predefinedPromptVersions: PromptVersion[] = [ 5 | { 6 | id: "1", 7 | name: "Support Assistant v1", 8 | systemPrompt: 9 | "You are an AI-powered Aircall Customer Support Assistant. Your role is to provide accurate, helpful, and concise responses to user queries about Aircall's products and services.", 10 | model: "gpt-4o", 11 | temperature: 1.0, 12 | max_tokens: 2048, 13 | top_p: 1.0, 14 | }, 15 | { 16 | id: "2", 17 | name: "Support Assistant v2", 18 | systemPrompt: 19 | "You are an AI-powered Aircall Customer Support Assistant. Your role is to provide accurate, helpful, and concise responses to user queries about Aircall's products and services. Focus on being friendly and personable in your responses.", 20 | model: "gpt-4o", 21 | temperature: 1.0, 22 | max_tokens: 2048, 23 | top_p: 1.0, 24 | }, 25 | { 26 | id: "3", 27 | name: "Support Assistant v3", 28 | systemPrompt: 29 | "You are an AI-powered Aircall Customer Support Assistant. Your role is to provide accurate, helpful, and concise responses to user queries about Aircall's products and services. Be direct and to the point, focusing on technical accuracy above all else.", 30 | model: "gpt-4o", 31 | temperature: 1.0, 32 | max_tokens: 2048, 33 | top_p: 1.0, 34 | }, 35 | ] 36 | -------------------------------------------------------------------------------- /packages/frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "lib": ["dom", "dom.iterable", "esnext"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "strict": false, 8 | "forceConsistentCasingInFileNames": true, 9 | "noEmit": true, 10 | "esModuleInterop": true, 11 | "module": "esnext", 12 | "moduleResolution": "bundler", 13 | "resolveJsonModule": true, 14 | "isolatedModules": true, 15 | "jsx": "preserve", 16 | "baseUrl": ".", 17 | "incremental": true, 18 | "paths": { 19 | "@/*": ["./*"], 20 | "@/lib/*": ["lib/*"], 21 | "@/components/*": ["components/*"] 22 | }, 23 | "plugins": [ 24 | { 25 | "name": "next" 26 | } 27 | ], 28 | "strictNullChecks": true 29 | }, 30 | "include": [ 31 | "next-env.d.ts", 32 | "**/*.ts", 33 | "**/*.tsx", 34 | ".next/types/**/*.ts", 35 | "../backend/src/api/stripe.ts", 36 | "../backend/src/api/signup.ts" 37 | ], 38 | "exclude": ["node_modules"] 39 | } 40 | -------------------------------------------------------------------------------- /packages/frontend/types/evaluator-types.ts: -------------------------------------------------------------------------------- 1 | export interface EvaluatorConfig { 2 | id: string 3 | name: string 4 | description: string 5 | enabled: boolean 6 | parameters: Record 7 | parameterDefinitions: EvaluatorParameterDefinition[] 8 | } 9 | 10 | export interface EvaluatorParameterDefinition { 11 | id: string 12 | name: string 13 | description: string 14 | type: "string" | "number" | "boolean" | "select" 15 | default: any 16 | options?: { label: string; value: any }[] 17 | min?: number 18 | max?: number 19 | } 20 | 21 | export interface EvaluationResult { 22 | evaluatorId: string 23 | score: number 24 | feedback: string 25 | metadata?: Record 26 | } 27 | 28 | export interface ComparisonRow { 29 | [key: string]: any // Define ComparisonRow as an interface 30 | } 31 | 32 | // Add evaluation results to the comparison row 33 | export interface ComparisonRowWithEvaluation extends ComparisonRow { 34 | evaluationResults: Record> 35 | // Format: { columnId: { evaluatorId: EvaluationResult } } 36 | } 37 | -------------------------------------------------------------------------------- /packages/frontend/utils/config.ts: -------------------------------------------------------------------------------- 1 | const config = { 2 | IS_SELF_HOSTED: 3 | process.env.NEXT_PUBLIC_IS_SELF_HOSTED === "true" ? true : false, 4 | IS_CLOUD: 5 | process.env.NEXT_PUBLIC_IS_SELF_HOSTED !== "true" && 6 | process.env.NODE_ENV === "production", 7 | RECAPTCHA_SITE_KEY: "6LeDTNsqAAAAANXZDQATUpJ8s1vttWkZabP6g3O6", 8 | }; 9 | 10 | export default config; 11 | -------------------------------------------------------------------------------- /packages/frontend/utils/context.ts: -------------------------------------------------------------------------------- 1 | import { createContext } from "react"; 2 | 3 | type ProjectContextType = { 4 | projectId?: string; 5 | setProjectId: (appId: string | null) => void; 6 | }; 7 | 8 | export const ProjectContext = createContext({ 9 | projectId: undefined, 10 | setProjectId: () => {}, 11 | }); 12 | -------------------------------------------------------------------------------- /packages/frontend/utils/dataHooks/audit-logs.ts: -------------------------------------------------------------------------------- 1 | import { useContext } from "react"; 2 | import useSWRInfinite from "swr/infinite"; 3 | import { ProjectContext } from "../context"; 4 | 5 | export function useAuditLogs() { 6 | const { projectId } = useContext(ProjectContext); 7 | const PAGE_SIZE = 10; 8 | 9 | function getKey(pageIndex, previousPageData) { 10 | if (previousPageData && !previousPageData.length) return null; 11 | return `/audit-logs?project_id=${projectId}&page=${pageIndex}&limit=${PAGE_SIZE}`; 12 | } 13 | 14 | const { data, isLoading, size, setSize } = useSWRInfinite(getKey); 15 | const items = data?.length ? data[size - 1] || [] : []; 16 | const hasMore = items.length === PAGE_SIZE; 17 | 18 | return { auditLogs: items, isLoading, page: size, setPage: setSize, hasMore }; 19 | } 20 | -------------------------------------------------------------------------------- /packages/frontend/utils/dataHooks/charts.ts: -------------------------------------------------------------------------------- 1 | import { hasAccess } from "shared"; 2 | import { useProjectMutation, useProjectSWR, useUser } from "."; 3 | import { fetcher } from "../fetcher"; 4 | 5 | export function useCharts() { 6 | const { user } = useUser(); 7 | const { data, isLoading, mutate } = useProjectSWR( 8 | hasAccess(user?.role, "charts", "list") ? `/charts` : null, 9 | ); 10 | 11 | const { trigger: insert, isMutating: isInserting } = useProjectMutation( 12 | `/charts`, 13 | fetcher.post, 14 | ); 15 | 16 | return { 17 | charts: data, 18 | insert, 19 | isInserting, 20 | mutate, 21 | loading: isLoading, 22 | }; 23 | } 24 | 25 | export function useChart(id: string | null, initialData?: any) { 26 | const { mutate: mutateViews } = useCharts(); 27 | 28 | const { 29 | data: chart, 30 | isLoading, 31 | mutate, 32 | } = useProjectSWR(id && `/charts/${id}`, { 33 | fallbackData: initialData, 34 | }); 35 | 36 | const { trigger: update } = useProjectMutation( 37 | id && `/charts/${id}`, 38 | fetcher.patch, 39 | { 40 | onSuccess(data) { 41 | mutate(data); 42 | mutateViews(); 43 | }, 44 | }, 45 | ); 46 | 47 | const { trigger: remove } = useProjectMutation( 48 | id && `/charts/${id}`, 49 | fetcher.delete, 50 | { 51 | revalidate: false, 52 | onSuccess() { 53 | mutateViews(); 54 | }, 55 | }, 56 | ); 57 | 58 | return { 59 | chart, 60 | update, 61 | remove, 62 | mutate, 63 | loading: isLoading, 64 | }; 65 | } 66 | -------------------------------------------------------------------------------- /packages/frontend/utils/dataHooks/data-warehouse.ts: -------------------------------------------------------------------------------- 1 | import { useProjectMutation, useProjectSWR } from "."; 2 | import { fetcher } from "../fetcher"; 3 | 4 | export function useBigQuery() { 5 | const { data, isLoading, mutate } = useProjectSWR("/data-warehouse/bigquery"); 6 | const { trigger: insertMutation } = useProjectMutation( 7 | `/data-warehouse/bigquery`, 8 | fetcher.post, 9 | ); 10 | 11 | async function insert(apiKey: string) { 12 | await insertMutation({ apiKey }); 13 | await mutate(); 14 | } 15 | 16 | return { insert, connector: data, isLoading }; 17 | } 18 | -------------------------------------------------------------------------------- /packages/frontend/utils/dataHooks/external-users.ts: -------------------------------------------------------------------------------- 1 | import { useProjectInfiniteSWR, useProjectSWR } from "."; 2 | import { useSortParams } from "../hooks"; 3 | 4 | export function useExternalUsers({ 5 | startDate, 6 | endDate, 7 | search, 8 | checks, 9 | }: { 10 | startDate?: Date; 11 | endDate?: Date; 12 | search?: string | null; 13 | checks?: string; 14 | }) { 15 | const queryParams = new URLSearchParams(); 16 | if (startDate && endDate) { 17 | const timeZone = new window.Intl.DateTimeFormat().resolvedOptions() 18 | .timeZone; 19 | queryParams.append("startDate", startDate.toISOString()); 20 | queryParams.append("endDate", endDate.toISOString()); 21 | queryParams.append("timeZone", timeZone); 22 | } 23 | 24 | if (search) { 25 | queryParams.append("search", search); 26 | } 27 | 28 | if (checks) { 29 | queryParams.append("checks", checks); 30 | } 31 | 32 | const { sortParams } = useSortParams(); 33 | const { 34 | data, 35 | isLoading: loading, 36 | isValidating: validating, 37 | loadMore, 38 | } = useProjectInfiniteSWR( 39 | `/external-users?${queryParams.toString()}&${sortParams}`, 40 | ); 41 | 42 | return { users: data, loading, validating, loadMore }; 43 | } 44 | 45 | export function useExternalUsersProps() { 46 | const { data, isLoading } = useProjectSWR( 47 | "/external-users/props/keys", 48 | ); 49 | return { props: data || [], isLoading }; 50 | } 51 | -------------------------------------------------------------------------------- /packages/frontend/utils/dataHooks/models.ts: -------------------------------------------------------------------------------- 1 | import { useProjectMutation, useProjectSWR } from "."; 2 | import { fetcher } from "../fetcher"; 3 | 4 | export function useModelMappings() { 5 | const { data, isLoading, mutate } = useProjectSWR(`/models`); 6 | 7 | const { trigger: insert, isMutating: isInserting } = useProjectMutation( 8 | `/models`, 9 | fetcher.post, 10 | { 11 | onSuccess: () => { 12 | mutate(); 13 | }, 14 | optimisticData: (currentData, newData) => { 15 | return [newData, ...currentData]; 16 | }, 17 | }, 18 | ); 19 | 20 | const { trigger: update, isMutating: isUpdating } = useProjectMutation( 21 | `/models`, 22 | fetcher.patch, 23 | ); 24 | 25 | return { 26 | customModels: data, 27 | insert, 28 | isInserting, 29 | update, 30 | isUpdating, 31 | mutate, 32 | loading: isLoading, 33 | }; 34 | } 35 | -------------------------------------------------------------------------------- /packages/frontend/utils/dataHooks/prompts.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Prompt, 3 | promptSchema, 4 | PromptVersion, 5 | promptVersionSchema, 6 | } from "shared/schemas/prompt"; 7 | import { useProjectSWR } from "."; 8 | import { z } from "zod"; 9 | 10 | export function usePrompts() { 11 | const { data, isLoading, mutate } = useProjectSWR(`/prompts`); 12 | const prompts = z.array(promptSchema).parse(data || []); 13 | 14 | return { 15 | prompts, 16 | isLoading, 17 | mutate, 18 | }; 19 | } 20 | 21 | export function usePromptVersions(id: number | undefined) { 22 | const { data, isLoading, mutate } = useProjectSWR( 23 | id !== undefined && `/prompts/${id}/versions`, 24 | ); 25 | 26 | const promptVersions = z.array(promptVersionSchema).parse(data || []); 27 | 28 | return { 29 | promptVersions, 30 | loading: isLoading, 31 | mutate, 32 | }; 33 | } 34 | -------------------------------------------------------------------------------- /packages/frontend/utils/dataHooks/users.ts: -------------------------------------------------------------------------------- 1 | import { useProjectMutation, useProjectSWR } from "."; 2 | import { fetcher } from "../fetcher"; 3 | import { useSWRConfig } from "swr"; 4 | 5 | export interface InvitationRequest { 6 | email: string; 7 | role: string; 8 | projects: string[]; 9 | } 10 | 11 | export interface InvitedUser { 12 | id: string; 13 | email: string; 14 | role: string; 15 | projects: string[]; 16 | orgId: string; 17 | token: string; 18 | } 19 | 20 | export function useInvitedUsers() { 21 | const { data, error, isLoading, mutate } = 22 | useProjectSWR("/users/invited"); 23 | 24 | return { 25 | invitedUsers: data ?? [], 26 | error, 27 | isLoading, 28 | mutate, 29 | }; 30 | } 31 | 32 | export function useInviteUser() { 33 | const { mutate: mutateUsers } = 34 | useProjectSWR("/users/invited"); 35 | 36 | const { trigger: inviteTrigger, isMutating: isInviting } = useProjectMutation( 37 | "/users/invitation", 38 | fetcher.post, 39 | { 40 | onSuccess: async () => { 41 | await mutateUsers(); 42 | }, 43 | }, 44 | ); 45 | 46 | async function invite(body: InvitationRequest) { 47 | await inviteTrigger(body); 48 | } 49 | 50 | return { 51 | invite, 52 | isInviting, 53 | }; 54 | } 55 | -------------------------------------------------------------------------------- /packages/frontend/utils/errors.tsx: -------------------------------------------------------------------------------- 1 | import { notifications } from "@mantine/notifications"; 2 | import { IconX } from "@tabler/icons-react"; 3 | 4 | // Error handler for fetch requests 5 | const errorHandler = async (promise: Promise) => { 6 | try { 7 | let res = await promise; 8 | 9 | if (!res) { 10 | return null; 11 | } 12 | 13 | if (res?.json) { 14 | res = await res.json(); 15 | } 16 | 17 | const { data, error } = res; 18 | if (error) throw error; 19 | return data || res; 20 | } catch (error: any) { 21 | console.error(error); 22 | 23 | notifications.show({ 24 | icon: , 25 | color: "red", 26 | id: "error-alert", 27 | title: "Error", 28 | autoClose: 4000, 29 | message: error.error_description || error.message || error, 30 | }); 31 | 32 | return null; 33 | } 34 | }; 35 | 36 | export function showErrorNotification(title: any, message?: string) { 37 | console.error(message); 38 | 39 | // prevent 10x same error from being shown 40 | notifications.hide("error-alert"); 41 | 42 | notifications.show({ 43 | icon: , 44 | id: "error-alert", 45 | title: title || "Server error", 46 | message: message || "Something went wrong", 47 | color: "red", 48 | autoClose: 4000, 49 | }); 50 | } 51 | 52 | export default errorHandler; 53 | -------------------------------------------------------------------------------- /packages/frontend/utils/pricing.ts: -------------------------------------------------------------------------------- 1 | export const SEAT_ALLOWANCE = { 2 | free: 1, 3 | pro: 4, 4 | team: 10, 5 | unlimited: 10, 6 | custom: 100, 7 | }; 8 | 9 | export const EVENTS_ALLOWANCE = { 10 | free: 1000, 11 | pro: 4000, 12 | team: 1666, 13 | unlimited: 100000, 14 | custom: 1000000, 15 | }; 16 | -------------------------------------------------------------------------------- /packages/python-sdk/.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | .env 3 | build 4 | .DS_Store 5 | .vscode/settings.json 6 | __pycache__/ 7 | lunary/test.py 8 | lunary/demo.py 9 | data 10 | -------------------------------------------------------------------------------- /packages/python-sdk/README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |

Lunary Python SDK

5 | 6 | **📈 Python monitoring for AI apps and agent** 7 | 8 | [website](https://lunary.ai) - [docs](https://lunary.ai/docs/py/) - ![PyPI - Version](https://img.shields.io/pypi/v/lunary) 9 | 10 | --- 11 | 12 |
13 | 14 | Use it with any LLM model and custom agents (not limited to OpenAI). 15 | 16 | To get started, get a project ID by registering [here](https://lunary.ai). 17 | 18 | ## 🛠️ Installation 19 | 20 | ```bash 21 | pip install lunary 22 | ``` 23 | 24 | ## 📖 Documentation 25 | 26 | Full docs are available [here](https://lunary.ai/docs/py). 27 | -------------------------------------------------------------------------------- /packages/python-sdk/examples/anthropic/async-streaming.py: -------------------------------------------------------------------------------- 1 | import os 2 | import asyncio 3 | from anthropic import AsyncAnthropic 4 | import lunary 5 | 6 | client = AsyncAnthropic( 7 | api_key=os.environ.get("ANTHROPIC_API_KEY"), # This is the default and can be omitted 8 | ) 9 | lunary.monitor(client) 10 | 11 | 12 | async def main() -> None: 13 | stream = await client.messages.create( 14 | max_tokens=1024, 15 | messages=[ 16 | { 17 | "role": "user", 18 | "content": "Hello, Claude", 19 | } 20 | ], 21 | model="claude-3-opus-20240229", 22 | ) 23 | for event in stream: 24 | pass 25 | 26 | 27 | asyncio.run(main()) -------------------------------------------------------------------------------- /packages/python-sdk/examples/anthropic/async.py: -------------------------------------------------------------------------------- 1 | import os 2 | import asyncio 3 | from anthropic import AsyncAnthropic 4 | import lunary 5 | 6 | client = AsyncAnthropic( 7 | api_key=os.environ.get("ANTHROPIC_API_KEY"), # This is the default and can be omitted 8 | ) 9 | lunary.monitor(client) 10 | 11 | 12 | async def main() -> None: 13 | message = await client.messages.create( 14 | max_tokens=1024, 15 | messages=[ 16 | { 17 | "role": "user", 18 | "content": "Hello, Claude", 19 | } 20 | ], 21 | model="claude-3-opus-20240229", 22 | ) 23 | print(message.content) 24 | 25 | 26 | asyncio.run(main()) -------------------------------------------------------------------------------- /packages/python-sdk/examples/anthropic/basic.py: -------------------------------------------------------------------------------- 1 | import os 2 | from anthropic import Anthropic 3 | import lunary 4 | 5 | client = Anthropic( 6 | api_key=os.environ.get("ANTHROPIC_API_KEY"), 7 | ) 8 | lunary.monitor(client) 9 | 10 | 11 | message = client.messages.create( 12 | max_tokens=1024, 13 | messages=[ 14 | { 15 | "role": "user", 16 | "content": "Hello, Claude", 17 | } 18 | ], 19 | model="claude-3-opus-20240229", 20 | ) 21 | -------------------------------------------------------------------------------- /packages/python-sdk/examples/anthropic/streaming.py: -------------------------------------------------------------------------------- 1 | import os 2 | from anthropic import Anthropic 3 | import lunary 4 | 5 | client = Anthropic( 6 | api_key=os.environ.get("ANTHROPIC_API_KEY"), 7 | ) 8 | lunary.monitor(client) 9 | 10 | 11 | stream = client.messages.create( 12 | max_tokens=1024, 13 | messages=[ 14 | { 15 | "role": "user", 16 | "content": "Hello, Claude", 17 | } 18 | ], 19 | model="claude-3-opus-20240229", 20 | stream=True 21 | ) 22 | 23 | for event in stream: 24 | pass 25 | 26 | -------------------------------------------------------------------------------- /packages/python-sdk/examples/azure-openai/async-stream.py: -------------------------------------------------------------------------------- 1 | import os, asyncio 2 | from openai import AsyncAzureOpenAI 3 | import lunary 4 | 5 | DEPLOYMENT_ID = os.environ.get("AZURE_OPENAI_DEPLOYMENT_ID") 6 | RESOURCE_NAME = os.environ.get("AZURE_OPENAI_RESOURCE_NAME") 7 | API_KEY = os.environ.get("AZURE_OPENAI_API_KEY") 8 | 9 | 10 | client = AsyncAzureOpenAI( 11 | api_version="2023-07-01-preview", 12 | api_key=API_KEY, 13 | azure_endpoint=f"https://{DEPLOYMENT_ID}.openai.azure.com", 14 | ) 15 | 16 | lunary.monitor(client) 17 | 18 | async def main() -> None: 19 | stream = await client.chat.completions.create( 20 | model=RESOURCE_NAME, 21 | stream=True, 22 | messages=[ 23 | { 24 | "role": "user", 25 | "content": "Say this is an async stream test", 26 | } 27 | ], 28 | ) 29 | async for chunk in stream: 30 | if not chunk.choices: 31 | continue 32 | print(chunk.choices[0].delta.content, end="") 33 | 34 | 35 | 36 | asyncio.run(main()) -------------------------------------------------------------------------------- /packages/python-sdk/examples/azure-openai/async.py: -------------------------------------------------------------------------------- 1 | import os, asyncio 2 | from openai import AsyncAzureOpenAI 3 | import lunary 4 | 5 | DEPLOYMENT_ID = os.environ.get("AZURE_OPENAI_DEPLOYMENT_ID") 6 | RESOURCE_NAME = os.environ.get("AZURE_OPENAI_RESOURCE_NAME") 7 | API_KEY = os.environ.get("AZURE_OPENAI_API_KEY") 8 | 9 | 10 | client = AsyncAzureOpenAI( 11 | api_version="2023-07-01-preview", 12 | api_key=API_KEY, 13 | azure_endpoint=f"https://{DEPLOYMENT_ID}.openai.azure.com", 14 | ) 15 | 16 | lunary.monitor(client) 17 | 18 | async def main() -> None: 19 | completion = await client.chat.completions.create( 20 | model=RESOURCE_NAME, 21 | messages=[ 22 | { 23 | "role": "user", 24 | "content": "Say this is an Async test", 25 | } 26 | ], 27 | ) 28 | print(completion.to_json()) 29 | 30 | 31 | 32 | asyncio.run(main()) -------------------------------------------------------------------------------- /packages/python-sdk/examples/azure-openai/basic.py: -------------------------------------------------------------------------------- 1 | import os 2 | from openai import AzureOpenAI 3 | import lunary 4 | 5 | API_VERSION = os.environ.get("OPENAI_API_VERSION") 6 | API_KEY = os.environ.get("AZURE_OPENAI_API_KEY") 7 | AZURE_ENDPOINT = os.environ.get("AZURE_OPENAI_ENDPOINT") 8 | RESOURCE_NAME = os.environ.get("AZURE_OPENAI_RESOURCE_NAME") 9 | 10 | 11 | client = AzureOpenAI( 12 | api_version=API_VERSION, 13 | azure_endpoint=AZURE_ENDPOINT, 14 | api_key=API_KEY 15 | ) 16 | lunary.monitor(client) 17 | 18 | completion = client.chat.completions.create( 19 | model=RESOURCE_NAME, 20 | messages=[ 21 | { 22 | "role": "user", 23 | "content": "How do I output all files in a directory using Python?", 24 | }, 25 | ], 26 | ) 27 | print(completion.to_json()) -------------------------------------------------------------------------------- /packages/python-sdk/examples/azure-openai/stream.py: -------------------------------------------------------------------------------- 1 | import os 2 | from openai import AzureOpenAI 3 | import lunary 4 | 5 | DEPLOYMENT_ID = os.environ.get("AZURE_OPENAI_DEPLOYMENT_ID") 6 | RESOURCE_NAME = os.environ.get("AZURE_OPENAI_RESOURCE_NAME") 7 | API_KEY = os.environ.get("AZURE_OPENAI_API_KEY") 8 | 9 | 10 | client = AzureOpenAI( 11 | api_version="2023-07-01-preview", 12 | api_key=API_KEY, 13 | azure_endpoint=f"https://{DEPLOYMENT_ID}.openai.azure.com", 14 | ) 15 | 16 | lunary.monitor(client) 17 | 18 | stream = client.chat.completions.create( 19 | model=RESOURCE_NAME, 20 | stream=True, 21 | messages=[ 22 | { 23 | "role": "user", 24 | "content": "Say sync stream", 25 | }, 26 | ], 27 | ) 28 | for chunk in stream: 29 | if not chunk.choices: 30 | continue 31 | print(chunk.choices[0].delta.content, end="") -------------------------------------------------------------------------------- /packages/python-sdk/examples/functions.py: -------------------------------------------------------------------------------- 1 | import openai 2 | import os 3 | import lunary 4 | 5 | 6 | openai.api_key = os.environ.get("OPENAI_API_KEY") 7 | 8 | lunary.monitor(openai) 9 | 10 | functions = [ 11 | { 12 | "name": "get_current_weather", 13 | "description": "Get the current weather in a given location", 14 | "parameters": { 15 | "type": "object", 16 | "properties": { 17 | "location": { 18 | "type": "string", 19 | "description": "The city and state, e.g. San Francisco, CA", 20 | }, 21 | "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, 22 | }, 23 | "required": ["location"], 24 | }, 25 | } 26 | ] 27 | 28 | completion = openai.ChatCompletion.create( 29 | model="gpt-3.5-turbo", 30 | messages=[{"role": "user", "content": "What's the weather like in Boston?"}], 31 | temperature=0, 32 | functions=functions, 33 | ) 34 | 35 | print(completion.choices[0].message) 36 | -------------------------------------------------------------------------------- /packages/python-sdk/examples/ibm/async.py: -------------------------------------------------------------------------------- 1 | import os, asyncio 2 | from ibm_watsonx_ai import Credentials 3 | from ibm_watsonx_ai.foundation_models import ModelInference 4 | import lunary 5 | 6 | model = ModelInference( 7 | model_id="meta-llama/llama-3-1-8b-instruct", 8 | credentials=Credentials( 9 | api_key=os.environ.get("IBM_API_KEY"), 10 | url = "https://us-south.ml.cloud.ibm.com"), 11 | project_id=os.environ.get("IBM_PROJECT_ID") 12 | ) 13 | lunary.monitor(model) 14 | 15 | async def main(): 16 | messages = [ 17 | {"role": "system", "content": "You are a helpful assistant."}, 18 | {"role": "user", "content": "Who won the world series in 2020?"} 19 | ] 20 | response = await model.achat(messages=messages) 21 | print(response) 22 | 23 | asyncio.run(main()) -------------------------------------------------------------------------------- /packages/python-sdk/examples/ibm/basic.py: -------------------------------------------------------------------------------- 1 | import os 2 | from ibm_watsonx_ai import Credentials 3 | from ibm_watsonx_ai.foundation_models import ModelInference 4 | import lunary 5 | 6 | model = ModelInference( 7 | model_id="meta-llama/llama-3-1-8b-instruct", 8 | credentials=Credentials( 9 | api_key=os.environ.get("IBM_API_KEY"), 10 | url = "https://us-south.ml.cloud.ibm.com"), 11 | project_id=os.environ.get("IBM_PROJECT_ID") 12 | ) 13 | lunary.monitor(model) 14 | 15 | messages = [ 16 | {"role": "system", "content": "You are a helpful assistant."}, 17 | {"role": "user", "content": "Who won the world series in 2020?"} 18 | ] 19 | # response = model.chat(messages=messages, tags=["baseball"], user_id="1234", user_props={"name": "Alice"}) 20 | response = model.chat(messages=messages) -------------------------------------------------------------------------------- /packages/python-sdk/examples/ibm/stream.py: -------------------------------------------------------------------------------- 1 | import os 2 | from ibm_watsonx_ai import Credentials 3 | from ibm_watsonx_ai.foundation_models import ModelInference 4 | import lunary 5 | 6 | model = ModelInference( 7 | model_id="meta-llama/llama-3-1-8b-instruct", 8 | credentials=Credentials( 9 | api_key=os.environ.get("IBM_API_KEY"), 10 | url = "https://us-south.ml.cloud.ibm.com"), 11 | project_id=os.environ.get("IBM_PROJECT_ID") 12 | ) 13 | lunary.monitor(model) 14 | 15 | messages = [ 16 | {"role": "system", "content": "You are a helpful assistant."}, 17 | {"role": "user", "content": "Who won the world series in 2020?"} 18 | ] 19 | response = model.chat_stream(messages=messages) 20 | 21 | for chunk in response: 22 | pass 23 | -------------------------------------------------------------------------------- /packages/python-sdk/examples/ibm/tools.py: -------------------------------------------------------------------------------- 1 | import os 2 | from ibm_watsonx_ai import Credentials 3 | from ibm_watsonx_ai.foundation_models import ModelInference 4 | import lunary 5 | 6 | model = ModelInference( 7 | model_id="meta-llama/llama-3-1-8b-instruct", 8 | credentials=Credentials( 9 | api_key=os.environ.get("IBM_API_KEY"), 10 | url = "https://us-south.ml.cloud.ibm.com"), 11 | project_id=os.environ.get("IBM_PROJECT_ID") 12 | ) 13 | lunary.monitor(model) 14 | 15 | tools = [ 16 | { 17 | "type": "function", 18 | "function": { 19 | "name": "get_weather", 20 | "parameters": { 21 | "type": "object", 22 | "properties": { 23 | "location": {"type": "string"} 24 | }, 25 | }, 26 | }, 27 | } 28 | ] 29 | messages = [ 30 | {"role": "system", "content": "You are a helpful assistant."}, 31 | {"role": "user", "content": "What's the weather like in Paris today?"} 32 | ] 33 | response = model.chat( 34 | messages=messages, 35 | tools=tools, 36 | ) 37 | -------------------------------------------------------------------------------- /packages/python-sdk/examples/ibm/tools_stream.py: -------------------------------------------------------------------------------- 1 | import os 2 | from ibm_watsonx_ai import Credentials 3 | from ibm_watsonx_ai.foundation_models import ModelInference 4 | import lunary 5 | 6 | model = ModelInference( 7 | model_id="meta-llama/llama-3-1-8b-instruct", 8 | credentials=Credentials( 9 | api_key=os.environ.get("IBM_API_KEY"), 10 | url = "https://us-south.ml.cloud.ibm.com"), 11 | project_id=os.environ.get("IBM_PROJECT_ID") 12 | ) 13 | lunary.monitor(model) 14 | 15 | tools = [ 16 | { 17 | "type": "function", 18 | "function": { 19 | "name": "get_weather", 20 | "parameters": { 21 | "type": "object", 22 | "properties": { 23 | "location": {"type": "string"} 24 | }, 25 | }, 26 | }, 27 | } 28 | ] 29 | messages = [ 30 | {"role": "system", "content": "You are a helpful assistant."}, 31 | {"role": "user", "content": "What's the weather like in Paris today?"} 32 | ] 33 | response = model.chat_stream( 34 | messages=messages, 35 | tools=tools, 36 | ) 37 | 38 | for chunk in response: 39 | print(chunk['choices'][0], '\n') -------------------------------------------------------------------------------- /packages/python-sdk/examples/langchain/v0.0.1/langchain-rag.py: -------------------------------------------------------------------------------- 1 | from langchain_community.vectorstores import DocArrayInMemorySearch 2 | from langchain_core.output_parsers import StrOutputParser 3 | from langchain_core.prompts import ChatPromptTemplate 4 | from langchain_core.runnables import RunnableParallel, RunnablePassthrough 5 | from langchain_openai.chat_models import ChatOpenAI 6 | from langchain_openai.embeddings import OpenAIEmbeddings 7 | 8 | vectorstore = DocArrayInMemorySearch.from_texts( 9 | ["the maine coon is grumpy", "the dog is happy"], 10 | embedding=OpenAIEmbeddings(), 11 | ) 12 | retriever = vectorstore.as_retriever() 13 | 14 | template = """Answer the question based only on the following context: 15 | {context} 16 | 17 | Question: {question} 18 | """ 19 | 20 | prompt = ChatPromptTemplate.from_template(template) 21 | model = ChatOpenAI() 22 | output_parser = StrOutputParser() 23 | 24 | setup_and_retrieval = RunnableParallel( 25 | {"context": retriever, "question": RunnablePassthrough()} 26 | ) 27 | chain = setup_and_retrieval | prompt | model | output_parser 28 | 29 | chain.invoke("how is the cat?") -------------------------------------------------------------------------------- /packages/python-sdk/examples/langchain/v0.0.1/multiple_cb.py: -------------------------------------------------------------------------------- 1 | from lunary import LunaryCallbackHandler 2 | from langchain_openai import ChatOpenAI 3 | 4 | handler1 = LunaryCallbackHandler(app_id="07ff18c9-f052-4260-9e89-ea93fe9ba8c5") 5 | handler2 = LunaryCallbackHandler(app_id="5f3553ff-028c-4b8c-86c4-134627ab5e51") 6 | 7 | chat = ChatOpenAI( 8 | callbacks=[handler1, handler2], 9 | ) 10 | chat.stream("Write a random string of 4 letters") -------------------------------------------------------------------------------- /packages/python-sdk/examples/openai/audio.py: -------------------------------------------------------------------------------- 1 | import lunary 2 | from openai import OpenAI 3 | import os 4 | 5 | client = OpenAI( 6 | api_key=os.environ.get("OPENAI_API_KEY"), 7 | ) 8 | 9 | lunary.monitor(client) 10 | 11 | completion = client.chat.completions.create( 12 | model="gpt-4o-audio-preview", 13 | modalities=["audio", "text"], 14 | audio={"voice": "alloy", "format": "wav"}, 15 | messages=[{"role": "user", "content": "Tell me a short story"}], 16 | ) 17 | 18 | print(completion.choices[0]) 19 | -------------------------------------------------------------------------------- /packages/python-sdk/examples/openai/basic.py: -------------------------------------------------------------------------------- 1 | import lunary 2 | from openai import OpenAI 3 | import os 4 | 5 | client = OpenAI( 6 | api_key=os.environ.get("OPENAI_API_KEY"), 7 | ) 8 | 9 | lunary.monitor(client) 10 | 11 | 12 | with lunary.users.identify("user1", user_props={"email": "123@gle.com"}): 13 | completion = client.chat.completions.create( 14 | model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hello world"}] 15 | ) 16 | print(completion.choices[0].message.content) 17 | 18 | 19 | @lunary.agent("My great agent", user_id="123", tags=["test", "test2"]) 20 | def my_agent(a, b, c, test, test2): 21 | tool1("hello") 22 | output = client.chat.completions.create( 23 | model="gpt-3.5-turbo", 24 | messages=[{"role": "user", "content": "Hello world"}], 25 | ) 26 | print(output) 27 | tool2("hello") 28 | return "Agent output" 29 | 30 | 31 | @lunary.tool(name="tool 1", user_id="123") 32 | def tool1(input): 33 | return "Output 1" 34 | 35 | 36 | @lunary.tool() 37 | def tool2(input): 38 | return "Output 2" 39 | 40 | 41 | my_agent(1, 2, 3, test="sdkj", test2="sdkj") 42 | -------------------------------------------------------------------------------- /packages/python-sdk/examples/openai/tools.py: -------------------------------------------------------------------------------- 1 | from openai import OpenAI 2 | import lunary 3 | 4 | client = OpenAI() 5 | lunary.monitor(client) 6 | 7 | 8 | tools = [ 9 | { 10 | "type": "function", 11 | "function": { 12 | "name": "get_weather", 13 | "parameters": { 14 | "type": "object", 15 | "properties": { 16 | "location": {"type": "string"} 17 | }, 18 | }, 19 | }, 20 | } 21 | ] 22 | messages = [ 23 | {"role": "system", "content": "You are a helpful assistant."}, 24 | {"role": "user", "content": "What's the weather like in Paris today?"}, 25 | { 26 | "role": "assistant", 27 | "tool_calls": [ 28 | { 29 | "id": "call_abc123", 30 | "type": "function", 31 | "function": { 32 | "name": "get_weather", 33 | "arguments": "{\"location\": \"Paris\"}" 34 | } 35 | } 36 | ] 37 | }, 38 | { 39 | "role": "tool", 40 | "tool_call_id": "call_abc123", 41 | "name": "get_weather", 42 | "content": "12 degrees celsius" 43 | }, 44 | { 45 | "role": "user", 46 | "content": "Is it cold then?" 47 | } 48 | ] 49 | 50 | 51 | completion = client.chat.completions.create( 52 | model="gpt-4o", 53 | messages=messages, 54 | tools=tools, 55 | ) 56 | 57 | print(completion) -------------------------------------------------------------------------------- /packages/python-sdk/examples/templates.py: -------------------------------------------------------------------------------- 1 | import lunary 2 | import os 3 | from openai import OpenAI 4 | 5 | client = OpenAI() 6 | 7 | # The OpenAI client needs to be monitored by Lunary in order for templates to work 8 | lunary.monitor(client) 9 | 10 | template = lunary.render_template("wailing-waitress") # Replace with the name of the template you want to use 11 | 12 | res = client.chat.completions.create(**template) 13 | 14 | print(res) -------------------------------------------------------------------------------- /packages/python-sdk/examples/threads.py: -------------------------------------------------------------------------------- 1 | import lunary 2 | from openai import OpenAI 3 | import time 4 | 5 | client = OpenAI() 6 | lunary.monitor(client) 7 | 8 | thread = lunary.open_thread() 9 | 10 | message = { "role": "user", "content": "Hello!" } 11 | msg_id = thread.track_message(message) 12 | chat_completion = client.chat.completions.create( 13 | messages=[message], 14 | model="gpt-4o", 15 | parent=msg_id 16 | ) 17 | 18 | 19 | thread.track_message({ 20 | "role": "assistant", 21 | "content": chat_completion.choices[0].message.content 22 | }) 23 | 24 | time.sleep(0.5) 25 | 26 | thread.track_message({ 27 | "role": "user", 28 | "content": "I have a question about your product." 29 | }) 30 | time.sleep(0.5) 31 | 32 | msg_id = thread.track_message({ 33 | "role": "assistant", 34 | "content": "Sure, happy to help. What's your question?" 35 | }) 36 | #time.sleep(0.5) 37 | 38 | # lunary.track_feedback(msg_id, { 39 | # "thumbs": "down", 40 | # "comment": "I don't feel comfortable sharing my credit card number." 41 | # }) 42 | 43 | -------------------------------------------------------------------------------- /packages/python-sdk/lunary/agent.py: -------------------------------------------------------------------------------- 1 | from contextvars import ContextVar 2 | 3 | tags_ctx = ContextVar("tag_ctx", default=None) 4 | 5 | 6 | class TagsContextManager: 7 | def __init__(self, tags: [str]): 8 | tags_ctx.set(tags) 9 | 10 | def __enter__(self): 11 | pass 12 | 13 | def __exit__(self, exc_type, exc_value, exc_tb): 14 | tags_ctx.set(None) 15 | 16 | 17 | def tags(tags: [str]) -> TagsContextManager: 18 | return TagsContextManager(tags) 19 | -------------------------------------------------------------------------------- /packages/python-sdk/lunary/event_queue.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from .consumer import Consumer 3 | from contextvars import ContextVar 4 | 5 | class EventQueue: 6 | def __init__(self): 7 | self.lock = threading.Lock() 8 | self.events = [] 9 | self.consumer = Consumer(self) 10 | self.consumer.start() 11 | 12 | def append(self, event): 13 | with self.lock: 14 | if isinstance(event, list): 15 | self.events.extend(event) 16 | else: 17 | self.events.append(event) 18 | 19 | def get_batch(self): 20 | if self.lock.acquire(False): # non-blocking 21 | try: 22 | events = self.events 23 | self.events = [] 24 | return events 25 | finally: 26 | self.lock.release() 27 | else: 28 | return [] 29 | -------------------------------------------------------------------------------- /packages/python-sdk/lunary/exceptions.py: -------------------------------------------------------------------------------- 1 | class LunaryError(Exception): 2 | """Base exception for all Lunary errors""" 3 | pass 4 | 5 | class TemplateError(LunaryError): 6 | """Raised when there's any error with templates""" 7 | pass 8 | 9 | class DatasetError(LunaryError): 10 | """Raised when there's any error with datasets""" 11 | pass 12 | 13 | class EvaluationError(LunaryError): 14 | """Raised when there's any error with evaluations""" 15 | pass 16 | 17 | class ThreadError(LunaryError): 18 | """Raised when there's any error with thread operations""" 19 | pass 20 | 21 | class FeedbackError(LunaryError): 22 | """Raised when there's any error with feedback operations""" 23 | pass 24 | 25 | class ScoringError(LunaryError): 26 | """Raised when there's any error with scoring operations""" 27 | pass -------------------------------------------------------------------------------- /packages/python-sdk/lunary/parent.py: -------------------------------------------------------------------------------- 1 | # used for reconciliating messages with Langchain: https://lunary.ai/docs/features/chats#reconciliate-with-llm-calls--agents 2 | from contextvars import ContextVar 3 | 4 | parent_ctx = ContextVar("parent_ctx", default=None) 5 | 6 | class ParentContextManager: 7 | def __init__(self, message_id: str): 8 | parent_ctx.set({"message_id": message_id, "retrieved": False}) 9 | 10 | def __enter__(self): 11 | pass 12 | 13 | def __exit__(self, exc_type, exc_value, exc_tb): 14 | parent_ctx.set(None) 15 | 16 | 17 | def parent(id: str) -> ParentContextManager: 18 | return ParentContextManager(id) 19 | -------------------------------------------------------------------------------- /packages/python-sdk/lunary/project.py: -------------------------------------------------------------------------------- 1 | from contextvars import ContextVar 2 | 3 | project_ctx = ContextVar("project_ctx", default=None) 4 | 5 | class ProjectContextManager: 6 | def __init__(self, project_id: str): 7 | project_ctx.set(project_id) 8 | 9 | def __enter__(self): 10 | pass 11 | 12 | def __exit__(self, exc_type, exc_value, exc_tb): 13 | project_ctx.set(None) 14 | 15 | 16 | def project(project_id: str) -> ProjectContextManager: 17 | return ProjectContextManager(project_id) 18 | 19 | -------------------------------------------------------------------------------- /packages/python-sdk/lunary/py.typed: -------------------------------------------------------------------------------- 1 | partial -------------------------------------------------------------------------------- /packages/python-sdk/lunary/tags.py: -------------------------------------------------------------------------------- 1 | from contextvars import ContextVar 2 | 3 | tags_ctx = ContextVar("tag_ctx", default=None) 4 | 5 | 6 | class TagsContextManager: 7 | def __init__(self, tags: [str]): 8 | tags_ctx.set(tags) 9 | 10 | def __enter__(self): 11 | pass 12 | 13 | def __exit__(self, exc_type, exc_value, exc_tb): 14 | tags_ctx.set(None) 15 | 16 | 17 | def tags(tags: [str]) -> TagsContextManager: 18 | return TagsContextManager(tags) 19 | -------------------------------------------------------------------------------- /packages/python-sdk/lunary/users.py: -------------------------------------------------------------------------------- 1 | from contextvars import ContextVar 2 | 3 | user_ctx = ContextVar("user_ctx", default=None) 4 | user_props_ctx = ContextVar("user_props_ctx", default=None) 5 | 6 | class UserContextManager: 7 | def __init__(self, user_id: str, user_props = None): 8 | user_ctx.set(user_id) 9 | user_props_ctx.set(user_props) 10 | 11 | def __enter__(self): 12 | pass 13 | 14 | def __exit__(self, exc_type, exc_value, exc_tb): 15 | user_ctx.set(None) 16 | user_props_ctx.set(None) 17 | 18 | 19 | def identify(user_id: str, user_props = None) -> UserContextManager: 20 | return UserContextManager(user_id, user_props) -------------------------------------------------------------------------------- /packages/python-sdk/lunary/utils.py: -------------------------------------------------------------------------------- 1 | import uuid, hashlib 2 | 3 | def clean_nones(value): 4 | """ 5 | Recursively remove all None values from dictionaries and lists, and returns 6 | the result as a new dictionary or list. 7 | """ 8 | try: 9 | if isinstance(value, list): 10 | return [clean_nones(x) for x in value if x is not None] 11 | elif isinstance(value, dict): 12 | return { 13 | key: clean_nones(val) 14 | for key, val in value.items() 15 | if val is not None 16 | } 17 | else: 18 | return value 19 | except Exception as e: 20 | return value 21 | 22 | def create_uuid_from_string(seed_string): 23 | seed_bytes = seed_string.encode('utf-8') 24 | sha256_hash = hashlib.sha256() 25 | sha256_hash.update(seed_bytes) 26 | hash_hex = sha256_hash.hexdigest() 27 | uuid_hex = hash_hex[:32] 28 | uuid_obj = uuid.UUID(uuid_hex) 29 | return uuid_obj 30 | -------------------------------------------------------------------------------- /packages/python-sdk/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "lunary" 3 | version = "1.4.12" 4 | description = "Python SDK for Lunary, the open-source platform where GenAI teams manage and improve LLM chatbots." 5 | authors = ["lunary "] 6 | readme = "README.md" 7 | repository = "https://github.com/lunary-ai/lunary/tree/main/packages/python-sdk" 8 | documentation = "https://lunary.ai/docs/py" 9 | homepage = "https://lunary.ai/docs/py" 10 | keywords = [ 11 | "Lunary", 12 | "lunary.ai", 13 | "Langchain", 14 | "AI", 15 | "Analytics", 16 | "Monitoring", 17 | "LLM", 18 | "GenAI", 19 | "SDK", 20 | ] 21 | 22 | [tool.poetry.dependencies] 23 | python = ">=3.10.0,<4.0.0" 24 | requests = "^2.31.0" 25 | setuptools = ">=78.1.1" 26 | tenacity = "^8.2.3" 27 | packaging = "^23.2" 28 | chevron = "^0.14.0" 29 | pyhumps = "^3.8.0" 30 | aiohttp = "^3.9.5" 31 | jsonpickle = "^3.0.4" 32 | pydantic = "^2.10.2" 33 | 34 | [tool.poetry.group.dev.dependencies] 35 | langchain-core = "^0.3.13" 36 | openai = "^1.12.0" 37 | pytest = "^8.3.3" 38 | greenlet = "^3.1.1" 39 | # ibm-watsonx-ai = "^1.1.26" 40 | 41 | [build-system] 42 | requires = ["poetry-core"] 43 | build-backend = "poetry.core.masonry.api" 44 | 45 | [tool.pytest.ini_options] 46 | testpaths = ["tests"] 47 | python_files = "test_*.py" 48 | -------------------------------------------------------------------------------- /packages/python-sdk/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lunary-ai/lunary/27c09c35f982122a2d0b316a9cbbb97bd72711fe/packages/python-sdk/tests/__init__.py -------------------------------------------------------------------------------- /packages/shared/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lunary-ai/lunary/27c09c35f982122a2d0b316a9cbbb97bd72711fe/packages/shared/README.md -------------------------------------------------------------------------------- /packages/shared/access-control/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./roles"; 2 | -------------------------------------------------------------------------------- /packages/shared/checks/types.ts: -------------------------------------------------------------------------------- 1 | export type CheckLabel = { 2 | type: "label"; 3 | label: string; 4 | }; 5 | 6 | export type CheckParam = { 7 | type: "select" | "text" | "number" | "date" | "users"; 8 | id: string; 9 | unit?: string; 10 | max?: number; 11 | min?: number; 12 | step?: number; 13 | width?: number; 14 | placeholder?: string; 15 | defaultValue?: string | number | boolean | string[]; 16 | searchable?: boolean; 17 | allowCustom?: boolean; 18 | getItemValue?: (item: any) => string; // custom function to get value from item, for selects 19 | customSearch?: (query: string, item: any) => boolean; // custom search function for search in selects 20 | multiple?: boolean; 21 | options?: 22 | | Array<{ label: string; value: string }> 23 | | ((projectId: string, type: string) => string); 24 | }; 25 | 26 | export type Check = { 27 | id: string; 28 | uiType?: "basic" | "smart" | "ai"; 29 | name: string; 30 | description?: string; 31 | soon?: boolean; 32 | params: (CheckParam | CheckLabel)[]; 33 | disableInEvals?: boolean; 34 | uniqueInBar?: boolean; // if true, only one check of this type can be in a filter bar 35 | onlyInEvals?: boolean; 36 | }; 37 | 38 | // [ 'AND, {id, params}, {id, params}, {id, params}, ['OR', {id, params}, {id, params}], ['OR', {id, params}, ['AND', {id, params}, {id, params}]] ] 39 | 40 | export type LogicData = { 41 | id: string; 42 | params: any; 43 | }; 44 | 45 | export type LogicNode = ["AND" | "OR", ...LogicElement[]]; 46 | 47 | export type LogicElement = LogicData | LogicNode; 48 | 49 | export type CheckLogic = LogicNode; 50 | -------------------------------------------------------------------------------- /packages/shared/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./models"; 2 | export * from "./checks"; 3 | export * from "./schemas"; 4 | export * from "./access-control"; 5 | export * from "./enrichers"; 6 | export * from "./dashboards"; 7 | export * from "./providers"; 8 | export * from "./audit-logs"; 9 | -------------------------------------------------------------------------------- /packages/shared/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "shared", 3 | "type": "module", 4 | "dependencies": { 5 | "compromise": "^14.14.4", 6 | "franc": "^6.2.0", 7 | "postgres": "3.4.5", 8 | "zod": "^3.24.3" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /packages/shared/schemas/evaluation.ts: -------------------------------------------------------------------------------- 1 | import { Message } from "./old-openai"; 2 | 3 | export interface Evaluation { 4 | name?: string; 5 | prompts: Prompt[]; 6 | providers: any[]; 7 | checklistId: string; 8 | datasetId: string; 9 | } 10 | 11 | export interface Prompt { 12 | messages: Message[]; 13 | variations: Variation[]; 14 | } 15 | 16 | export interface Variation { 17 | variables: Record; 18 | idealOutput?: string; 19 | } 20 | 21 | export interface OldProvider { 22 | model: string; 23 | config: Record; 24 | } 25 | -------------------------------------------------------------------------------- /packages/shared/schemas/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./evaluation"; 2 | export * from "./prompt"; 3 | export * from "./run"; 4 | export * from "./messages"; 5 | export * from "./user"; 6 | export * from "./project"; 7 | -------------------------------------------------------------------------------- /packages/shared/schemas/messages.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | 3 | const textContentPartSchema = z.object({ 4 | type: z.literal("text"), 5 | text: z.string(), 6 | }); 7 | 8 | const imageContentPartSchema = z.object({ 9 | type: z.literal("image_url"), 10 | image_url: z.object({ 11 | url: z.string(), 12 | detail: z.string().optional(), 13 | }), 14 | }); 15 | 16 | const contentPartSchema = z.union([ 17 | textContentPartSchema, 18 | imageContentPartSchema, 19 | ]); 20 | 21 | export const userMessageSchema = z.object({ 22 | role: z.literal("user"), 23 | content: z.union([z.string(), z.array(contentPartSchema)]), 24 | name: z.string().optional(), 25 | }); 26 | export type UserMessage = z.infer; 27 | -------------------------------------------------------------------------------- /packages/shared/schemas/old-openai.ts: -------------------------------------------------------------------------------- 1 | export type Message = 2 | | SystemMessage 3 | | UserMessage 4 | | AssistantMessage 5 | | ToolMessage; 6 | 7 | interface SystemMessage { 8 | role: "system"; 9 | content: string; 10 | name?: string; 11 | } 12 | 13 | interface UserMessage { 14 | role: "user"; 15 | content: string; 16 | name?: string; 17 | } 18 | 19 | interface AssistantMessage { 20 | role: "assistant"; 21 | content?: string | null; 22 | name?: string; 23 | 24 | tool_calls?: ToolCall[]; 25 | } 26 | 27 | interface ToolMessage { 28 | role: "tool"; 29 | content: string; 30 | tool_call_id: string; 31 | } 32 | 33 | interface ToolCall { 34 | id: string; 35 | function: { 36 | arguments: string; 37 | name: string; 38 | }; 39 | type: "function"; 40 | } 41 | -------------------------------------------------------------------------------- /packages/shared/schemas/project.ts: -------------------------------------------------------------------------------- 1 | export interface Project { 2 | id: string; 3 | created_at: Date; 4 | name: string; 5 | orgId: string; 6 | } 7 | -------------------------------------------------------------------------------- /packages/shared/schemas/prompt.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { MessageSchema } from "./openai"; 3 | 4 | export type TemplateVariables = Record; // TODO: remove 5 | 6 | export const promptSchema = z.object({ 7 | id: z.number(), 8 | createdAt: z.string(), 9 | slug: z.string(), 10 | projectId: z.string(), 11 | mode: z.enum(["openai", "text"]), 12 | }); 13 | export type Prompt = z.infer; 14 | 15 | export const promptVersionSchema = z.object({ 16 | id: z.number(), 17 | createdAt: z.string(), 18 | extra: z.object({ 19 | model: z.string(), 20 | max_tokens: z.number().default(4096), 21 | temperature: z.number().default(1), 22 | }), 23 | content: z.array(MessageSchema), 24 | version: z.number(), 25 | isDraft: z.boolean(), 26 | }); 27 | export type PromptVersion = z.infer; 28 | -------------------------------------------------------------------------------- /packages/shared/schemas/run.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | 3 | export const Feedback = z.union([ 4 | z.object({ 5 | thumb: z.enum(["up", "down"]).nullable().optional(), 6 | comment: z.string().nullable().optional(), 7 | }), 8 | z.null(), 9 | ]); 10 | export type Feedback = z.infer; 11 | 12 | export const Score = z.object({ 13 | label: z.string(), 14 | value: z.union([z.number(), z.string(), z.boolean()]), 15 | comment: z.string().nullable().optional(), 16 | }); 17 | export type Score = z.infer; 18 | 19 | export interface Run { 20 | id: string; 21 | createdAt: string; 22 | endedAt?: string; 23 | duration?: string; 24 | tags?: string[]; 25 | projectId: string; 26 | status?: string; 27 | name?: string; 28 | input?: unknown; 29 | output?: unknown; 30 | error?: unknown; 31 | params?: Record; 32 | type: string; 33 | parentRunId?: string; 34 | promptTokens?: number; 35 | completionTokens?: number; 36 | cost?: number; 37 | externalUserId?: number; 38 | feedback?: Record; // TODO: real feedback type, but need to check before 39 | isPublic: boolean; 40 | siblingRunId?: string; 41 | templateVersionId?: number; 42 | runtime?: string; 43 | metadata?: Record; 44 | ipAddresses?: string[]; 45 | } 46 | -------------------------------------------------------------------------------- /packages/shared/schemas/user.ts: -------------------------------------------------------------------------------- 1 | import { Role } from "../access-control"; 2 | 3 | export interface User { 4 | id: string; 5 | created_at: Date; 6 | email: string | null; 7 | password_hash: string | null; 8 | recovery_token: string | null; 9 | name: string | null; 10 | org_id: string | null; 11 | role: Role; 12 | verified: boolean; 13 | avatar_url: string | null; 14 | last_login_at: Date | null; 15 | single_use_token: string | null; 16 | export_single_use_token: string | null; 17 | } 18 | -------------------------------------------------------------------------------- /packages/shared/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["ESNext"], 4 | "module": "esnext", 5 | "target": "esnext", 6 | "moduleResolution": "bundler", 7 | "moduleDetection": "force", 8 | "allowImportingTsExtensions": true, 9 | "noEmit": true, 10 | "composite": true, 11 | "strict": true, 12 | "downlevelIteration": true, 13 | "skipLibCheck": true, 14 | "jsx": "react-jsx", 15 | "allowSyntheticDefaultImports": true, 16 | "forceConsistentCasingInFileNames": true, 17 | "allowJs": true 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /packages/shared/utils/date.ts: -------------------------------------------------------------------------------- 1 | // TODO (analytics date range helper + backend date.ts) 2 | -------------------------------------------------------------------------------- /packages/tokenizer/external.ts: -------------------------------------------------------------------------------- 1 | export const getAnthropicTokenCount = async ( 2 | msg_content : string, 3 | model : string = "claude-3-5-sonnet-20241022" 4 | ) => { 5 | const url = "https://api.anthropic.com/v1/messages/count_tokens" 6 | const apiKey = process.env.ANTHROPIC_API_KEY 7 | if (!apiKey) { 8 | throw new Error("API key is not defined") 9 | } 10 | 11 | const headers = { 12 | "x-api-key": apiKey, 13 | "content-type": "application/json", 14 | "anthropic-version": "2023-06-01", 15 | } 16 | 17 | const body = JSON.stringify({ 18 | model: model, 19 | messages: [ 20 | { 21 | role: "user", 22 | content: msg_content, 23 | }, 24 | ], 25 | }) 26 | 27 | try { 28 | const response = await fetch(url, { 29 | method: "POST", 30 | headers, 31 | body, 32 | }) 33 | 34 | if (!response.ok) { 35 | throw new Error(`HTTP error! Status: ${response.status}`) 36 | } 37 | 38 | const data = await response.json() 39 | return data 40 | } catch (error) { 41 | console.error("Error fetching token count:", error) 42 | throw error 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /packages/tokenizer/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tokenizers", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.ts", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "start": "bun index.ts", 9 | "dev": "bun --watch index.ts" 10 | }, 11 | "author": "", 12 | "license": "ISC", 13 | "dependencies": { 14 | "@anthropic-ai/tokenizer": "^0.0.4", 15 | "@koa/cors": "^5.0.0", 16 | "@xenova/transformers": "^2.17.2", 17 | "bun": "^1.1.12", 18 | "js-tiktoken": "^1.0.12", 19 | "koa": "^2.16.1", 20 | "koa-bodyparser": "^4.4.1", 21 | "koa-ratelimit": "^5.1.0", 22 | "koa-router": "^12.0.1", 23 | "llama3-tokenizer-js": "^1.2.0", 24 | "mistral-tokenizer-js": "^1.0.0", 25 | "node-cache": "^5.1.2", 26 | "seed-color": "^2.0.1" 27 | }, 28 | "prettier": { 29 | "semi": false 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["ESNext"], 4 | "module": "esnext", 5 | "target": "esnext", 6 | "moduleResolution": "bundler", 7 | "moduleDetection": "force", 8 | "allowImportingTsExtensions": true, 9 | "noEmit": true, 10 | "composite": true, 11 | "strict": true, 12 | "downlevelIteration": true, 13 | "skipLibCheck": true, 14 | "jsx": "react-jsx", 15 | "allowSyntheticDefaultImports": true, 16 | "forceConsistentCasingInFileNames": true, 17 | "allowJs": true 18 | } 19 | } 20 | --------------------------------------------------------------------------------