├── docs ├── static │ ├── .nojekyll │ └── img │ │ ├── ui-demo.gif │ │ ├── rag-diagram.webp │ │ ├── ingest-diagram.webp │ │ └── server-diagram.webp ├── docs │ ├── reference │ │ └── .gitinclude │ └── contact.md ├── typedoc.json ├── babel.config.js ├── tsconfig.json ├── .gitignore └── README.md ├── packages ├── datasets │ ├── .python-version │ ├── .gitignore │ ├── src │ │ ├── modules.d.ts │ │ ├── treeGeneration │ │ │ ├── databaseNlQueries │ │ │ │ ├── databaseNodes │ │ │ │ │ ├── generateAtlasSearchCode.eval.ts │ │ │ │ │ ├── generateNaturalLanguageQueriesAtlasSearch.eval.ts │ │ │ │ │ └── generateUseCases.test.ts │ │ │ │ └── databaseMetadata │ │ │ │ │ └── prettyPrintMongoDbDocument.ts │ │ │ ├── GenerationNode.ts │ │ │ └── mongoDbDatabases │ │ │ │ └── README.md │ │ ├── README.md │ │ ├── test │ │ │ ├── globals.ts │ │ │ ├── constants.ts │ │ │ ├── globalTeardown.ts │ │ │ └── globalSetup.ts │ │ ├── EnvVars.ts │ │ ├── uploadDatasetToBraintrust.ts │ │ ├── openAi.ts │ │ ├── bin │ │ │ ├── countExamplesByLength.ts │ │ │ ├── countUsableExamples.ts │ │ │ └── analyzeNlToMongoShDatasetBraintrust.ts │ │ ├── codeExampleDataset │ │ │ └── AstExtractedCodeBlock.ts │ │ └── uploadDatasetToHuggingFace.test.ts │ ├── .eslintrc.cjs │ ├── tsconfig.build.json │ ├── mongodb_datasets │ │ └── __init__.py │ ├── .prettierrc.cjs │ ├── tsconfig.json │ ├── jest.config.cjs │ ├── .env.example │ ├── pyproject.toml │ └── .release-it.json ├── mongodb-chatbot-server │ ├── .nvmrc │ ├── src │ │ ├── modules.d.ts │ │ ├── test │ │ │ ├── jestEnvSetUp.ts │ │ │ ├── jestSetUp.ts │ │ │ ├── README.md │ │ │ ├── globals.ts │ │ │ ├── constants.ts │ │ │ ├── globalTeardown.ts │ │ │ ├── globalSetup.ts │ │ │ └── middlewareTestHelpers.ts │ │ ├── routes │ │ │ ├── conversations │ │ │ │ ├── index.ts │ │ │ │ └── constants.ts │ │ │ ├── content │ │ │ │ └── index.ts │ │ │ ├── responses │ │ │ │ └── index.ts │ │ │ └── index.ts │ │ ├── middleware │ │ │ ├── index.ts │ │ │ ├── ipAddressKeyGenerator.ts │ │ │ ├── slowDown.test.ts │ │ │ ├── rateLimit.test.ts │ │ │ ├── slowDown.ts │ │ │ └── rateLimit.ts │ │ ├── index.ts │ │ └── processors │ │ │ ├── filterOnlySystemPrompt.ts │ │ │ ├── FilterPreviousMessages.ts │ │ │ ├── makeFilterNPreviousMessages.ts │ │ │ ├── MakeReferenceLinksFunc.ts │ │ │ ├── index.ts │ │ │ ├── includeChunksForMaxTokensPossible.ts │ │ │ ├── InputGuardrail.ts │ │ │ └── QueryPreprocessorFunc.ts │ ├── .prettierignore │ ├── .eslintrc.js │ ├── tsconfig.build.json │ ├── .gitignore │ ├── .prettierrc.cjs │ ├── babel.config.js │ ├── tsconfig.json │ ├── jest.config.json │ ├── .env.example │ └── .release-it.json ├── performance-tests │ ├── .gitignore │ ├── tsconfig.build.json │ ├── .prettierrc.cjs │ ├── tsconfig.json │ └── package.json ├── chatbot-server-mongodb-public │ ├── .nvmrc │ ├── static │ │ └── .gitinclude │ ├── src │ │ ├── test │ │ │ ├── jestEnvSetUp.ts │ │ │ ├── README.md │ │ │ ├── globals.ts │ │ │ ├── constants.ts │ │ │ ├── globalTeardown.ts │ │ │ ├── jestSetUp.ts │ │ │ ├── cosineSimilarity.ts │ │ │ ├── mockOpenAi.ts │ │ │ └── globalSetup.ts │ │ ├── modules.d.ts │ │ ├── tracing │ │ │ ├── tagify.ts │ │ │ ├── traceHandlers │ │ │ │ └── index.ts │ │ │ └── scrubbedMessages │ │ │ │ └── ScrubbedMessageStore.ts │ │ ├── middleware │ │ │ ├── blockGetRequests.ts │ │ │ └── useSegmentIds.ts │ │ ├── lib.ts │ │ ├── EnvVars.ts │ │ └── corsOptions.ts │ ├── .prettierignore │ ├── .eslintrc.js │ ├── .gitignore │ ├── .prettierrc.cjs │ ├── tsconfig.build.json │ ├── babel.config.js │ ├── tsconfig.json │ ├── jest.config.json │ ├── global.d.ts │ ├── .release-it.json │ └── environments │ │ ├── qa.yml │ │ └── dev-rel.yml ├── scripts │ ├── .gitignore │ ├── .eslintrc.cjs │ ├── tsconfig.build.json │ ├── .prettierrc.cjs │ ├── src │ │ ├── test │ │ │ ├── constants.ts │ │ │ ├── jestSetUp.ts │ │ │ ├── globals.ts │ │ │ ├── globalTeardown.ts │ │ │ └── globalSetup.ts │ │ ├── findCentroid.ts │ │ └── findCentroid.test.ts │ ├── tsconfig.json │ ├── README.md │ ├── .env.example │ ├── jest.config.cjs │ ├── testData │ │ └── meta2rst │ │ │ ├── no-meta-no-description.rst │ │ │ ├── yes-meta-no-description.rst │ │ │ └── yes-meta-yes-description.rst │ └── .release-it.json ├── benchmarks │ ├── src │ │ ├── textToDriver │ │ │ ├── .gitignore │ │ │ ├── notebooks-and-scripts │ │ │ │ └── .gitignore │ │ │ ├── README.md │ │ │ ├── generateDriverCode │ │ │ │ ├── languagePrompts │ │ │ │ │ ├── listExamplesInPrompt.ts │ │ │ │ │ ├── FewShotExample.ts │ │ │ │ │ ├── PromptStrategies.ts │ │ │ │ │ └── listExamplesInPrompt.test.ts │ │ │ │ └── tools │ │ │ │ │ └── submitFinalSolution.ts │ │ │ ├── TextToDriverEnvVars.ts │ │ │ ├── DatabaseMetadata.ts │ │ │ ├── makeDatabases │ │ │ │ ├── applyCollectionIndexes.ts │ │ │ │ └── insertCollectionDocuments.ts │ │ │ └── makeExperimentName.ts │ │ ├── test │ │ │ ├── makeSampleLlmOptions.ts │ │ │ ├── globals.ts │ │ │ ├── constants.ts │ │ │ ├── globalTeardown.ts │ │ │ └── globalSetup.ts │ │ ├── index.ts │ │ ├── envVars.ts │ │ ├── makeExperimentName.ts │ │ └── reporting │ │ │ ├── getBraintrustExperimentSummary.test.ts │ │ │ ├── bin │ │ │ ├── reportTechSupportBenchmarkResults.ts │ │ │ └── reportMultipleChoiceBenchmarkResults.ts │ │ │ └── convertTagStatsToFlatObject.ts │ ├── tsconfig.build.json │ ├── .eslintrc.cjs │ ├── .prettierrc.cjs │ ├── tsconfig.json │ ├── jest.config.cjs │ ├── .release-it.json │ └── .env.example ├── mongodb-rag-core │ ├── src │ │ ├── openai.ts │ │ ├── mongodb.ts │ │ ├── embed │ │ │ ├── index.ts │ │ │ └── Embedder.ts │ │ ├── eval │ │ │ ├── retrievalMetrics │ │ │ │ ├── MatchFunc.ts │ │ │ │ ├── Primitive.ts │ │ │ │ ├── assertKIsValid.ts │ │ │ │ └── index.ts │ │ │ └── index.ts │ │ ├── findContent │ │ │ ├── index.ts │ │ │ └── FindContentFunc.ts │ │ ├── conversations │ │ │ └── index.ts │ │ ├── modules.d.ts │ │ ├── models │ │ │ ├── index.ts │ │ │ └── getOpenAiEndpointAndApiKey.ts │ │ ├── frontMatter │ │ │ ├── index.ts │ │ │ ├── removeFrontMatter.ts │ │ │ ├── updateFrontMatter.ts │ │ │ ├── updateFrontMatter.test.ts │ │ │ └── removeFrontMatter.test.ts │ │ ├── test │ │ │ ├── jestSetUp.ts │ │ │ ├── globals.ts │ │ │ ├── constants.ts │ │ │ ├── globalTeardown.ts │ │ │ ├── MockPageStore.ts │ │ │ └── globalSetup.ts │ │ ├── chunk │ │ │ └── index.ts │ │ ├── mongoDbMetadata │ │ │ └── index.ts │ │ ├── verifiedAnswers │ │ │ ├── index.ts │ │ │ └── FindVerifiedAnswerFunc.ts │ │ ├── executeCode │ │ │ ├── databaseMetadata │ │ │ │ ├── LlmOptions.ts │ │ │ │ └── index.ts │ │ │ ├── prettyPrintMongoDbDocument.ts │ │ │ ├── redactMongoDbConnectionUri.ts │ │ │ ├── extractCodeFromMarkdown.ts │ │ │ ├── index.ts │ │ │ └── DatabaseExecutionResult.ts │ │ ├── contentStore │ │ │ ├── pageIdentity.ts │ │ │ └── index.ts │ │ ├── dataSources │ │ │ ├── makeMarkdownLists.ts │ │ │ ├── DataSource.ts │ │ │ ├── removeMarkdownFileExtension.ts │ │ │ ├── index.ts │ │ │ ├── ProjectBase.ts │ │ │ ├── extractMarkdownH1.test.ts │ │ │ ├── removeMarkdownImagesAndLinks.ts │ │ │ ├── extractMarkdownH1.ts │ │ │ └── GitDataSource.test.ts │ │ ├── DatabaseConnection.ts │ │ ├── arrayFilters.ts │ │ ├── References.ts │ │ ├── assertEnvVars.ts │ │ ├── MongoDbDatabaseConnection.ts │ │ └── logger.ts │ ├── .eslintrc.cjs │ ├── tsconfig.build.json │ ├── .prettierrc.cjs │ ├── tsconfig.json │ ├── README.md │ ├── jest.config.cjs │ ├── testData │ │ └── sampleMdxFile.mdx │ ├── .release-it.json │ └── .env.example ├── mongodb-artifact-generator │ ├── src │ │ ├── release-notes │ │ │ ├── release-notes.yaml │ │ │ ├── ReleaseInfo.ts │ │ │ └── jira.test.ts │ │ ├── index.ts │ │ ├── test │ │ │ ├── jestSetUp.ts │ │ │ └── MockPageStore.ts │ │ ├── ArtifactGeneratorEnvVars.ts │ │ ├── createCommand.ts │ │ ├── runId.ts │ │ ├── withConfig.test.ts │ │ ├── main.ts │ │ ├── chat │ │ │ └── index.ts │ │ ├── docs-metadata │ │ │ └── generateMetadata.ts │ │ └── yaml.ts │ ├── tsconfig.build.json │ ├── .prettierrc.cjs │ ├── jest.config.cjs │ ├── testData │ │ └── config │ │ │ └── configTest.cjs │ ├── tsconfig.json │ ├── .eslintrc.cjs │ ├── .env.example │ ├── context │ │ └── tdbx-content-types │ │ │ ├── contribution-guide.yaml │ │ │ ├── whats-new.yaml │ │ │ ├── migrate-legacy-version.yaml │ │ │ ├── upgrade-version.yaml │ │ │ ├── issue-help.yaml │ │ │ ├── troubleshooting.yaml │ │ │ ├── faq.yaml │ │ │ └── landing-page.yml │ └── .gitignore ├── release-notes-generator │ ├── .gitignore │ ├── src │ │ ├── jira │ │ │ └── index.ts │ │ ├── github │ │ │ └── index.ts │ │ ├── configs │ │ │ └── index.ts │ │ ├── llm │ │ │ └── index.ts │ │ ├── index.ts │ │ ├── utils.ts │ │ └── utils.test.ts │ ├── .prettierignore │ ├── vitest.config.ts │ ├── .prettierrc.cjs │ ├── tsconfig.json │ └── tsup.config.ts ├── ingest-mongodb-public │ ├── src │ │ ├── modules.d.ts │ │ ├── sources │ │ │ ├── mongodb-university │ │ │ │ └── index.ts │ │ │ ├── mongodbDotCom │ │ │ │ └── index.ts │ │ │ ├── snooty │ │ │ │ ├── index.ts │ │ │ │ └── truncateEmbeddings.ts │ │ │ ├── mongoose.test.ts │ │ │ ├── snootySources.test.ts │ │ │ └── practicalAggregations.test.ts │ │ ├── PublicIngestEnvVars.ts │ │ └── test │ │ │ └── jestSetUp.ts │ ├── tsconfig.build.json │ ├── .eslintrc.cjs │ ├── .prettierrc.cjs │ ├── tsconfig.json │ ├── jest.config.cjs │ ├── README.md │ ├── .release-it.json │ ├── testData │ │ └── sitemap-pages.xml │ └── .env.example ├── mercury-case-analysis │ ├── .gitignore │ ├── vitest.config.ts │ ├── tsconfig.build.json │ ├── tsconfig.json │ ├── src │ │ ├── index.ts │ │ └── calculateEmbeddings.ts │ ├── tsup.config.ts │ └── .release-it.json ├── mongodb-chatbot-ui │ ├── .env.staging │ ├── .env.qa │ ├── src │ │ ├── DarkMode.ts │ │ ├── ChatbotTrigger.tsx │ │ ├── useLinkData.tsx │ │ ├── useUser.ts │ │ ├── ui-text.ts │ │ ├── main.tsx │ │ ├── vite-env.d.ts │ │ ├── useConversationStateContext.tsx │ │ ├── useChatbotContext.tsx │ │ ├── UserProvider.tsx │ │ ├── test-utils.ts │ │ ├── ChatbotProvider.tsx │ │ ├── LinkDataProvider.tsx │ │ ├── Banner.tsx │ │ ├── ChatbotView.tsx │ │ ├── Banner.test.tsx │ │ ├── App.module.css │ │ ├── FloatingActionButtonTrigger.tsx │ │ ├── createMessage.ts │ │ └── ActionButtonTrigger.tsx │ ├── .env.example │ ├── .prettierignore │ ├── .prettierrc.cjs │ ├── tsconfig.build.json │ ├── tsconfig.node.json │ ├── .storybook │ │ ├── preview.ts │ │ └── main.ts │ ├── vitest.config.ts │ ├── index.html │ ├── .gitignore │ ├── .release-it.json │ ├── tsconfig.json │ ├── vite.config.static-site.ts │ ├── vitest.setup.ts │ └── .eslintrc.cjs ├── mongodb-rag-ingest │ ├── src │ │ ├── IngestEnvVars.ts │ │ ├── test │ │ │ ├── jestSetUp.ts │ │ │ ├── globals.ts │ │ │ ├── constants.ts │ │ │ ├── globalTeardown.ts │ │ │ └── globalSetup.ts │ │ ├── index.ts │ │ ├── withConfig.test.ts │ │ └── main.ts │ ├── .eslintrc.cjs │ ├── tsconfig.build.json │ ├── .prettierrc.cjs │ ├── tsconfig.json │ ├── .env.example │ ├── jest.config.cjs │ ├── testData │ │ └── config │ │ │ └── configTest.cjs │ └── .release-it.json ├── mongodb-chatbot-verified-answers │ ├── src │ │ ├── index.ts │ │ ├── main.ts │ │ └── parseVerifiedAnswersYaml.ts │ ├── .eslintrc.cjs │ ├── tsconfig.build.json │ ├── .prettierrc.cjs │ ├── jest.config.cjs │ ├── tsconfig.json │ ├── example-verified-answers.yaml │ ├── .env.example │ ├── README.md │ └── .release-it.json └── mongodb-atlas │ ├── dump-restore │ ├── dump.sh │ └── restore.sh │ ├── .prettierrc.cjs │ ├── .gitignore │ ├── utils.ts │ └── .env.example ├── .npmrc ├── .dockerignore ├── .prettierrc.cjs ├── .github └── pull_request_template.md ├── scripts-service.dockerfile ├── .gitignore ├── lerna.json ├── mongodb-corp └── README.md ├── datasets.dockerfile ├── tsconfig.json ├── ingest-service.dockerfile ├── THIRD_PARTY_NOTICES └── chat-server.dockerfile /docs/static/.nojekyll: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/docs/reference/.gitinclude: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/datasets/.python-version: -------------------------------------------------------------------------------- 1 | 3.11 2 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/.nvmrc: -------------------------------------------------------------------------------- 1 | v18 2 | -------------------------------------------------------------------------------- /packages/performance-tests/.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/.nvmrc: -------------------------------------------------------------------------------- 1 | v18 2 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/static/.gitinclude: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/datasets/.gitignore: -------------------------------------------------------------------------------- 1 | dataOut/ 2 | __pycache__/ 3 | -------------------------------------------------------------------------------- /packages/scripts/.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | 3 | build/ 4 | -------------------------------------------------------------------------------- /docs/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "excludeExternals": true 3 | } 4 | -------------------------------------------------------------------------------- /packages/benchmarks/src/textToDriver/.gitignore: -------------------------------------------------------------------------------- 1 | data_out 2 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | //registry.npmjs.org/:_authToken=${MONGODB_EAI_NPM_TOKEN} 2 | -------------------------------------------------------------------------------- /packages/datasets/src/modules.d.ts: -------------------------------------------------------------------------------- 1 | declare module "lang-map"; 2 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/openai.ts: -------------------------------------------------------------------------------- 1 | export * from "openai"; 2 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/release-notes/release-notes.yaml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/mongodb.ts: -------------------------------------------------------------------------------- 1 | export * from "mongodb"; 2 | -------------------------------------------------------------------------------- /packages/release-notes-generator/.gitignore: -------------------------------------------------------------------------------- 1 | logs/ 2 | build/ 3 | dist/ 4 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | tests/ 3 | jest* 4 | README.md 5 | .env.example 6 | -------------------------------------------------------------------------------- /packages/benchmarks/src/textToDriver/notebooks-and-scripts/.gitignore: -------------------------------------------------------------------------------- 1 | data_out 2 | -------------------------------------------------------------------------------- /packages/release-notes-generator/src/jira/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./artifacts"; 2 | -------------------------------------------------------------------------------- /.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | embeddedLanguageFormatting: "off", 3 | }; 4 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/modules.d.ts: -------------------------------------------------------------------------------- 1 | declare module "rate-limit-mongo"; 2 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/test/jestEnvSetUp.ts: -------------------------------------------------------------------------------- 1 | import "dotenv/config"; 2 | -------------------------------------------------------------------------------- /packages/release-notes-generator/src/github/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./artifacts"; 2 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/test/jestEnvSetUp.ts: -------------------------------------------------------------------------------- 1 | import "dotenv/config"; 2 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/src/modules.d.ts: -------------------------------------------------------------------------------- 1 | declare module "turndown-plugin-gfm"; 2 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/modules.d.ts: -------------------------------------------------------------------------------- 1 | declare module "rate-limit-mongo"; 2 | -------------------------------------------------------------------------------- /packages/mercury-case-analysis/.gitignore: -------------------------------------------------------------------------------- 1 | tsconfig.build.tsbuildinfo 2 | node_modules 3 | build 4 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/.env.staging: -------------------------------------------------------------------------------- 1 | VITE_SERVER_BASE_URL=/api/v1 2 | NODE_ENV="development" 3 | -------------------------------------------------------------------------------- /docs/static/img/ui-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mongodb/chatbot/HEAD/docs/static/img/ui-demo.gif -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/routes/conversations/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./conversationsRouter"; 2 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/.env.qa: -------------------------------------------------------------------------------- 1 | VITE_SERVER_BASE_URL=/api/v1 2 | VITE_QA=true 3 | NODE_ENV="development" 4 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/DarkMode.ts: -------------------------------------------------------------------------------- 1 | export type DarkModeProps = { 2 | darkMode?: boolean; 3 | }; 4 | -------------------------------------------------------------------------------- /docs/static/img/rag-diagram.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mongodb/chatbot/HEAD/docs/static/img/rag-diagram.webp -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/.prettierignore: -------------------------------------------------------------------------------- 1 | build/ 2 | package.json 3 | package-lock.json 4 | tsconfig.json 5 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/embed/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./Embedder"; 2 | export * from "./OpenAiEmbedder"; 3 | -------------------------------------------------------------------------------- /docs/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [require.resolve('@docusaurus/core/lib/babel/preset')], 3 | }; 4 | -------------------------------------------------------------------------------- /docs/static/img/ingest-diagram.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mongodb/chatbot/HEAD/docs/static/img/ingest-diagram.webp -------------------------------------------------------------------------------- /docs/static/img/server-diagram.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mongodb/chatbot/HEAD/docs/static/img/server-diagram.webp -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/.prettierignore: -------------------------------------------------------------------------------- 1 | build/ 2 | package.json 3 | package-lock.json 4 | tsconfig.json 5 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/src/sources/mongodb-university/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./MongoDbUniversityDataSource"; 2 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/.env.example: -------------------------------------------------------------------------------- 1 | VITE_SERVER_BASE_URL=http://localhost:3000/api/v1 2 | NODE_ENV="development" 3 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/routes/conversations/constants.ts: -------------------------------------------------------------------------------- 1 | export const creationInterface = "conversations-api"; 2 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/src/IngestEnvVars.ts: -------------------------------------------------------------------------------- 1 | export const INGEST_ENV_VARS = { 2 | MONGODB_CONNECTION_URI: "", 3 | }; 4 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/routes/content/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./contentRouter"; 2 | export * from "./searchContent"; 3 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | 4 | package-lock.json 5 | pnpm-lock.yaml 6 | yarn.lock 7 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/eval/retrievalMetrics/MatchFunc.ts: -------------------------------------------------------------------------------- 1 | export type MatchFunc = (expected: T, actual: T) => boolean; 2 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/findContent/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./DefaultFindContent"; 2 | export * from "./FindContentFunc"; 3 | -------------------------------------------------------------------------------- /packages/benchmarks/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["**/*.test.ts", "test/**/*.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /packages/datasets/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require("../../.eslintrc.cjs"); 2 | module.exports = { 3 | ...baseConfig, 4 | }; 5 | -------------------------------------------------------------------------------- /packages/datasets/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["**/*.test.ts", "test/**/*.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/src/sources/mongodbDotCom/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./WebDataSource"; 2 | export * from "./webSources"; 3 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["**/*.test.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/routes/responses/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./responsesRouter"; 2 | export * from "./createResponse"; 3 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/eval/retrievalMetrics/Primitive.ts: -------------------------------------------------------------------------------- 1 | export type Primitive = string | number | boolean | null | undefined; 2 | -------------------------------------------------------------------------------- /packages/scripts/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require("../../.eslintrc.cjs"); 2 | module.exports = { 3 | ...baseConfig, 4 | }; 5 | -------------------------------------------------------------------------------- /packages/scripts/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["**/*.test.ts", "test/**/*.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /packages/benchmarks/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require("../../.eslintrc.cjs"); 2 | module.exports = { 3 | ...baseConfig, 4 | }; 5 | -------------------------------------------------------------------------------- /packages/datasets/src/treeGeneration/databaseNlQueries/databaseNodes/generateAtlasSearchCode.eval.ts: -------------------------------------------------------------------------------- 1 | // TODO: Iterate on this in EAI-1231 2 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/conversations/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./ConversationsService"; 2 | export * from "./MongoDbConversations"; 3 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["**/*.test.ts", "test/**/*.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-verified-answers/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./importVerifiedAnswers"; 2 | export * from "./parseVerifiedAnswersYaml"; 3 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require("../../.eslintrc.cjs"); 2 | module.exports = { 3 | ...baseConfig, 4 | }; 5 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require("../../.eslintrc.cjs"); 2 | module.exports = { 3 | ...baseConfig, 4 | }; 5 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["**/*.test.ts", "test/**/*.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /packages/performance-tests/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["**/*.test.ts", "test/**/*.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /packages/datasets/mongodb_datasets/__init__.py: -------------------------------------------------------------------------------- 1 | """MongoDB Datasets - Utilities for importing datasets into MongoDB.""" 2 | 3 | __version__ = "0.1.0" 4 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require("../../.eslintrc.cjs"); 2 | module.exports = { 3 | ...baseConfig, 4 | }; 5 | -------------------------------------------------------------------------------- /packages/mongodb-atlas/dump-restore/dump.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source ../.env.dev 3 | mongodump -d $DUMP_DB -o mongodump/ --uri $MONGODB_CONNECTION_URI 4 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/.eslintrc.js: -------------------------------------------------------------------------------- 1 | const baseConfig = require("../../.eslintrc.cjs"); 2 | module.exports = { 3 | ...baseConfig, 4 | }; 5 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/routes/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./conversations"; 2 | export * from "./content"; 3 | export * from "./responses"; 4 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["**/*.test.ts", "**/test/**/*"] 4 | } 5 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/modules.d.ts: -------------------------------------------------------------------------------- 1 | declare module "acquit-require"; 2 | declare module "acquit"; 3 | declare module "turndown-plugin-gfm"; 4 | -------------------------------------------------------------------------------- /packages/release-notes-generator/.prettierignore: -------------------------------------------------------------------------------- 1 | package.json 2 | package-lock.json 3 | tsconfig.json 4 | 5 | node_modules/ 6 | build/ 7 | dist/ 8 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | Jira: https://jira.mongodb.org/browse/EAI- 2 | 3 | ## Changes 4 | 5 | - 6 | - 7 | 8 | ## Notes 9 | 10 | - 11 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/.eslintrc.js: -------------------------------------------------------------------------------- 1 | const baseConfig = require("../../.eslintrc.cjs"); 2 | module.exports = { 3 | ...baseConfig, 4 | }; 5 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/tracing/tagify.ts: -------------------------------------------------------------------------------- 1 | export function tagify(s: string) { 2 | return s.replaceAll(/ /g, "_").toLowerCase(); 3 | } 4 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/.gitignore: -------------------------------------------------------------------------------- 1 | .env* 2 | !.env.example 3 | .secrets* 4 | build/ 5 | node_modules/ 6 | 7 | static/**/* 8 | !static/.gitinclude 9 | -------------------------------------------------------------------------------- /packages/scripts/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/scripts/src/test/constants.ts: -------------------------------------------------------------------------------- 1 | export const IP = "127.0.0.1"; 2 | export const SERVER_PORT = 27017; 3 | export const REPLICA_SET_PORT = 27018; 4 | -------------------------------------------------------------------------------- /packages/benchmarks/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/datasets/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/datasets/src/treeGeneration/databaseNlQueries/databaseNodes/generateNaturalLanguageQueriesAtlasSearch.eval.ts: -------------------------------------------------------------------------------- 1 | // TODO: Iterate on this in EAI-1231 2 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./Config"; 2 | export * from "./ArtifactGeneratorEnvVars"; 3 | export * from "./withConfig"; 4 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-verified-answers/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require("../../.eslintrc.cjs"); 2 | module.exports = { 3 | ...baseConfig, 4 | }; 5 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-verified-answers/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["**/*.test.ts", "test/**/*.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["**/*.test.ts", "**/*.test-d.ts", "test/**/*.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/.gitignore: -------------------------------------------------------------------------------- 1 | .env* 2 | !.env.example 3 | .secrets* 4 | build/ 5 | node_modules/ 6 | 7 | static/**/* 8 | !static/.gitinclude 9 | -------------------------------------------------------------------------------- /packages/mongodb-atlas/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/performance-tests/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/models/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./getOpenAiEndpointAndApiKey"; 2 | export * from "./models"; 3 | export * from "./makeOpenAiClientFactory"; 4 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/frontMatter/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./extractFrontMatter"; 2 | export * from "./removeFrontMatter"; 3 | export * from "./updateFrontMatter"; 4 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/test/jestSetUp.ts: -------------------------------------------------------------------------------- 1 | import { logger } from "../logger"; 2 | // silence logger for tests 3 | logger.transports.forEach((t) => (t.silent = true)); 4 | -------------------------------------------------------------------------------- /packages/benchmarks/src/textToDriver/README.md: -------------------------------------------------------------------------------- 1 | # Text to Driver Benchmark 2 | 3 | This directory contains code for benchmarking generating driver code from natural language text. 4 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/datasets/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./build", 5 | }, 6 | "include": ["./src/**/*.ts"] 7 | } 8 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-verified-answers/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | }; 6 | -------------------------------------------------------------------------------- /scripts-service.dockerfile: -------------------------------------------------------------------------------- 1 | # Build stage 2 | FROM node:20 3 | 4 | WORKDIR /app 5 | COPY . ./ 6 | RUN npm install lerna && npm run bootstrap && npm run build 7 | 8 | WORKDIR /app 9 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["**/*.test.ts", "**/*.eval.ts", "**/eval/**/*", "**/test/**/*"] 4 | } 5 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/src/test/jestSetUp.ts: -------------------------------------------------------------------------------- 1 | import { logger } from "mongodb-rag-core"; 2 | // silence logger for tests 3 | logger.transports.forEach((t) => (t.silent = true)); 4 | -------------------------------------------------------------------------------- /packages/mercury-case-analysis/vitest.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vitest/config"; 2 | 3 | export default defineConfig({ 4 | test: { 5 | globals: true, 6 | }, 7 | }); 8 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/test/jestSetUp.ts: -------------------------------------------------------------------------------- 1 | import { logger } from "mongodb-rag-core"; 2 | // silence logger for tests 3 | logger.transports.forEach((t) => (t.silent = true)); 4 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/test/jestSetUp.ts: -------------------------------------------------------------------------------- 1 | import { logger } from "mongodb-rag-core"; 2 | // silence logger for tests 3 | logger.transports.forEach((t) => (t.silent = true)); 4 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./build" 5 | }, 6 | "include": ["./src/**/*.ts"] 7 | } 8 | -------------------------------------------------------------------------------- /packages/performance-tests/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./build" 5 | }, 6 | "include": ["./src/**/*.ts"] 7 | } 8 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/tracing/traceHandlers/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./addMessageToConversation"; 2 | export * from "./commentMessage"; 3 | export * from "./rateMessage"; 4 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./build" 5 | }, 6 | "include": ["./src/**/*.ts"] 7 | } 8 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/test/README.md: -------------------------------------------------------------------------------- 1 | # Test Utilities 2 | 3 | This directory contains utilities for testing the chatbot server. 4 | They are not included in the production build. 5 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-verified-answers/jest.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | testPathIgnorePatterns: ["/build"], 5 | }; 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | node_modules 4 | 5 | .env* 6 | !.env.qa 7 | !.env.example 8 | 9 | .secrets.* 10 | !.secrets.example 11 | 12 | .vscode 13 | build/ 14 | .ipynb_checkpoints/ 15 | -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "node_modules/lerna/schemas/lerna-schema.json", 3 | "useNx": true, 4 | "useWorkspaces": true, 5 | "version": "independent", 6 | "packages": ["packages/*"] 7 | } 8 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/ArtifactGeneratorEnvVars.ts: -------------------------------------------------------------------------------- 1 | import { CORE_ENV_VARS } from "mongodb-rag-core"; 2 | 3 | export const ArtifactGeneratorEnvVars = { 4 | ...CORE_ENV_VARS, 5 | }; 6 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | ["@babel/preset-env", { targets: { node: "current" } }], 4 | "@babel/preset-typescript", 5 | ], 6 | }; 7 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-verified-answers/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./build" 5 | }, 6 | "include": ["./src/**/*.ts"] 7 | } 8 | -------------------------------------------------------------------------------- /packages/scripts/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./build", 5 | "lib": ["ESNext"] 6 | }, 7 | "include": ["./src/**/*.ts"] 8 | } 9 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | ["@babel/preset-env", { targets: { node: "current" } }], 4 | "@babel/preset-typescript", 5 | ], 6 | }; 7 | -------------------------------------------------------------------------------- /packages/mongodb-atlas/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | 3 | # Ignore all .env files 4 | .env* 5 | # But don't ignore .env.example 6 | !.env.example 7 | 8 | .DS_Store 9 | 10 | dump-restore/mongodump 11 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/ChatbotTrigger.tsx: -------------------------------------------------------------------------------- 1 | import { DarkModeProps } from "./DarkMode"; 2 | 3 | export type ChatbotTriggerProps = DarkModeProps & { 4 | className?: HTMLElement["className"]; 5 | }; 6 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/test/README.md: -------------------------------------------------------------------------------- 1 | # Test Utilities 2 | 3 | This directory contains utilities for testing/evaluating the chatbot server. 4 | They are not included in the production build. 5 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./build", 5 | "rootDir": "./src" 6 | }, 7 | "include": ["./src/**/*.ts"] 8 | } 9 | -------------------------------------------------------------------------------- /packages/release-notes-generator/vitest.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vitest/config"; 2 | 3 | export default defineConfig({ 4 | test: { 5 | globals: true, 6 | watch: false, 7 | }, 8 | }); 9 | -------------------------------------------------------------------------------- /packages/scripts/README.md: -------------------------------------------------------------------------------- 1 | # MongoDB EAI Scripts 2 | 3 | A collection of scripts, some of which are run as cronjobs that the MongoDB EAI team uses. Cronjob scripts can be found in the `environments/` subdirectory. 4 | -------------------------------------------------------------------------------- /docs/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | // This file is not used in compilation. It is here just for a nice editor experience. 3 | "extends": "@docusaurus/tsconfig", 4 | "compilerOptions": { 5 | "baseUrl": "." 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /packages/benchmarks/src/textToDriver/generateDriverCode/languagePrompts/listExamplesInPrompt.ts: -------------------------------------------------------------------------------- 1 | export const listExamplesInPrompt = (topics: string[]) => 2 | topics.map((topic, i) => `${i + 1}. ${topic}`).join("\n"); 3 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./build", 5 | "lib": ["ES2023"] 6 | }, 7 | "include": ["./src/**/*.ts"] 8 | } 9 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/chunk/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./chunkCode"; 2 | export * from "./chunkMd"; 3 | export * from "./chunkOpenApiSpecYaml"; 4 | export * from "./chunkPage"; 5 | export * from "./ChunkTransformer"; 6 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/mongoDbMetadata/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./classifyMetadata"; 2 | export * from "./products"; 3 | export * from "./programmingLanguages"; 4 | export * from "./tags"; 5 | export * from "./topics"; 6 | -------------------------------------------------------------------------------- /packages/mercury-case-analysis/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": [ 4 | "**/*.test.ts", 5 | "**/*.test-d.ts", 6 | "test/**/*.ts", 7 | "**/*.eval.ts" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": [ 4 | "**/*.test.ts", 5 | "**/*.test.tsx", 6 | "test/**/*.ts", 7 | "test/**/*.tsx" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/README.md: -------------------------------------------------------------------------------- 1 | # MongoDB RAG Core 2 | 3 | Core library with shared components used by other MongoDB RAG packages. 4 | 5 | ## Installation 6 | 7 | ```bash 8 | npm install mongodb-rag-core 9 | ``` 10 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./Config"; 2 | export * from "./IngestEnvVars"; 3 | export * from "./IngestMetaStore"; 4 | export * from "./withConfig"; 5 | export * as ragCore from "mongodb-rag-core"; 6 | -------------------------------------------------------------------------------- /packages/benchmarks/src/textToDriver/generateDriverCode/languagePrompts/FewShotExample.ts: -------------------------------------------------------------------------------- 1 | export interface FewShotExample { 2 | input: string; 3 | output: { 4 | content: string; 5 | chainOfThought: string; 6 | }; 7 | } 8 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./build", 5 | "resolveJsonModule": true 6 | }, 7 | "include": ["./src/**/*.ts"] 8 | } 9 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/jest.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | setupFiles: ["/src/test/jestSetUp.ts"], 5 | testPathIgnorePatterns: ["/build"], 6 | }; 7 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/verifiedAnswers/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./DefaultFindVerifiedAnswer"; 2 | export * from "./FindVerifiedAnswerFunc"; 3 | export * from "./MongoDbVerifiedAnswerStore"; 4 | export * from "./VerifiedAnswer"; 5 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/jest.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | setupFiles: ["/src/test/jestSetUp.ts"], 5 | testPathIgnorePatterns: ["/build"], 6 | }; 7 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/eval/retrievalMetrics/assertKIsValid.ts: -------------------------------------------------------------------------------- 1 | import { strict as assert } from "assert"; 2 | export const assertKIsValid = (k: number) => 3 | assert(k > 0 && Number.isInteger(k), "k must be a positive integer"); 4 | -------------------------------------------------------------------------------- /packages/scripts/src/test/jestSetUp.ts: -------------------------------------------------------------------------------- 1 | // Add any Jest setup code here 2 | // For example, you could set up global mocks or extend Jest matchers 3 | 4 | // Increase timeout for tests that interact with MongoDB 5 | jest.setTimeout(30000); 6 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/eval/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./getConversationEvalCasesFromYaml"; 2 | export * from "./getConversationEvalCasesFromCSV"; 3 | export * from "./getConversationEvalCasesFromBraintrust"; 4 | export * from "./retrievalMetrics"; 5 | -------------------------------------------------------------------------------- /docs/docs/contact.md: -------------------------------------------------------------------------------- 1 | # Contact 2 | 3 | 4 | - MongoDB employees can reach out to the Education AI team on Slack at `#ask-education-ai` 5 | - External users can create an issue on the [mongodb/chatbot GitHub repo](https://github.com/mongodb/chatbot/issues/new) -------------------------------------------------------------------------------- /mongodb-corp/README.md: -------------------------------------------------------------------------------- 1 | # A Data Source for MongoDB Corporate Information 2 | 3 | ## Contents 4 | 5 | - The `chatbot` directory contains meta information about the chatbot. This lets it answer questions about itself. 6 | 7 | ## Ingest 8 | 9 | TODO 10 | -------------------------------------------------------------------------------- /packages/benchmarks/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./build", 5 | "module": "NodeNext", 6 | "moduleResolution": "NodeNext" 7 | }, 8 | "include": ["./src/**/*.ts"] 9 | } 10 | -------------------------------------------------------------------------------- /packages/benchmarks/src/test/makeSampleLlmOptions.ts: -------------------------------------------------------------------------------- 1 | import { LlmOptions } from "mongodb-rag-core/executeCode"; 2 | 3 | export const makeSampleLlmOptions = () => { 4 | return { 5 | model: "gpt-4o", 6 | temperature: 0, 7 | } satisfies LlmOptions; 8 | }; 9 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/executeCode/databaseMetadata/LlmOptions.ts: -------------------------------------------------------------------------------- 1 | import { OpenAI } from "openai"; 2 | 3 | export type LlmOptions = Omit< 4 | OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming, 5 | "messages" | "tools" | "tool_choice" 6 | >; 7 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/testData/config/configTest.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | pageStore: () => { 3 | return {}; 4 | }, 5 | embeddedContentStore: () => { 6 | return {}; 7 | }, 8 | embedder: () => { 9 | return {}; 10 | }, 11 | }; 12 | -------------------------------------------------------------------------------- /packages/release-notes-generator/.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | const baseConfig = require(`${__dirname}/../../.prettierrc.cjs`); 2 | 3 | module.exports = { 4 | ...baseConfig, 5 | trailingComma: "all", 6 | tabWidth: 2, 7 | semi: true, 8 | singleQuote: false, 9 | }; 10 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/useLinkData.tsx: -------------------------------------------------------------------------------- 1 | import { useContext } from "react"; 2 | import { LinkDataContext } from "./LinkDataProvider"; 3 | 4 | export function useLinkData() { 5 | const contextLinkData = useContext(LinkDataContext); 6 | return contextLinkData; 7 | } 8 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/useUser.ts: -------------------------------------------------------------------------------- 1 | import { useContext } from "react"; 2 | import { UserContext } from "./UserProvider"; 3 | 4 | export type User = { 5 | name: string; 6 | }; 7 | 8 | export function useUser() { 9 | return useContext(UserContext); 10 | } 11 | -------------------------------------------------------------------------------- /datasets.dockerfile: -------------------------------------------------------------------------------- 1 | # Build stage 2 | FROM node:20 as builder 3 | 4 | WORKDIR /bin 5 | COPY . ./ 6 | RUN npm install lerna && npm run bootstrap && npm run build -- --scope=datasets --include-dependencies 7 | 8 | ENV NODE_ENV=production 9 | 10 | WORKDIR /bin/packages/datasets 11 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/src/sources/snooty/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./SnootyDataSource"; 2 | export * from "./SnootyProjectsInfo"; 3 | export * from "./renderSnootyTable"; 4 | export * from "./rstToSnootyAst"; 5 | export * from "./snootyAstToMd"; 6 | export * from "./snootyAstToOpenApiSpec"; 7 | -------------------------------------------------------------------------------- /packages/mercury-case-analysis/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./build", 5 | "rootDir": "./src", 6 | "module": "ESNext", 7 | "moduleResolution": "Bundler" 8 | }, 9 | "include": ["./src/**/*.ts"] 10 | } 11 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/middleware/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./ipAddressKeyGenerator"; 2 | export * from "./validateRequestSchema"; 3 | export * from "./rateLimit"; 4 | export * from "./requireRequestOrigin"; 5 | export * from "./requireValidIpAddress"; 6 | export * from "./slowDown"; 7 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/jest.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | setupFiles: ["/src/test/jestSetUp.ts"], 5 | globalSetup: "/src/test/globalSetup.ts", 6 | globalTeardown: "/src/test/globalTeardown.ts", 7 | }; 8 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/contentStore/pageIdentity.ts: -------------------------------------------------------------------------------- 1 | import { Page } from "./Page"; 2 | 3 | /** 4 | Returns a query filter that represents a unique page in the system. 5 | */ 6 | export const pageIdentity = ({ url, sourceName }: Page) => ({ 7 | url, 8 | sourceName, 9 | }); 10 | -------------------------------------------------------------------------------- /packages/release-notes-generator/src/configs/index.ts: -------------------------------------------------------------------------------- 1 | export { default as atlasCliConfig } from "./atlasCli.config"; 2 | export { default as mongoshConfig } from "./mongosh.config"; 3 | export { default as relationalMigratorConfig } from "./relationalMigrator.config"; 4 | export * from "./helpers"; 5 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/ui-text.ts: -------------------------------------------------------------------------------- 1 | export const mongoDbVerifyInformationMessage = 2 | "Our chatbot can make mistakes, so double-check important info."; 3 | 4 | export const defaultChatbotFatalErrorMessage = 5 | "Something went wrong. Try reloading the page and starting a new conversation."; 6 | -------------------------------------------------------------------------------- /packages/benchmarks/jest.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | testPathIgnorePatterns: ["/build"], 5 | verbose: true, 6 | globalSetup: "/src/test/globalSetup.ts", 7 | globalTeardown: "/src/test/globalTeardown.ts", 8 | }; 9 | -------------------------------------------------------------------------------- /packages/benchmarks/src/index.ts: -------------------------------------------------------------------------------- 1 | export { makeReferenceAlignment } from "./nlPromptResponse/metrics"; 2 | export { 3 | NlPromptResponseEvalCaseInput, 4 | NlPromptResponseMetadata, 5 | NlPromptResponseTaskExpected, 6 | NlPromptResponseTaskOutput, 7 | } from "./nlPromptResponse/NlQuestionAnswerEval"; 8 | -------------------------------------------------------------------------------- /packages/datasets/jest.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | testPathIgnorePatterns: ["/build"], 5 | verbose: true, 6 | globalSetup: "/src/test/globalSetup.ts", 7 | globalTeardown: "/src/test/globalTeardown.ts", 8 | }; 9 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/middleware/ipAddressKeyGenerator.ts: -------------------------------------------------------------------------------- 1 | import { Request } from "express"; 2 | 3 | export function ipAddressKeyGenerator(request: Request) { 4 | if (!request.ip) { 5 | throw new Error("Request IP is not defined"); 6 | } 7 | 8 | return request.ip; 9 | } 10 | -------------------------------------------------------------------------------- /packages/release-notes-generator/src/llm/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./classifyChangelog"; 2 | export * from "./classifyChangelogAudience"; 3 | export * from "./classifyChangelogScope"; 4 | export * from "./createChangelogEntry"; 5 | export * from "./openai-api"; 6 | export * from "./summarizeReleaseArtifact"; 7 | -------------------------------------------------------------------------------- /packages/benchmarks/src/textToDriver/TextToDriverEnvVars.ts: -------------------------------------------------------------------------------- 1 | export const BRAINTRUST_ENV_VARS = { 2 | BRAINTRUST_API_KEY: "", 3 | BRAINTRUST_TEXT_TO_DRIVER_PROJECT_NAME: "", 4 | }; 5 | export const TEXT_TO_DRIVER_ENV_VARS = { 6 | ...BRAINTRUST_ENV_VARS, 7 | MONGODB_TEXT_TO_DRIVER_CONNECTION_URI: "", 8 | }; 9 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/main.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import ReactDOM from "react-dom/client"; 3 | import App from "./App.tsx"; 4 | 5 | ReactDOM.createRoot(document.getElementById("root") as HTMLElement).render( 6 | 7 | 8 | 9 | ); 10 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "lib": ["ES2022"], 5 | "target": "ES2022", 6 | "outDir": "./build", 7 | "resolveJsonModule": true, 8 | }, 9 | "include": ["./src/**/*.ts", "./src/**/*.json"], 10 | } 11 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "composite": true, 4 | "skipLibCheck": true, 5 | "module": "ESNext", 6 | "moduleResolution": "bundler", 7 | "allowSyntheticDefaultImports": true 8 | }, 9 | "include": ["vite.config.component.ts"] 10 | } 11 | -------------------------------------------------------------------------------- /packages/release-notes-generator/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./artifact"; 2 | export * from "./change"; 3 | export * from "./config"; 4 | export * from "./configs"; 5 | export * from "./generate"; 6 | export * from "./llm"; 7 | export * from "./logger"; 8 | export * from "./github"; 9 | export * from "./jira"; 10 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/dataSources/makeMarkdownLists.ts: -------------------------------------------------------------------------------- 1 | export function makeMarkdownNumberedList(items: string[]) { 2 | return items.map((item, i) => `${i + 1}. ${item}`).join("\n"); 3 | } 4 | 5 | export function makeMarkdownUnorderedList(items: string[]) { 6 | return items.map((item) => `- ${item}`).join("\n"); 7 | } 8 | -------------------------------------------------------------------------------- /packages/scripts/.env.example: -------------------------------------------------------------------------------- 1 | MONGODB_CONNECTION_URI= 2 | MONGODB_DATABASE_NAME=docs-chatbot-qa 3 | 4 | FROM_CONNECTION_URI= 5 | FROM_DATABASE_NAME= 6 | FROM_FAQ_COLLECTION_NAME= 7 | FROM_SCRUBBED_COLLECTION_NAME= 8 | TO_CONNECTION_URI= 9 | TO_DATABASE_NAME= 10 | TO_FAQ_COLLECTION_NAME= 11 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/eval/retrievalMetrics/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./assertKIsValid"; 2 | export * from "./averagePrecisionAtK"; 3 | export * from "./binaryNdcgAtK"; 4 | export * from "./f1AtK"; 5 | export * from "./MatchFunc"; 6 | export * from "./precisionAtK"; 7 | export * from "./Primitive"; 8 | export * from "./recallAtK"; 9 | -------------------------------------------------------------------------------- /packages/scripts/jest.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | testPathIgnorePatterns: ["/build"], 5 | setupFiles: ["/src/test/jestSetUp.ts"], 6 | globalSetup: "/src/test/globalSetup.ts", 7 | globalTeardown: "/src/test/globalTeardown.ts", 8 | }; 9 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | @fileoverview This file contains the exported modules and types 3 | for using the chat server library. 4 | */ 5 | export * from "./app"; 6 | export * from "./routes"; 7 | export * from "./processors"; 8 | export * from "./middleware"; 9 | export * from "mongodb-rag-core"; 10 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/.env.example: -------------------------------------------------------------------------------- 1 | # Chatbot App 2 | MONGODB_CONNECTION_URI= 3 | 4 | # Data Sources 5 | ## MongoDB Developer Center 6 | DEVCENTER_CONNECTION_URI="mongodb+srv://:@devhub-cluster.sewho.mongodb.net/?retryWrites=true&w=majority" 7 | ## MongoDB University 8 | UNIVERSITY_DATA_API_KEY=abc123 9 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/jest.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | setupFiles: ["/src/test/jestSetUp.ts"], 5 | testPathIgnorePatterns: ["/build"], 6 | globalSetup: "/src/test/globalSetup.ts", 7 | globalTeardown: "/src/test/globalTeardown.ts", 8 | }; 9 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | interface ImportMetaEnv { 3 | readonly VITE_SERVER_BASE_URL: string; 4 | readonly VITE_QA?: string; 5 | readonly VITE_GIT_COMMIT?: string; 6 | // more env variables... 7 | } 8 | 9 | interface ImportMeta { 10 | readonly env: ImportMetaEnv; 11 | } 12 | -------------------------------------------------------------------------------- /packages/release-notes-generator/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "target": "ES2020", 5 | "module": "ESNext", 6 | "moduleResolution": "Bundler", 7 | "esModuleInterop": true, 8 | "strict": true, 9 | "skipLibCheck": true 10 | }, 11 | "include": ["src/**/*"] 12 | } 13 | -------------------------------------------------------------------------------- /packages/datasets/src/README.md: -------------------------------------------------------------------------------- 1 | # MongoDB Education Datasets 2 | 3 | This package contains scripts to generate datasets related to MongoDB Education. 4 | 5 | These datasets can be useful for use in AI systems, such as RAG chatbots or fine-tuning LLMs. 6 | 7 | Public datasets may be found on the [MongoDB Education AI HuggingFace space](https://huggingface.co/mongodb-eai). -------------------------------------------------------------------------------- /packages/scripts/src/test/globals.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | 3 | declare global { 4 | // eslint-disable-next-line no-var 5 | var __MONGO_MEMORY_SERVER_INSTANCE: MongoMemoryServer; 6 | // eslint-disable-next-line no-var 7 | var __MONGO_MEMORY_REPLICA_SET: MongoMemoryReplSet; 8 | } 9 | 10 | export {}; 11 | -------------------------------------------------------------------------------- /packages/benchmarks/src/test/globals.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | 3 | declare global { 4 | // eslint-disable-next-line no-var 5 | var __MONGO_MEMORY_SERVER_INSTANCE: MongoMemoryServer; 6 | // eslint-disable-next-line no-var 7 | var __MONGO_MEMORY_REPLICA_SET: MongoMemoryReplSet; 8 | } 9 | 10 | export {}; 11 | -------------------------------------------------------------------------------- /packages/datasets/src/test/globals.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | 3 | declare global { 4 | // eslint-disable-next-line no-var 5 | var __MONGO_MEMORY_SERVER_INSTANCE: MongoMemoryServer; 6 | // eslint-disable-next-line no-var 7 | var __MONGO_MEMORY_REPLICA_SET: MongoMemoryReplSet; 8 | } 9 | 10 | export {}; 11 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/jest.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "preset": "ts-jest", 3 | "setupFiles": ["/src/test/jestEnvSetUp.ts"], 4 | "setupFilesAfterEnv": ["/src/test/jestSetUp.ts"], 5 | "testPathIgnorePatterns": ["/build"], 6 | "globalSetup": "/src/test/globalSetup.ts", 7 | "globalTeardown": "/src/test/globalTeardown.ts" 8 | } 9 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/executeCode/prettyPrintMongoDbDocument.ts: -------------------------------------------------------------------------------- 1 | import { BSON } from "mongodb"; 2 | 3 | /** 4 | Converts MongoDB documents with BSON types to a readable string format using EJSON. 5 | */ 6 | export function prettyPrintMongoDbDocument(document: unknown): string { 7 | return BSON.EJSON.stringify(document, { 8 | relaxed: true, 9 | }); 10 | } 11 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/test/globals.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | 3 | declare global { 4 | // eslint-disable-next-line no-var 5 | var __MONGO_MEMORY_SERVER_INSTANCE: MongoMemoryServer; 6 | // eslint-disable-next-line no-var 7 | var __MONGO_MEMORY_REPLICA_SET: MongoMemoryReplSet; 8 | } 9 | 10 | export {}; 11 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/testData/config/configTest.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ingestMetaStore: () => { 3 | return {}; 4 | }, 5 | pageStore: () => { 6 | return {}; 7 | }, 8 | embeddedContentStore: () => { 9 | return {}; 10 | }, 11 | dataSources: () => { 12 | return {}; 13 | }, 14 | embedder: () => { 15 | return {}; 16 | }, 17 | }; 18 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/jest.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "preset": "ts-jest", 3 | "testEnvironment": "node", 4 | "setupFiles": [ 5 | "/src/test/jestSetUp.ts" 6 | ], 7 | "testPathIgnorePatterns": [ 8 | "/build" 9 | ], 10 | "globalSetup": "/src/test/globalSetup.ts", 11 | "globalTeardown": "/src/test/globalTeardown.ts" 12 | } -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/test/globals.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | 3 | declare global { 4 | // eslint-disable-next-line no-var 5 | var __MONGO_MEMORY_SERVER_INSTANCE: MongoMemoryServer; 6 | // eslint-disable-next-line no-var 7 | var __MONGO_MEMORY_REPLICA_SET: MongoMemoryReplSet; 8 | } 9 | 10 | export {}; 11 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/frontMatter/removeFrontMatter.ts: -------------------------------------------------------------------------------- 1 | import frontmatter, { FrontMatterOptions } from "front-matter"; 2 | export function removeFrontMatter( 3 | content: string, 4 | options?: FrontMatterOptions 5 | ): string { 6 | const noFrontMatterContent = frontmatter(content.trimStart(), options).body; 7 | 8 | return noFrontMatterContent.trimStart(); 9 | } 10 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/src/test/globals.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | 3 | declare global { 4 | // eslint-disable-next-line no-var 5 | var __MONGO_MEMORY_SERVER_INSTANCE: MongoMemoryServer; 6 | // eslint-disable-next-line no-var 7 | var __MONGO_MEMORY_REPLICA_SET: MongoMemoryReplSet; 8 | } 9 | 10 | export {}; 11 | -------------------------------------------------------------------------------- /packages/benchmarks/src/envVars.ts: -------------------------------------------------------------------------------- 1 | export const envVars = { 2 | MONGODB_DATABASE_NAME: "", 3 | MONGODB_CONNECTION_URI: "", 4 | MONGODB_QUIZ_QUESTIONS_DATABASE_NAME: "", 5 | MONGODB_QUIZ_QUESTIONS_CONNECTION_URI: "", 6 | MONGODB_QUIZ_QUESTIONS_COLLECTION_NAME: "", 7 | }; 8 | 9 | export const TEXT_TO_DRIVER_ENV_VARS = { 10 | MONGODB_TEXT_TO_DRIVER_CONNECTION_URI: "", 11 | }; 12 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/test/globals.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | 3 | declare global { 4 | // eslint-disable-next-line no-var 5 | var __MONGO_MEMORY_SERVER_INSTANCE: MongoMemoryServer; 6 | // eslint-disable-next-line no-var 7 | var __MONGO_MEMORY_REPLICA_SET: MongoMemoryReplSet; 8 | } 9 | 10 | export {}; 11 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | /node_modules 3 | 4 | # Production 5 | /build 6 | 7 | # Generated files 8 | .docusaurus 9 | .cache-loader 10 | 11 | # Misc 12 | .DS_Store 13 | .env.local 14 | .env.development.local 15 | .env.test.local 16 | .env.production.local 17 | 18 | npm-debug.log* 19 | yarn-debug.log* 20 | yarn-error.log* 21 | 22 | /docs/reference/* 23 | !/docs/reference/.gitinclude 24 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/middleware/blockGetRequests.ts: -------------------------------------------------------------------------------- 1 | import { ConversationsMiddleware } from "mongodb-chatbot-server"; 2 | 3 | export const blockGetRequests: ConversationsMiddleware = (req, res, next) => { 4 | if (req.method === "GET") { 5 | return res.status(404).json({ 6 | message: "Route not found", 7 | }); 8 | } else { 9 | next(); 10 | } 11 | }; 12 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/.storybook/preview.ts: -------------------------------------------------------------------------------- 1 | import type { Preview } from "@storybook/react"; 2 | 3 | const preview: Preview = { 4 | parameters: { 5 | actions: { argTypesRegex: "^on[A-Z].*" }, 6 | controls: { 7 | matchers: { 8 | color: /(background|color)$/i, 9 | date: /Date$/, 10 | }, 11 | }, 12 | }, 13 | }; 14 | 15 | export default preview; 16 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/useConversationStateContext.tsx: -------------------------------------------------------------------------------- 1 | import { useContext } from "react"; 2 | import { ConversationStateContext } from "./ConversationStateProvider"; 3 | import { useStore } from "zustand/react"; 4 | 5 | export function useConversationStateContext() { 6 | const store = useContext(ConversationStateContext); 7 | const state = useStore(store); 8 | return state; 9 | } 10 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/dataSources/DataSource.ts: -------------------------------------------------------------------------------- 1 | import { Page } from "../contentStore"; 2 | 3 | /** 4 | Represents a source of page data. 5 | */ 6 | export type DataSource = { 7 | /** 8 | The unique name among registered data sources. 9 | */ 10 | name: string; 11 | 12 | /** 13 | Fetches pages in the data source. 14 | */ 15 | fetchPages(): Promise; 16 | }; 17 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/lib.ts: -------------------------------------------------------------------------------- 1 | /** 2 | @fileoverview 3 | Export some modules from the implementation for use in things like evaluation. 4 | */ 5 | export { systemPrompt } from "./systemPrompt"; 6 | export * as mongoDbMetadata from "mongodb-rag-core"; 7 | export * from "./tracing/scrubbedMessages/ScrubbedMessage"; 8 | export { MessageAnalysis } from "./tracing/scrubbedMessages/analyzeMessage"; 9 | -------------------------------------------------------------------------------- /packages/mercury-case-analysis/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./analyzeCases"; 2 | export * from "./rating"; 3 | export * from "./relevance"; 4 | export * from "./suggestRewrite"; 5 | export { 6 | EmbeddingModel, 7 | LanguageModel, 8 | createAzure, 9 | createOpenAI, 10 | wrapLanguageModel, 11 | } from "mongodb-rag-core/aiSdk"; 12 | export { BraintrustMiddleware } from "mongodb-rag-core/braintrust"; 13 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/useChatbotContext.tsx: -------------------------------------------------------------------------------- 1 | import { useContext } from "react"; 2 | import { ChatbotContext } from "./ChatbotProvider"; 3 | 4 | export function useChatbotContext() { 5 | const chatbotContext = useContext(ChatbotContext); 6 | if (!chatbotContext) { 7 | throw new Error("useChatbotContext must be used within a ChatbotProvider"); 8 | } 9 | return chatbotContext; 10 | } 11 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/middleware/slowDown.test.ts: -------------------------------------------------------------------------------- 1 | import { makeSlowDown } from "./slowDown"; 2 | 3 | describe("makeSlowDown", () => { 4 | it("should return a middleware function", () => { 5 | const middleware = makeSlowDown(); 6 | 7 | expect(typeof middleware).toBe("function"); 8 | expect(middleware.length).toBe(3); // Express middleware has 3 params: req, res, next 9 | }); 10 | }); 11 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/executeCode/redactMongoDbConnectionUri.ts: -------------------------------------------------------------------------------- 1 | /** 2 | Replace MongoDB conneciton URI username and password 3 | with `":"` so that the URI is safe to log/persist. 4 | */ 5 | export function redactMongoDbConnectionUri(uri: string) { 6 | return uri.replace( 7 | /mongodb(\+srv)?:\/\/([^:]+):([^@]+)@/g, 8 | "mongodb$1://:@" 9 | ); 10 | } 11 | -------------------------------------------------------------------------------- /packages/datasets/src/treeGeneration/databaseNlQueries/databaseMetadata/prettyPrintMongoDbDocument.ts: -------------------------------------------------------------------------------- 1 | import { BSON } from "mongodb-rag-core/mongodb"; 2 | 3 | /** 4 | Converts MongoDB documents with BSON types to a readable string format using EJSON. 5 | */ 6 | export function prettyPrintMongoDbDocument(document: unknown): string { 7 | return BSON.EJSON.stringify(document, { 8 | relaxed: true, 9 | }); 10 | } 11 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/middleware/rateLimit.test.ts: -------------------------------------------------------------------------------- 1 | import { makeRateLimit } from "./rateLimit"; 2 | 3 | describe("makeRateLimit", () => { 4 | it("should return a middleware function", () => { 5 | const middleware = makeRateLimit(); 6 | 7 | expect(typeof middleware).toBe("function"); 8 | expect(middleware.length).toBe(3); // Express middleware has 3 params: req, res, next 9 | }); 10 | }); 11 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/testData/sampleMdxFile.mdx: -------------------------------------------------------------------------------- 1 | +++ 2 | title = "Hello, MDX!" 3 | date = "2024-01-01T00:00:00.000Z" 4 | +++ 5 | 6 | # Hello, MDX! 7 | 8 | This is an MDX file! 9 | 10 | It's got standard markdown features like code fences: 11 | 12 | ```js title="SomeComponent.js" 13 | import { sum } from "./math"; 14 | const three = sum(1, 2); 15 | ``` 16 | 17 | 18 | 19 | It renders JSX wow! 20 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/vitest.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig, mergeConfig } from "vitest/config"; 2 | import viteConfig from "./vite.config.component"; 3 | 4 | const testConfig = defineConfig({ 5 | test: { 6 | environment: "jsdom", 7 | globals: true, 8 | setupFiles: ["./vitest.setup.ts"], 9 | }, 10 | }); 11 | 12 | export default defineConfig((configEnv) => 13 | mergeConfig(viteConfig(configEnv), testConfig) 14 | ); 15 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/createCommand.ts: -------------------------------------------------------------------------------- 1 | import { CommandModule } from "yargs"; 2 | import { LoadConfigArgs } from "./withConfig"; 3 | 4 | type CMD> = 5 | CommandModule; 6 | 7 | export function createCommand< 8 | T extends { [k: string]: unknown } = Record 9 | >(commandModule: CMD) { 10 | return commandModule; 11 | } 12 | -------------------------------------------------------------------------------- /packages/benchmarks/src/test/constants.ts: -------------------------------------------------------------------------------- 1 | const IP = "127.0.0.1"; 2 | const REPLICA_SET_PORT = 47018; 3 | const SERVER_PORT = 47017; 4 | const MONGO_MEMORY_REPLICA_SET_URI = `mongodb://${IP}:${REPLICA_SET_PORT}/?replicaSet=testset`; 5 | const MONGO_MEMORY_SERVER_URI = `mongodb://${IP}:${SERVER_PORT}/`; 6 | 7 | export { 8 | IP, 9 | REPLICA_SET_PORT, 10 | SERVER_PORT, 11 | MONGO_MEMORY_REPLICA_SET_URI, 12 | MONGO_MEMORY_SERVER_URI, 13 | }; 14 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/global.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-namespace */ 2 | import "jest"; 3 | declare global { 4 | namespace jest { 5 | interface Matchers { 6 | /** 7 | For use with the LLM qualitative testing framework. 8 | @param expected - The expected output description 9 | */ 10 | toMeetChatQualityStandard: (expected: string) => Promise; 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /packages/datasets/src/test/constants.ts: -------------------------------------------------------------------------------- 1 | const IP = "127.0.0.1"; 2 | const REPLICA_SET_PORT = 47018; 3 | const SERVER_PORT = 47017; 4 | const MONGO_MEMORY_REPLICA_SET_URI = `mongodb://${IP}:${REPLICA_SET_PORT}/?replicaSet=testset`; 5 | const MONGO_MEMORY_SERVER_URI = `mongodb://${IP}:${SERVER_PORT}/`; 6 | 7 | export { 8 | IP, 9 | REPLICA_SET_PORT, 10 | SERVER_PORT, 11 | MONGO_MEMORY_REPLICA_SET_URI, 12 | MONGO_MEMORY_SERVER_URI, 13 | }; 14 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/processors/filterOnlySystemPrompt.ts: -------------------------------------------------------------------------------- 1 | import { FilterPreviousMessages } from "./FilterPreviousMessages"; 2 | 3 | /** 4 | Only include the initial system prompt. 5 | */ 6 | export const filterOnlySystemPrompt: FilterPreviousMessages = async ( 7 | conversation 8 | ) => { 9 | const systemPrompt = conversation.messages.find((m) => m.role === "system"); 10 | return systemPrompt ? [systemPrompt] : []; 11 | }; 12 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/executeCode/databaseMetadata/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./extractDeterministicSampleOfDocuments"; 2 | export * from "./generateAnnotatedCollectionSchema"; 3 | export * from "./generateHighLevelDbDescriptions"; 4 | export * from "./generateAnnotatedDatabaseInfo"; 5 | export * from "./getDatabaseMetadata"; 6 | export * from "./getVerySimplifiedSchema"; 7 | export * from "./getOpenAiFunctionResponse"; 8 | export * from "./LlmOptions"; 9 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/test/constants.ts: -------------------------------------------------------------------------------- 1 | const IP = "127.0.0.1"; 2 | const REPLICA_SET_PORT = 27018; 3 | const SERVER_PORT = 27017; 4 | const MONGO_MEMORY_REPLICA_SET_URI = `mongodb://${IP}:${REPLICA_SET_PORT}/?replicaSet=testset`; 5 | const MONGO_MEMORY_SERVER_URI = `mongodb://${IP}:${SERVER_PORT}/`; 6 | 7 | export { 8 | IP, 9 | REPLICA_SET_PORT, 10 | SERVER_PORT, 11 | MONGO_MEMORY_REPLICA_SET_URI, 12 | MONGO_MEMORY_SERVER_URI, 13 | }; 14 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/test/constants.ts: -------------------------------------------------------------------------------- 1 | const IP = "127.0.0.1"; 2 | const REPLICA_SET_PORT = 57018; 3 | const SERVER_PORT = 57017; 4 | const MONGO_MEMORY_REPLICA_SET_URI = `mongodb://${IP}:${REPLICA_SET_PORT}/?replicaSet=testset`; 5 | const MONGO_MEMORY_SERVER_URI = `mongodb://${IP}:${SERVER_PORT}/`; 6 | 7 | export { 8 | IP, 9 | REPLICA_SET_PORT, 10 | SERVER_PORT, 11 | MONGO_MEMORY_REPLICA_SET_URI, 12 | MONGO_MEMORY_SERVER_URI, 13 | }; 14 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | MongoDB Chatbot Tester 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/src/test/constants.ts: -------------------------------------------------------------------------------- 1 | const IP = "127.0.0.1"; 2 | const REPLICA_SET_PORT = 37018; 3 | const SERVER_PORT = 37017; 4 | const MONGO_MEMORY_REPLICA_SET_URI = `mongodb://${IP}:${REPLICA_SET_PORT}/?replicaSet=testset`; 5 | const MONGO_MEMORY_SERVER_URI = `mongodb://${IP}:${SERVER_PORT}/`; 6 | 7 | export { 8 | IP, 9 | REPLICA_SET_PORT, 10 | SERVER_PORT, 11 | MONGO_MEMORY_REPLICA_SET_URI, 12 | MONGO_MEMORY_SERVER_URI, 13 | }; 14 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | build 12 | build-ssr 13 | *.local 14 | 15 | !.env.staging 16 | 17 | # Editor directories and files 18 | .vscode/* 19 | !.vscode/extensions.json 20 | .idea 21 | .DS_Store 22 | *.suo 23 | *.ntvs* 24 | *.njsproj 25 | *.sln 26 | *.sw? 27 | 28 | # Build analysis 29 | stats.html 30 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/test/constants.ts: -------------------------------------------------------------------------------- 1 | const IP = "127.0.0.1"; 2 | const REPLICA_SET_PORT = 47018; 3 | const SERVER_PORT = 47017; 4 | const MONGO_MEMORY_REPLICA_SET_URI = `mongodb://${IP}:${REPLICA_SET_PORT}/?replicaSet=testset`; 5 | const MONGO_MEMORY_SERVER_URI = `mongodb://${IP}:${SERVER_PORT}/`; 6 | 7 | export { 8 | IP, 9 | REPLICA_SET_PORT, 10 | SERVER_PORT, 11 | MONGO_MEMORY_REPLICA_SET_URI, 12 | MONGO_MEMORY_SERVER_URI, 13 | }; 14 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/UserProvider.tsx: -------------------------------------------------------------------------------- 1 | import { createContext, ReactNode } from "react"; 2 | import { User } from "./useUser"; 3 | 4 | export const UserContext = createContext(undefined); 5 | 6 | type UserProviderProps = { 7 | children: ReactNode; 8 | user?: User; 9 | }; 10 | 11 | export function UserProvider({ children, user }: UserProviderProps) { 12 | return {children}; 13 | } 14 | -------------------------------------------------------------------------------- /packages/release-notes-generator/src/utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | Given an arbitrary string, replace invalid characters with a dash to make it a valid filename. 3 | */ 4 | export function safeFileName(fileName: string): string { 5 | return fileName.replace(/[/\\?%*:|"<>]/g, "-"); 6 | } 7 | 8 | /** 9 | Returns a timestamp in the format YYYY-MM-DD-HH-MM-SS 10 | */ 11 | export function currentTimestamp(): string { 12 | return new Date().toISOString().replace(/[:.]/g, "-"); 13 | } 14 | -------------------------------------------------------------------------------- /packages/datasets/src/test/globalTeardown.ts: -------------------------------------------------------------------------------- 1 | import "./globals"; 2 | 3 | module.exports = async function () { 4 | try { 5 | if (global.__MONGO_MEMORY_SERVER_INSTANCE) { 6 | await global.__MONGO_MEMORY_SERVER_INSTANCE.stop(); 7 | } 8 | if (global.__MONGO_MEMORY_REPLICA_SET) { 9 | await global.__MONGO_MEMORY_REPLICA_SET.stop(); 10 | } 11 | } catch (error) { 12 | console.error("Error in global teardown:", error); 13 | throw error; 14 | } 15 | }; 16 | -------------------------------------------------------------------------------- /packages/scripts/src/test/globalTeardown.ts: -------------------------------------------------------------------------------- 1 | import "./globals"; 2 | 3 | export default async function () { 4 | try { 5 | if (global.__MONGO_MEMORY_SERVER_INSTANCE) { 6 | await global.__MONGO_MEMORY_SERVER_INSTANCE.stop(); 7 | } 8 | if (global.__MONGO_MEMORY_REPLICA_SET) { 9 | await global.__MONGO_MEMORY_REPLICA_SET.stop(); 10 | } 11 | } catch (error) { 12 | console.error("Error in global teardown:", error); 13 | throw error; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /packages/benchmarks/src/test/globalTeardown.ts: -------------------------------------------------------------------------------- 1 | import "./globals"; 2 | 3 | module.exports = async function () { 4 | try { 5 | if (global.__MONGO_MEMORY_SERVER_INSTANCE) { 6 | await global.__MONGO_MEMORY_SERVER_INSTANCE.stop(); 7 | } 8 | if (global.__MONGO_MEMORY_REPLICA_SET) { 9 | await global.__MONGO_MEMORY_REPLICA_SET.stop(); 10 | } 11 | } catch (error) { 12 | console.error("Error in global teardown:", error); 13 | throw error; 14 | } 15 | }; 16 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/src/test/globalTeardown.ts: -------------------------------------------------------------------------------- 1 | import "./globals"; 2 | module.exports = async function () { 3 | try { 4 | if (global.__MONGO_MEMORY_SERVER_INSTANCE) { 5 | await global.__MONGO_MEMORY_SERVER_INSTANCE.stop(); 6 | } 7 | if (global.__MONGO_MEMORY_REPLICA_SET) { 8 | await global.__MONGO_MEMORY_REPLICA_SET.stop(); 9 | } 10 | } catch (error) { 11 | console.error("Error in global teardown:", error); 12 | throw error; 13 | } 14 | }; 15 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/test/globalTeardown.ts: -------------------------------------------------------------------------------- 1 | import "./globals"; 2 | 3 | module.exports = async function () { 4 | try { 5 | if (global.__MONGO_MEMORY_SERVER_INSTANCE) { 6 | await global.__MONGO_MEMORY_SERVER_INSTANCE.stop(); 7 | } 8 | if (global.__MONGO_MEMORY_REPLICA_SET) { 9 | await global.__MONGO_MEMORY_REPLICA_SET.stop(); 10 | } 11 | } catch (error) { 12 | console.error("Error in global teardown:", error); 13 | throw error; 14 | } 15 | }; 16 | -------------------------------------------------------------------------------- /packages/benchmarks/src/textToDriver/DatabaseMetadata.ts: -------------------------------------------------------------------------------- 1 | import { CreateIndexesOptions, IndexDirection } from "mongodb-rag-core/mongodb"; 2 | 3 | export interface DatabaseMetadata { 4 | databaseName: string; 5 | collections: CollectionMetadata[]; 6 | } 7 | 8 | export interface CollectionMetadata { 9 | collectionName: string; 10 | indexes: { 11 | v: CreateIndexesOptions["version"]; 12 | key: Map; 13 | name: CreateIndexesOptions["name"]; 14 | }[]; 15 | } 16 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/test/globalTeardown.ts: -------------------------------------------------------------------------------- 1 | import "./globals"; 2 | 3 | module.exports = async function () { 4 | try { 5 | if (global.__MONGO_MEMORY_SERVER_INSTANCE) { 6 | await global.__MONGO_MEMORY_SERVER_INSTANCE.stop(); 7 | } 8 | if (global.__MONGO_MEMORY_REPLICA_SET) { 9 | await global.__MONGO_MEMORY_REPLICA_SET.stop(); 10 | } 11 | } catch (error) { 12 | console.error("Error in global teardown:", error); 13 | throw error; 14 | } 15 | }; 16 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/processors/FilterPreviousMessages.ts: -------------------------------------------------------------------------------- 1 | import { Conversation, Message } from "mongodb-rag-core"; 2 | 3 | /** 4 | Filters which previous conversation messages are sent to the LLM 5 | along with the user prompt. 6 | For example, you may only want to send the system prompt to the LLM 7 | with the user message or the system prompt and X prior messages. 8 | */ 9 | export type FilterPreviousMessages = ( 10 | conversation: Conversation 11 | ) => Promise; 12 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/test/globalTeardown.ts: -------------------------------------------------------------------------------- 1 | import "./globals"; 2 | 3 | module.exports = async function () { 4 | try { 5 | if (global.__MONGO_MEMORY_SERVER_INSTANCE) { 6 | await global.__MONGO_MEMORY_SERVER_INSTANCE.stop(); 7 | } 8 | if (global.__MONGO_MEMORY_REPLICA_SET) { 9 | await global.__MONGO_MEMORY_REPLICA_SET.stop(); 10 | } 11 | } catch (error) { 12 | console.error("Error in global teardown:", error); 13 | throw error; 14 | } 15 | }; 16 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-verified-answers/example-verified-answers.yaml: -------------------------------------------------------------------------------- 1 | - questions: 2 | - How do I verify answers? 3 | answer: To verify an answer... 4 | author_email: someone@example.com 5 | references: 6 | - url: http://example.com 7 | title: Example 8 | - questions: 9 | - Why did the chicken cross the road? 10 | answer: To get to the other side. 11 | author_email: someone@example.com 12 | hidden: true 13 | references: 14 | - url: http://example.com 15 | title: Example 16 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/verifiedAnswers/FindVerifiedAnswerFunc.ts: -------------------------------------------------------------------------------- 1 | import { WithScore } from "../VectorStore"; 2 | import { VerifiedAnswer } from "./VerifiedAnswer"; 3 | 4 | export type FindVerifiedAnswerFuncArgs = { 5 | query: string; 6 | }; 7 | 8 | export type FindVerifiedAnswerFunc = ( 9 | args: FindVerifiedAnswerFuncArgs 10 | ) => Promise; 11 | 12 | export type FindVerifiedAnswerResult = { 13 | queryEmbedding: number[]; 14 | answer?: WithScore; 15 | }; 16 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2020", 4 | "lib": ["ES2023", "DOM"], 5 | "module": "NodeNext", 6 | "moduleResolution": "NodeNext", 7 | "allowJs": false, 8 | "declaration": true, 9 | "declarationMap": true, 10 | "sourceMap": true, 11 | "downlevelIteration": true, 12 | "esModuleInterop": true, 13 | "forceConsistentCasingInFileNames": true, 14 | "strictNullChecks": true, 15 | "skipLibCheck": true, 16 | "strict": true 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/dataSources/removeMarkdownFileExtension.ts: -------------------------------------------------------------------------------- 1 | /** 2 | Utility function to remove a markdown file extension (.md or .markdown) 3 | from a string. 4 | Can be used in MarkdownUrlDataSource to transform markdown URLs to page URLs. 5 | */ 6 | export function removeMarkdownFileExtension(url: string): string { 7 | if (url.endsWith(".md")) { 8 | return url.slice(0, -3); 9 | } else if (url.endsWith(".markdown")) { 10 | return url.slice(0, -9); 11 | } 12 | return url; 13 | } 14 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/test/jestSetUp.ts: -------------------------------------------------------------------------------- 1 | import { logger } from "mongodb-chatbot-server"; 2 | 3 | // silence logger for tests 4 | logger.transports.forEach((t) => (t.silent = true)); 5 | 6 | // Mock autoevals to avoid ESM parsing issues 7 | jest.mock("autoevals", () => ({ 8 | Faithfulness: jest.fn().mockImplementation(() => ({ score: 1 })), 9 | AnswerRelevancy: jest.fn().mockImplementation(() => ({ score: 1 })), 10 | ContextRelevancy: jest.fn().mockImplementation(() => ({ score: 1 })), 11 | })); 12 | -------------------------------------------------------------------------------- /packages/mongodb-atlas/utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Returns the value of the environment variable with the given name. 3 | * @param name The name of the environment variable. 4 | * @returns The value of the environment variable. 5 | * @throws If the environment variable is not set. 6 | */ 7 | export function getEnvironmentValue(name: string) { 8 | const value = process.env[name] 9 | if (!value) { 10 | throw new Error(`Environment value ${name} is not set. Define it in the .env file.`) 11 | } 12 | return value 13 | } 14 | -------------------------------------------------------------------------------- /packages/mercury-case-analysis/tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "tsup"; 2 | 3 | export default defineConfig([ 4 | { 5 | entry: ["src/index.ts"], 6 | format: ["esm"], 7 | dts: true, 8 | splitting: false, 9 | sourcemap: true, 10 | clean: true, 11 | outDir: "build/esm", 12 | }, 13 | { 14 | entry: ["src/index.ts"], 15 | format: ["cjs"], 16 | dts: true, 17 | splitting: false, 18 | sourcemap: true, 19 | clean: true, 20 | outDir: "build/cjs", 21 | }, 22 | ]); 23 | -------------------------------------------------------------------------------- /packages/release-notes-generator/tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "tsup"; 2 | 3 | export default defineConfig([ 4 | { 5 | entry: ["src/index.ts"], 6 | format: ["esm"], 7 | dts: true, 8 | splitting: false, 9 | sourcemap: true, 10 | clean: true, 11 | outDir: "build/esm", 12 | }, 13 | { 14 | entry: ["src/index.ts"], 15 | format: ["cjs"], 16 | dts: true, 17 | splitting: false, 18 | sourcemap: true, 19 | clean: true, 20 | outDir: "build/cjs", 21 | }, 22 | ]); 23 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/.storybook/main.ts: -------------------------------------------------------------------------------- 1 | import type { StorybookConfig } from "@storybook/react-vite"; 2 | const config: StorybookConfig = { 3 | stories: ["../src/**/*.mdx", "../src/**/*.stories.@(js|jsx|ts|tsx)"], 4 | addons: [ 5 | "@storybook/addon-links", 6 | "@storybook/addon-essentials", 7 | "@storybook/addon-interactions", 8 | ], 9 | framework: { 10 | name: "@storybook/react-vite", 11 | options: {}, 12 | }, 13 | docs: { 14 | autodocs: "tag", 15 | }, 16 | }; 17 | export default config; 18 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/test-utils.ts: -------------------------------------------------------------------------------- 1 | import { vi } from "vitest"; 2 | 3 | export type MockFetchResultArgs = { 4 | ok?: boolean; 5 | status?: number; 6 | json: T; 7 | }; 8 | 9 | /** 10 | Mock the next global.fetch result. 11 | */ 12 | export function mockNextFetchResult({ 13 | ok = true, 14 | status = 200, 15 | json, 16 | }: MockFetchResultArgs) { 17 | (global.fetch as ReturnType).mockResolvedValueOnce({ 18 | ok, 19 | status, 20 | json: async () => json, 21 | }); 22 | } 23 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/embed/Embedder.ts: -------------------------------------------------------------------------------- 1 | export type EmbedArgs = { 2 | /** 3 | The text to embed. 4 | */ 5 | text: string; 6 | }; 7 | 8 | export type EmbedResult = { 9 | /** 10 | Vector embedding of the text. 11 | */ 12 | embedding: number[]; 13 | }; 14 | 15 | /** 16 | Takes a string of text and returns an array of numbers representing the 17 | vector embedding of the text. 18 | */ 19 | export type Embedder = { 20 | embed(args: EmbedArgs): Promise; 21 | modelName?: string; 22 | }; 23 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/frontMatter/updateFrontMatter.ts: -------------------------------------------------------------------------------- 1 | import { extractFrontMatter } from "./extractFrontMatter"; 2 | import yaml from "yaml"; 3 | 4 | export function updateFrontMatter( 5 | text: string, 6 | metadataIn: Record 7 | ): string { 8 | const { metadata: existingMetadata, body } = extractFrontMatter(text); 9 | const metadata = { 10 | ...(existingMetadata ?? {}), 11 | ...metadataIn, 12 | }; 13 | return ["---", yaml.stringify(metadata).trim(), "---", "", body].join("\n"); 14 | } 15 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/DatabaseConnection.ts: -------------------------------------------------------------------------------- 1 | export type DatabaseConnection = { 2 | /** 3 | Close the connection. 4 | 5 | @param force - Force close, emitting no events 6 | */ 7 | close(force?: boolean): Promise; 8 | /** 9 | @internal 10 | Drop the database. Typically used for testing and debugging purposes. 11 | */ 12 | drop(): Promise; 13 | 14 | /** 15 | Initialize the the database. Things like index creation should be done here. 16 | */ 17 | init?(): Promise; 18 | }; 19 | -------------------------------------------------------------------------------- /packages/scripts/testData/meta2rst/no-meta-no-description.rst: -------------------------------------------------------------------------------- 1 | .. _test-page-no-meta-no-description: 2 | 3 | ================================= 4 | Test Page: No Meta No Description 5 | ================================= 6 | 7 | .. facet:: 8 | :name: genre 9 | :values: tutorial 10 | 11 | .. contents:: On this page 12 | :local: 13 | :backlinks: none 14 | :depth: 2 15 | :class: singlecol 16 | 17 | This is a test page with no meta description. 18 | 19 | First heading 20 | ------------ 21 | 22 | This is the first section. 23 | -------------------------------------------------------------------------------- /packages/benchmarks/src/textToDriver/makeDatabases/applyCollectionIndexes.ts: -------------------------------------------------------------------------------- 1 | import { MongoClient, IndexDescription } from "mongodb-rag-core/mongodb"; 2 | export async function applyCollectionIndexes({ 3 | client, 4 | collectionName, 5 | databaseName, 6 | indexes, 7 | }: { 8 | client: MongoClient; 9 | collectionName: string; 10 | databaseName: string; 11 | indexes: IndexDescription[]; 12 | }) { 13 | const collection = client.db(databaseName).collection(collectionName); 14 | await collection.createIndexes(indexes); 15 | } 16 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/src/sources/mongoose.test.ts: -------------------------------------------------------------------------------- 1 | import { mongooseSourceConstructor } from "./mongoose"; 2 | 3 | jest.setTimeout(90000); 4 | describe("mongooseSourceConstructor", () => { 5 | it("correctly loads files", async () => { 6 | const source = await mongooseSourceConstructor(); 7 | const pages = await source.fetchPages(); 8 | expect(pages.length).toBeGreaterThan(0); 9 | expect(pages[0].title).toBeTruthy(); 10 | expect(pages[0].url).toMatch(/^mongoosejs.com\/docs\/[A-z./]+\.html$/); 11 | }); 12 | }); 13 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/test/MockPageStore.ts: -------------------------------------------------------------------------------- 1 | import { PageStore, PersistedPage } from "mongodb-rag-core"; 2 | 3 | export const makeMockPageStore = (): PageStore => { 4 | let pages: PersistedPage[] = []; 5 | return { 6 | async loadPages() { 7 | return pages; 8 | }, 9 | async updatePages(args: PersistedPage[]) { 10 | pages = [...args]; 11 | }, 12 | async deletePages() { 13 | return; 14 | }, 15 | async getDataSourceVersions() { 16 | return {}; 17 | }, 18 | }; 19 | }; 20 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/test/MockPageStore.ts: -------------------------------------------------------------------------------- 1 | import { PageStore, PersistedPage } from "../contentStore"; 2 | 3 | export const makeMockPageStore = (): PageStore => { 4 | let pages: PersistedPage[] = []; 5 | return { 6 | async loadPages() { 7 | return pages; 8 | }, 9 | async updatePages(args: PersistedPage[]) { 10 | pages = [...args]; 11 | }, 12 | async deletePages(_args) { 13 | return; 14 | }, 15 | async getDataSourceVersions(_args) { 16 | return {}; 17 | }, 18 | }; 19 | }; 20 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/processors/makeFilterNPreviousMessages.ts: -------------------------------------------------------------------------------- 1 | import { FilterPreviousMessages } from "./FilterPreviousMessages"; 2 | /** 3 | Creates a filter that only includes the previous n messages in the conversations. 4 | @param n - Number of previous messages to include. 5 | */ 6 | export const makeFilterNPreviousMessages = ( 7 | n: number 8 | ): FilterPreviousMessages => { 9 | return async (conversation) => { 10 | // Get the n latest messages. 11 | return conversation.messages.slice(-n); 12 | }; 13 | }; 14 | -------------------------------------------------------------------------------- /packages/mongodb-atlas/dump-restore/restore.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ENVIRONMENT=$1 4 | if [ -z "$ENVIRONMENT" ]; then 5 | echo "No environment provided. Please provide an environment (e.g. dev, test, staging)" 6 | exit 1 7 | fi 8 | 9 | PROJECT_DIR=$(git rev-parse --show-toplevel)/mongodb-atlas 10 | ENV_PATH=$PROJECT_DIR/.env.$ENVIRONMENT 11 | source $ENV_PATH 12 | 13 | DB_DUMP_PATH=$PROJECT_DIR/dump-restore/mongodump/$DUMP_DB 14 | 15 | mongorestore -d $DB_NAME $DB_DUMP_PATH --uri $MONGODB_CONNECTION_URI --drop 16 | echo "Restored $DUMP_DB to $DB_NAME" 17 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/src/withConfig.test.ts: -------------------------------------------------------------------------------- 1 | import Path from "path"; 2 | import { withConfig } from "./withConfig"; 3 | 4 | describe("withConfig", () => { 5 | it("loads a config file", async () => { 6 | await withConfig( 7 | async (config) => { 8 | expect(config.dataSources).toBeDefined(); 9 | }, 10 | { 11 | config: Path.resolve( 12 | __dirname, 13 | "..", 14 | "testData", 15 | "config", 16 | "configTest.cjs" 17 | ), 18 | } 19 | ); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/ChatbotProvider.tsx: -------------------------------------------------------------------------------- 1 | import { createContext } from "react"; 2 | import { ChatbotData } from "./useChatbot"; 3 | 4 | export const ChatbotContext = createContext(null); 5 | 6 | export type ChatbotProviderProps = ChatbotData & { 7 | children: React.ReactNode; 8 | }; 9 | 10 | export function ChatbotProvider({ 11 | children, 12 | ...chatbotData 13 | }: ChatbotProviderProps) { 14 | return ( 15 | 16 | {children} 17 | 18 | ); 19 | } 20 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/LinkDataProvider.tsx: -------------------------------------------------------------------------------- 1 | import { createContext } from "react"; 2 | 3 | export type LinkData = { 4 | tck?: string; 5 | }; 6 | 7 | export const LinkDataContext = createContext({}); 8 | 9 | export type LinkDataProviderProps = LinkData & { 10 | children: React.ReactNode; 11 | }; 12 | 13 | export function LinkDataProvider({ 14 | children, 15 | ...linkData 16 | }: LinkDataProviderProps) { 17 | return ( 18 | 19 | {children} 20 | 21 | ); 22 | } 23 | -------------------------------------------------------------------------------- /packages/mongodb-atlas/.env.example: -------------------------------------------------------------------------------- 1 | GROUP_ID= 2 | CLUSTER_NAME= 3 | DB_NAME=docs-chatbot-dev 4 | ATLAS_VECTOR_SEARCH_INDEX_NAME=vector_index 5 | CONVERSATIONS_TTL_SECONDS=31536000 # 1 year 6 | CONVERSATIONS_COLL_NAME=conversations 7 | CONTENT_METADATA_COLL_NAME=pages 8 | EMBEDDED_CONTENT_COLL_NAME=embedded_content 9 | ATLAS_ADMIN_API_KEY= 10 | ATLAS_ADMIN_API_SECRET= 11 | MONGODB_CONNECTION_URI="" # note: having problems if not wrapped in quotes...not sure why 12 | DUMP_DB="docs-chatbot-dev-reset" 13 | -------------------------------------------------------------------------------- /ingest-service.dockerfile: -------------------------------------------------------------------------------- 1 | # Build stage 2 | FROM node:20 3 | 4 | WORKDIR /bin 5 | COPY . ./ 6 | RUN npm install lerna 7 | RUN npm run bootstrap -- --scope='{mongodb-rag-core,mongodb-rag-ingest,ingest-mongodb-public}' 8 | RUN npm run build -- --scope='{mongodb-rag-core,mongodb-rag-ingest,ingest-mongodb-public}' 9 | 10 | # Install Playwright with dependencies 11 | RUN npx playwright install chromium --with-deps 12 | 13 | # Add git for GitDataSource 14 | RUN apt-get update && apt-get install -y git 15 | 16 | ENV NODE_ENV=production 17 | 18 | WORKDIR /bin/packages/ingest-mongodb-public 19 | -------------------------------------------------------------------------------- /packages/datasets/src/EnvVars.ts: -------------------------------------------------------------------------------- 1 | export const OPENAI_ENV_VARS = { 2 | OPENAI_API_KEY: "", 3 | OPENAI_ENDPOINT: "", 4 | OPENAI_CHAT_COMPLETION_DEPLOYMENT: "", 5 | OPENAI_API_VERSION: "", 6 | }; 7 | 8 | export const HUGGINGFACE = { 9 | HUGGINGFACE_ACCESS_TOKEN: "", 10 | }; 11 | 12 | export const HUGGINGFACE_DOCS_CONTENT = { 13 | HUGGINGFACE_DOCS_CONTENT_REPO: "", 14 | }; 15 | export const HUGGINGFACE_DOCS_CODE_EXAMPLES = { 16 | HUGGINGFACE_DOCS_CODE_EXAMPLES_REPO: "", 17 | }; 18 | export const DATABASE_NL_QUERIES = { 19 | MONGODB_TEXT_TO_CODE_CONNECTION_URI: "", 20 | }; 21 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/contentStore/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./EmbeddedContent"; 2 | export * from "./getChangedPages"; 3 | export * from "./MongoDbEmbeddedContentStore"; 4 | export * from "./MongoDbPageStore"; 5 | export * from "./MongoDbSearchResultsStore"; 6 | export * from "./MongoDbTransformedContentStore"; 7 | export * from "./Page"; 8 | export * from "./PageFormat"; 9 | export * from "./pageIdentity"; 10 | export * from "./TransformedContent"; 11 | export * from "./updatePages"; 12 | export * from "./updateEmbeddedContent"; 13 | export * from "./updateTransformedContent"; 14 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | node: true, 4 | }, 5 | root: true, 6 | parser: "@typescript-eslint/parser", 7 | plugins: ["@typescript-eslint", "jsdoc"], 8 | extends: [ 9 | "eslint:recommended", 10 | "prettier", 11 | "plugin:@typescript-eslint/recommended", 12 | ], 13 | ignorePatterns: ["**/build/*"], 14 | rules: { 15 | "jsdoc/require-asterisk-prefix": ["error", "never"], 16 | "prettier/prettier": "off", 17 | "arrow-body-style": "off", 18 | "prefer-arrow-callback": "off", 19 | }, 20 | }; 21 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/processors/MakeReferenceLinksFunc.ts: -------------------------------------------------------------------------------- 1 | import { EmbeddedContent, PersistedPage, References } from "mongodb-rag-core"; 2 | 3 | /** 4 | Content that can be made into a Reference. 5 | */ 6 | export type ReferenceableContent = 7 | | (Partial & Pick) 8 | | (Partial & Pick); 9 | 10 | /** 11 | Function that generates the references in the response to user. 12 | */ 13 | export type MakeReferenceLinksFunc = ( 14 | searchResults: ReferenceableContent[] 15 | ) => References; 16 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/findContent/FindContentFunc.ts: -------------------------------------------------------------------------------- 1 | import { EmbeddedContent, QueryFilters } from "../contentStore/EmbeddedContent"; 2 | import { WithScore } from "../VectorStore"; 3 | 4 | export type FindContentFuncArgs = { 5 | query: string; 6 | filters?: QueryFilters; 7 | limit?: number; 8 | }; 9 | 10 | export type FindContentFunc = ( 11 | args: FindContentFuncArgs 12 | ) => Promise; 13 | 14 | export type FindContentResult = { 15 | queryEmbedding: number[]; 16 | content: WithScore[]; 17 | embeddingModelName?: string; 18 | }; 19 | -------------------------------------------------------------------------------- /THIRD_PARTY_NOTICES: -------------------------------------------------------------------------------- 1 | mongodb/chatbot uses third-party libraries or other resources that may 2 | be distributed under licenses different than the mongodb/chatbot software. 3 | 4 | In the event that we accidentally failed to list a required notice, 5 | please bring it to our attention through any of the ways detailed here: 6 | 7 | chatbot@mongodb.com 8 | mongodb-dev@googlegroups.com 9 | 10 | The attached notices are provided for information only. 11 | 12 | For any licenses that require disclosure of source, sources are available at 13 | https://github.com/mongodb/chatbot. 14 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/.env.example: -------------------------------------------------------------------------------- 1 | MONGODB_CONNECTION_URI= 2 | MONGODB_DATABASE_NAME= 3 | OPENAI_API_KEY= 4 | OPENAI_CHAT_COMPLETION_DEPLOYMENT= 5 | OPENAI_RETRIEVAL_EMBEDDING_DEPLOYMENT="docs-chatbot-embedding-ada-002" 6 | OPENAI_ENDPOINT=https://.openai.azure.com/ 7 | GITHUB_ACCESS_TOKEN= # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens 8 | JIRA_USERNAME= 9 | JIRA_PASSWORD= 10 | -------------------------------------------------------------------------------- /packages/benchmarks/src/makeExperimentName.ts: -------------------------------------------------------------------------------- 1 | /** 2 | Generate Braintrust experiment name 3 | */ 4 | export function makeExperimentName({ 5 | baseName, 6 | experimentType, 7 | model, 8 | ...rest 9 | }: { 10 | baseName: string; 11 | experimentType: string; 12 | model: string; 13 | [k: string]: string; 14 | }): string { 15 | return `${baseName}?experimentType=${experimentType}&model=${model}${ 16 | rest && Object.entries(rest).length > 0 17 | ? `&${Object.entries(rest) 18 | .map(([k, v]) => `${k}=${v}`) 19 | .join("&")}` 20 | : "" 21 | }`; 22 | } 23 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/src/sources/snootySources.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | snootyProjectConfig, 3 | makeSnootyDataSources, 4 | snootyDataApiBaseUrl, 5 | } from "./snootySources"; 6 | 7 | jest.setTimeout(20000); 8 | describe("Snooty data sources", () => { 9 | jest.retryTimes(3, { waitBeforeRetry: 5000, logErrorsBeforeRetry: true }); 10 | test.each(snootyProjectConfig)("$name should fetch data", async (project) => { 11 | const [source] = await makeSnootyDataSources(snootyDataApiBaseUrl, [ 12 | project, 13 | ]); 14 | expect(source.name).toBeDefined(); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/src/sources/practicalAggregations.test.ts: -------------------------------------------------------------------------------- 1 | import { practicalAggregationsDataSource } from "./practicalAggregations"; 2 | 3 | jest.setTimeout(90000); 4 | describe("practicalAggregationsDataSource", () => { 5 | it("correctly loads files", async () => { 6 | const source = await practicalAggregationsDataSource(); 7 | const pages = await source.fetchPages(); 8 | expect(pages.length).toBeGreaterThan(0); 9 | expect(pages[0].title).toBeTruthy(); 10 | expect(pages[0].url).toMatch( 11 | /^practical-mongodb-aggregations.com\/[A-z.]+$/ 12 | ); 13 | }); 14 | }); 15 | -------------------------------------------------------------------------------- /packages/scripts/testData/meta2rst/yes-meta-no-description.rst: -------------------------------------------------------------------------------- 1 | .. _test-page-yes-meta-no-description: 2 | 3 | ================================== 4 | Test Page: Yes Meta No Description 5 | ================================== 6 | 7 | .. facet:: 8 | :name: genre 9 | :values: tutorial 10 | 11 | .. meta:: 12 | :keywords: code example 13 | 14 | .. contents:: On this page 15 | :local: 16 | :backlinks: none 17 | :depth: 2 18 | :class: singlecol 19 | 20 | This is a test page with a meta directive but no meta description. 21 | 22 | First heading 23 | ------------ 24 | 25 | This is the first section. 26 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/README.md: -------------------------------------------------------------------------------- 1 | # Ingest MongoDB Public 2 | 3 | This is the standard ingest config for public MongoDB documentation. 4 | 5 | It also serves as an example of implementing a configuration/plugin for the `mongodb-rag-ingest` CLI. 6 | 7 | ## Building the Config 8 | 9 | Be sure to build `mongodb-rag-core` and `mongodb-rag-ingest` first. 10 | 11 | ```sh 12 | npm i 13 | npm run build 14 | ``` 15 | 16 | ## Using the Config 17 | 18 | ```sh 19 | npm run ingest:all 20 | ``` 21 | 22 | This runs the ingest script with the `--config` flag passing 23 | the built config file (`build/config.js`) to the tool. 24 | 25 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/models/getOpenAiEndpointAndApiKey.ts: -------------------------------------------------------------------------------- 1 | import { ModelConfig } from "./models"; 2 | import { BRAINTRUST_ENV_VARS } from "../CoreEnvVars"; 3 | import { assertEnvVars } from "../assertEnvVars"; 4 | 5 | export async function getOpenAiEndpointAndApiKey(model: ModelConfig) { 6 | if (model.provider === "braintrust") { 7 | const { BRAINTRUST_API_KEY, BRAINTRUST_ENDPOINT } = 8 | assertEnvVars(BRAINTRUST_ENV_VARS); 9 | return { 10 | baseURL: BRAINTRUST_ENDPOINT, 11 | apiKey: BRAINTRUST_API_KEY, 12 | }; 13 | } 14 | throw new Error(`Unknown provider: ${model.provider}`); 15 | } 16 | -------------------------------------------------------------------------------- /packages/datasets/.env.example: -------------------------------------------------------------------------------- 1 | OPENAI_API_KEY= 2 | OPENAI_CHAT_COMPLETION_DEPLOYMENT="gpt-4o-mini" 3 | OPENAI_CHAT_COMPLETION_MODEL_VERSION="placeholder...must include but value doesn't matter" 4 | OPENAI_ENDPOINT= 5 | OPENAI_API_VERSION="2024-06-01" 6 | BRAINTRUST_API_KEY="" 7 | HUGGINGFACE_ACCESS_TOKEN="hf_..." 8 | HUGGINGFACE_DOCS_CONTENT_REPO="someuser/some-repo" 9 | HUGGINGFACE_DOCS_CODE_EXAMPLE_REPO="someuser/some-repo" 10 | MONGODB_CONNECTION_URI="..." 11 | MONGODB_DATABASE_NAME="docs-chatbot-dev" 12 | MONGODB_TEXT_TO_CODE_CONNECTION_URI="..." 13 | MONGODB_ATLAS_SEARCH_CONNECTION_URI="..." 14 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/Banner.tsx: -------------------------------------------------------------------------------- 1 | import Banner from "@leafygreen-ui/banner"; 2 | import { useDarkMode } from "@leafygreen-ui/leafygreen-provider"; 3 | 4 | export type ErrorBannerProps = { 5 | message?: string; 6 | darkMode?: boolean; 7 | }; 8 | 9 | export function ErrorBanner(props: ErrorBannerProps) { 10 | const { message = "Something went wrong" } = props; 11 | const { darkMode } = useDarkMode(); 12 | return ( 13 | 14 | {message} 15 |
16 | Reload the page to start a new conversation. 17 | 18 | ); 19 | } 20 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/arrayFilters.ts: -------------------------------------------------------------------------------- 1 | /** 2 | Given an array of PromiseSettledResults, return those that are fulfilled as PromiseFulfilledResults. 3 | */ 4 | export const filterFulfilled = ( 5 | array: PromiseSettledResult[] 6 | ): PromiseFulfilledResult[] => 7 | array.filter( 8 | ({ status }) => status === "fulfilled" 9 | ) as PromiseFulfilledResult[]; 10 | 11 | /** 12 | Given an array of possibly undefined T, return those that are defined. 13 | */ 14 | export const filterDefined = (array: T[]): Exclude[] => 15 | array.filter((value) => value !== undefined) as Exclude[]; 16 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/context/tdbx-content-types/contribution-guide.yaml: -------------------------------------------------------------------------------- 1 | type: contribution-guide 2 | name: How to Contribute 3 | objectives: 4 | - Provide instructions on how to contribute to the driver 5 | pageStructure: | 6 | ================= 7 | How to Contribute 8 | ================= 9 | 10 | Summarize the actions required for authoring and submitting a contribution. 11 | 12 | Link to the GitHub repository for the driver or connector. 13 | 14 | Link to any other resources that can help the user author or submit a contribution. 15 | examples: 16 | - https://mongodb.com/docs/kafka-connector/current/contribute/ 17 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-verified-answers/.env.example: -------------------------------------------------------------------------------- 1 | MONGODB_DATABASE_NAME= 2 | MONGODB_CONNECTION_URI= 3 | OPENAI_EMBEDDING_MODEL_NAME=text-embedding-ada-002 4 | OPENAI_EMBEDDING_MODEL_VERSION=2023-03-15-preview 5 | OPENAI_VERIFIED_ANSWER_EMBEDDING_DEPLOYMENT="docs-chatbot-embedding-ada-002" 6 | OPENAI_ENDPOINT=https://.openai.azure.com/ 7 | OPENAI_API_KEY= 8 | OPENAI_API_VERSION="2024-06-01" 9 | OPENAI_RETRIEVAL_EMBEDDING_DEPLOYMENT=text-embedding-3-small 10 | VECTOR_SEARCH_INDEX_NAME=vector_index 11 | OPENAI_CHAT_COMPLETION_DEPLOYMENT="n/a" 12 | OPENAI_CHAT_COMPLETION_MODEL_VERSION="n/a" -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/processors/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./FilterPreviousMessages"; 2 | export * from "./MakeReferenceLinksFunc"; 3 | export * from "./QueryPreprocessorFunc"; 4 | export * from "./filterOnlySystemPrompt"; 5 | export * from "./makeDefaultReferenceLinks"; 6 | export * from "./makeFilterNPreviousMessages"; 7 | export * from "./includeChunksForMaxTokensPossible"; 8 | export * from "./InputGuardrail"; 9 | export * from "./makeVerifiedAnswerGenerateResponse"; 10 | export * from "./includeChunksForMaxTokensPossible"; 11 | export * from "./GenerateResponse"; 12 | export * from "./UpdateTraceFunc"; 13 | export * from "./addCustomData"; 14 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/runId.ts: -------------------------------------------------------------------------------- 1 | export function createRunId(namespace?: string) { 2 | const now = new Date(); 3 | const [date, time] = now.toISOString().split("T"); 4 | // Format a string for the current day in YYYYMMDD format 5 | const datestamp = date.replace(/-/g, ""); 6 | // Format a string for the current time in 24H HHMM format 7 | const minutestamp = time.replace(/:/g, "").slice(0, 4); 8 | const millistamp = now 9 | .getTime() 10 | .toString(36) 11 | .slice(2, 2 + 5); 12 | const timestamp = `${datestamp}-${minutestamp}-${millistamp}`; 13 | return namespace ? `${namespace}-${timestamp}` : timestamp; 14 | } 15 | -------------------------------------------------------------------------------- /packages/datasets/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "mongodb-datasets" 3 | version = "0.1.0" 4 | description = "MongoDB dataset utilities for importing and processing datasets" 5 | requires-python = ">=3.11" 6 | dependencies = [ 7 | "datasets>=2.18.0", 8 | "pymongo>=4.0.0", 9 | "python-dotenv>=1.0.0", 10 | "jsonc-parser>=1.1.5", 11 | ] 12 | 13 | [project.scripts] 14 | import-wikipedia = "mongodb_datasets.import_wikipedia:main" 15 | 16 | [build-system] 17 | requires = ["hatchling"] 18 | build-backend = "hatchling.build" 19 | 20 | [tool.uv] 21 | dev-dependencies = [ 22 | "pytest>=7.0.0", 23 | "black>=22.0.0", 24 | "ruff>=0.1.0", 25 | ] 26 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/dataSources/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./AcquitRequireMdOnGithubDataSource"; 2 | export * from "./DataSource"; 3 | export * from "./GitDataSource"; 4 | export * from "./GitHubDataSource"; 5 | export * from "./MdOnGithubDataSource"; 6 | export * from "./ProjectBase"; 7 | export * from "./handleHtmlDocument"; 8 | export * from "./removeMarkdownImagesAndLinks"; 9 | export * from "./removeMarkdownFileExtension"; 10 | export * from "./MarkdownUrlDataSource"; 11 | export * from "./LangchainDocumentLoaderDataSource"; 12 | export * from "./CodeOnGithubTextDataSource"; 13 | export * from "./normalizeUrl"; 14 | export * from "./makeMarkdownLists"; 15 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/ChatbotView.tsx: -------------------------------------------------------------------------------- 1 | import { References } from "./references"; 2 | import { ChatMessageFeedProps } from "./ChatMessageFeed"; 3 | import { DarkModeProps } from "./DarkMode"; 4 | 5 | export type ChatbotViewProps = DarkModeProps & 6 | Omit & { 7 | fatalErrorMessage?: string; 8 | initialMessageReferences?: References; 9 | initialMessageSuggestedPrompts?: string[]; 10 | initialMessageText?: string; 11 | inputBarId?: string; 12 | inputBarPlaceholder?: string; 13 | inputBottomText?: string; 14 | messageBottomContent?: React.ReactNode; 15 | windowTitle?: string; 16 | }; 17 | -------------------------------------------------------------------------------- /packages/datasets/src/treeGeneration/GenerationNode.ts: -------------------------------------------------------------------------------- 1 | import { ObjectId } from "mongodb-rag-core/mongodb"; 2 | 3 | export interface GenerationNode< 4 | T, 5 | TypeName extends string | undefined = undefined 6 | > { 7 | _id: ObjectId; 8 | parent: GenerationNode | null; 9 | children?: GenerationNode[]; 10 | data: T; 11 | updated: Date; 12 | type?: TypeName; 13 | } 14 | 15 | export type WithParentNode< 16 | NodeType extends GenerationNode, 17 | ParentType extends GenerationNode | null 18 | > = NodeType & { 19 | parent: ParentType; 20 | }; 21 | -------------------------------------------------------------------------------- /packages/performance-tests/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "performance-tests", 3 | "version": "1.0.0", 4 | "description": "Performance tests for the Docs AI Chatbot.", 5 | "main": "index.js", 6 | "private": true, 7 | "scripts": { 8 | "clean": "rm -rf build", 9 | "build": "npm run clean && tsc -b tsconfig.build.json", 10 | "test:serverLoad:dev": "npm run build && BASE_URL=http://localhost:3000 k6 run build/serverLoad.js", 11 | "test:serverLoad:staging": "echo 'see README'" 12 | }, 13 | "keywords": [], 14 | "author": "", 15 | "license": "ISC", 16 | "devDependencies": { 17 | "@types/k6": "^0.46.2", 18 | "typescript": "^5.2.2" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /packages/scripts/testData/meta2rst/yes-meta-yes-description.rst: -------------------------------------------------------------------------------- 1 | .. _test-page-yes-meta-yes-description: 2 | 3 | =================================== 4 | Test Page: Yes Meta Yes Description 5 | =================================== 6 | 7 | .. meta:: 8 | :description: Learn how to improve the performance of a MongoDB application. 9 | :keywords: code example 10 | 11 | .. facet:: 12 | :name: genre 13 | :values: tutorial 14 | 15 | .. contents:: On this page 16 | :local: 17 | :backlinks: none 18 | :depth: 2 19 | :class: singlecol 20 | 21 | This is a test page with a meta description. 22 | 23 | First heading 24 | ------------ 25 | 26 | This is the first section. 27 | -------------------------------------------------------------------------------- /packages/benchmarks/src/textToDriver/generateDriverCode/languagePrompts/PromptStrategies.ts: -------------------------------------------------------------------------------- 1 | /** 2 | - `chainOfThought`: Use chain of thought reasoning to generate the query. 3 | - `default`: Prompt-engineering optimized prompt. 4 | - `lazy`: Minimal viable prompt. Useful for baselining. 5 | */ 6 | export type SystemPromptStrategy = "chainOfThought" | "default" | "lazy"; 7 | 8 | /** 9 | - `annotated`: Use annotated schema. Generated by human/LLM. 10 | - `interpreted`: Use interpreted schema. Interpreted by code from database, no 'intelligence' involved. 11 | - `none`: Do not include schema. 12 | */ 13 | export type SchemaStrategy = "annotated" | "interpreted" | "none"; 14 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/withConfig.test.ts: -------------------------------------------------------------------------------- 1 | import Path from "path"; 2 | import { withConfig } from "./withConfig"; 3 | 4 | describe("withConfig", () => { 5 | it("loads a config file", async () => { 6 | await withConfig( 7 | async (config) => { 8 | expect(config.pageStore).toBeDefined(); 9 | expect(config.embedder).toBeDefined(); 10 | expect(config.embeddedContentStore).toBeDefined(); 11 | }, 12 | { 13 | config: Path.resolve( 14 | __dirname, 15 | "..", 16 | "testData", 17 | "config", 18 | "configTest.cjs" 19 | ), 20 | } 21 | ); 22 | }); 23 | }); 24 | -------------------------------------------------------------------------------- /packages/release-notes-generator/src/utils.test.ts: -------------------------------------------------------------------------------- 1 | import { currentTimestamp, safeFileName } from "./utils"; 2 | 3 | describe("currentTimestamp", () => { 4 | it("returns a timestamp in the format YYYY-MM-DD-HH-MM-SS", () => { 5 | expect(currentTimestamp()).toMatch( 6 | /^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z$/, 7 | ); 8 | }); 9 | }); 10 | 11 | describe("safeFileName", () => { 12 | it("replaces invalid characters with a dash", () => { 13 | expect(safeFileName("file?name")).toBe("file-name"); 14 | }); 15 | it("replaces multiple invalid characters with a dash", () => { 16 | expect(safeFileName("file?name*")).toBe("file-name-"); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/src/sources/snooty/truncateEmbeddings.ts: -------------------------------------------------------------------------------- 1 | export function truncateEmbeddings(body: string): string { 2 | // Matches elements of an embedding vector string with at least two elements. 3 | // Each element must be between -1 and 1, include the leading 0 or -0, and 4 | // include at least one digit after the decimal point. 5 | // e.g. "0.123, 0.456, -0.789, ..." 6 | const embeddingRegex = 7 | /([-]?0\.\d+d?),\s*([-]?0\.\d+d?)(?:\s*,\s*[-]?0\.\d+d?)*/g; 8 | return body.replace(embeddingRegex, (...matches) => { 9 | const [_wholeEmbedding, firstFloat, secondFloat] = matches; 10 | return `${firstFloat}, ${secondFloat}, ...`; 11 | }); 12 | } 13 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/.release-it.json: -------------------------------------------------------------------------------- 1 | { 2 | "git": { 3 | "commitArgs": ["-a"], 4 | "commitMessage": "Release ${npm.name} v${version}", 5 | "changelog": "git log --pretty=format:'* %s (%h)' ${latestTag}..HEAD -- .", 6 | "tagMatch": "${npm.name}-*", 7 | "getLatestTagFromAllRefs": true, 8 | "tag": true, 9 | "push": true, 10 | "pushRepo": "upstream", 11 | "tagName": "${npm.name}-v${version}" 12 | }, 13 | "npm": { 14 | "publish": false 15 | }, 16 | "github": { 17 | "releaseName": "${npm.name}-v${version}", 18 | "draft": true, 19 | "release": true 20 | }, 21 | "hooks": { 22 | "before:init": ["npm run build"] 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /chat-server.dockerfile: -------------------------------------------------------------------------------- 1 | # Build stage 2 | FROM node:20 3 | 4 | ARG ENVIRONMENT 5 | ENV ENVIRONMENT=${ENVIRONMENT} 6 | ARG GIT_COMMIT 7 | ENV VITE_GIT_COMMIT=${GIT_COMMIT} 8 | 9 | # Install all dependencies && set up project 10 | WORKDIR /app 11 | COPY . ./ 12 | RUN npm install lerna 13 | RUN npm run bootstrap -- --scope='{mongodb-rag-core,mongodb-chatbot-server,chatbot-server-mongodb-public,mongodb-chatbot-ui,mongodb-chatbot-verified-answers}' 14 | RUN npm run build -- --scope='{mongodb-rag-core,mongodb-chatbot-server,chatbot-server-mongodb-public,mongodb-chatbot-ui,mongodb-chatbot-verified-answers}' 15 | 16 | ENV NODE_ENV=production 17 | 18 | EXPOSE 3000 19 | CMD ["npm", "run", "server:start"] 20 | -------------------------------------------------------------------------------- /packages/mercury-case-analysis/src/calculateEmbeddings.ts: -------------------------------------------------------------------------------- 1 | import assert from "node:assert/strict"; 2 | import { Embedder } from "mongodb-rag-core"; 3 | 4 | export const calculateEmbeddings = async ({ 5 | text, 6 | embedders, 7 | }: { 8 | text: string; 9 | embedders: Embedder[]; 10 | }) => { 11 | return Object.fromEntries( 12 | await Promise.all( 13 | embedders.map(async (embedder): Promise<[string, number[]]> => { 14 | const { embedding } = await embedder.embed({ text }); 15 | const model = embedder.modelName; 16 | assert(model !== undefined, "Missing embedder model name!"); 17 | return [model, embedding]; 18 | }) 19 | ) 20 | ); 21 | }; 22 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/Banner.test.tsx: -------------------------------------------------------------------------------- 1 | import { render, screen } from "@testing-library/react"; 2 | import { ErrorBanner } from "./Banner"; 3 | 4 | describe("Banner", () => { 5 | it("should render a banner with a default error message", async () => { 6 | render(); 7 | const el = screen.queryByText(/Something went wrong/); 8 | expect(el).toBeInTheDocument(); 9 | }); 10 | 11 | it("should render the banner with a custom error message", async () => { 12 | render(); 13 | const el = screen.queryByText(/whoopsie that isn't quite right/); 14 | expect(el).toBeInTheDocument(); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /packages/benchmarks/.release-it.json: -------------------------------------------------------------------------------- 1 | { 2 | "git": { 3 | "commitArgs": ["-a"], 4 | "commitMessage": "Release ${npm.name} v${version}", 5 | "changelog": "git log --pretty=format:'* %s (%h)' ${latestTag}..HEAD -- .", 6 | "tagMatch": "${npm.name}-*", 7 | "getLatestTagFromAllRefs": true, 8 | "tag": true, 9 | "push": true, 10 | "pushRepo": "upstream", 11 | "tagName": "${npm.name}-v${version}" 12 | }, 13 | "npm": { 14 | "publish": false 15 | }, 16 | "github": { 17 | "releaseName": "${npm.name}-v${version}", 18 | "draft": true, 19 | "release": true 20 | }, 21 | "hooks": { 22 | "before:init": ["pushd .. && npm run build && popd"] 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/.release-it.json: -------------------------------------------------------------------------------- 1 | { 2 | "git": { 3 | "commitArgs": ["-a"], 4 | "commitMessage": "Release ${npm.name} v${version}", 5 | "changelog": "git log --pretty=format:'* %s (%h)' ${latestTag}..HEAD -- .", 6 | "tagMatch": "${npm.name}-*", 7 | "getLatestTagFromAllRefs": true, 8 | "tag": true, 9 | "push": true, 10 | "pushRepo": "upstream", 11 | "tagName": "${npm.name}-v${version}" 12 | }, 13 | "npm": { 14 | "publish": false 15 | }, 16 | "github": { 17 | "releaseName": "${npm.name}-v${version}", 18 | "draft": true, 19 | "release": true 20 | }, 21 | "hooks": { 22 | "before:init": ["npm run build", "npm run lint"] 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-verified-answers/README.md: -------------------------------------------------------------------------------- 1 | # MongoDB Verified Answer CLI 2 | 3 | The Verified Answer CLI imports answers specified in a yaml file into the database. 4 | 5 | ## To import answers into the database: 6 | 7 | Write answers in a yaml file like `verified-answers.yaml` or make updates to an existing file. 8 | 9 | In the root directory, run `npm run build`. 10 | 11 | Enter the mongodb-chatbot-verified-answers directory with `cd packages/mongodb-chatbot-verified-answers`. 12 | 13 | Run `npm run upload -- import `. 14 | 15 | If successful, you will see a success message that looks like: 16 | 17 | ```sh 18 | { failed: 0, updated: 0, created: 18, deleted: 0 } 19 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/test/cosineSimilarity.ts: -------------------------------------------------------------------------------- 1 | export function cosineSimilarity(a: number[], b: number[]) { 2 | // https://towardsdatascience.com/how-to-build-a-textual-similarity-analysis-web-app-aa3139d4fb71 3 | 4 | const magnitudeA = Math.sqrt(dotProduct(a, a)); 5 | const magnitudeB = Math.sqrt(dotProduct(b, b)); 6 | if (magnitudeA && magnitudeB) { 7 | // https://towardsdatascience.com/how-to-measure-distances-in-machine-learning-13a396aa34ce 8 | return dotProduct(a, b) / (magnitudeA * magnitudeB); 9 | } else { 10 | return 0; 11 | } 12 | } 13 | 14 | function dotProduct(a: number[], b: number[]) { 15 | return a.reduce((acc, cur, i) => acc + cur * b[i], 0); 16 | } 17 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./build", 5 | "module": "ESNext", 6 | 7 | /* Bundler mode */ 8 | "moduleResolution": "bundler", 9 | "allowImportingTsExtensions": true, 10 | "resolveJsonModule": true, 11 | "isolatedModules": true, 12 | "noEmit": true, 13 | "jsx": "react-jsx", 14 | 15 | /* Linting */ 16 | "noUnusedLocals": true, 17 | "noUnusedParameters": true, 18 | "noFallthroughCasesInSwitch": true, 19 | "noImplicitAny": true, 20 | "noImplicitReturns": true, 21 | "noImplicitThis": true 22 | }, 23 | "include": ["./src/**/*.ts", "./src/**/*.tsx"] 24 | } 25 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/vite.config.static-site.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vite"; 2 | import react from "@vitejs/plugin-react"; 3 | import { nodePolyfills } from "vite-plugin-node-polyfills"; 4 | 5 | // https://vitejs.dev/config/ 6 | export default defineConfig({ 7 | plugins: [ 8 | nodePolyfills({ 9 | exclude: [ 10 | "fs", // Excludes the polyfill for `fs` and `node:fs`. 11 | ], 12 | globals: { 13 | Buffer: true, 14 | global: true, 15 | process: true, 16 | }, 17 | protocolImports: true, 18 | }), 19 | react(), 20 | ], 21 | root: ".", 22 | build: { 23 | outDir: "../chatbot-server-mongodb-public/static", 24 | }, 25 | }); 26 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/processors/includeChunksForMaxTokensPossible.ts: -------------------------------------------------------------------------------- 1 | import { EmbeddedContent } from "mongodb-rag-core"; 2 | 3 | /** 4 | This function returns the chunks that can fit in the maxTokens. 5 | It limits the number of tokens that are sent to the LLM. 6 | */ 7 | export function includeChunksForMaxTokensPossible({ 8 | maxTokens, 9 | content, 10 | }: { 11 | maxTokens: number; 12 | content: EmbeddedContent[]; 13 | }): EmbeddedContent[] { 14 | let total = 0; 15 | const fitRangeEndIndex = content.findIndex( 16 | ({ tokenCount }) => (total += tokenCount) > maxTokens 17 | ); 18 | return fitRangeEndIndex === -1 ? content : content.slice(0, fitRangeEndIndex); 19 | } 20 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/middleware/slowDown.ts: -------------------------------------------------------------------------------- 1 | import slowDown, { Options } from "express-slow-down"; 2 | import { ipAddressKeyGenerator } from "./ipAddressKeyGenerator"; 3 | 4 | export type SlowDownOptions = Partial; 5 | 6 | export const defaultSlowDownOptions = { 7 | windowMs: 60 * 1000, // 1 minute 8 | delayAfter: 20, // allow 20 requests per windowMs before starting to delay 9 | delayMs: 500, // delay 500ms for each request after the 20th request 10 | keyGenerator: ipAddressKeyGenerator, 11 | } satisfies SlowDownOptions; 12 | 13 | export function makeSlowDown(options?: SlowDownOptions) { 14 | return slowDown({ 15 | ...defaultSlowDownOptions, 16 | ...options, 17 | }); 18 | } 19 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/middleware/useSegmentIds.ts: -------------------------------------------------------------------------------- 1 | import { ConversationsMiddleware } from "mongodb-chatbot-server"; 2 | 3 | export const SEGMENT_USER_ID_HEADER = "X-Segment-User-Id"; 4 | export const SEGMENT_ANONYMOUS_ID_HEADER = "X-Segment-Anonymous-Id"; 5 | 6 | export function useSegmentIds(): ConversationsMiddleware { 7 | return (req, res, next) => { 8 | const userId = req.header(SEGMENT_USER_ID_HEADER); 9 | const anonymousId = req.header(SEGMENT_ANONYMOUS_ID_HEADER); 10 | if (userId) { 11 | res.locals.customData.segmentUserId = userId; 12 | } 13 | if (anonymousId) { 14 | res.locals.customData.segmentAnonymousId = anonymousId; 15 | } 16 | next(); 17 | }; 18 | } 19 | -------------------------------------------------------------------------------- /packages/benchmarks/src/textToDriver/makeExperimentName.ts: -------------------------------------------------------------------------------- 1 | /** 2 | Generate Braintrust experiment name 3 | */ 4 | export function makeExperimentName({ 5 | baseName, 6 | experimentType, 7 | model, 8 | systemPromptStrategy, 9 | schemaStrategy, 10 | fewShot, 11 | }: { 12 | baseName: string; 13 | experimentType: string; 14 | model: string; 15 | systemPromptStrategy?: string; 16 | schemaStrategy?: string; 17 | fewShot?: boolean; 18 | }): string { 19 | return `${baseName}?experimentType=${experimentType}&model=${model}${ 20 | systemPromptStrategy ? `&prompt=${systemPromptStrategy}` : "" 21 | }${schemaStrategy ? `&schema=${schemaStrategy}` : ""}${ 22 | fewShot ? "&fewShot=true" : "" 23 | }`; 24 | } 25 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/processors/InputGuardrail.ts: -------------------------------------------------------------------------------- 1 | import { GenerateResponseParams } from "./GenerateResponse"; 2 | 3 | export interface InputGuardrailResult< 4 | Metadata extends Record | undefined = Record 5 | > { 6 | rejected: boolean; 7 | reason?: string; 8 | metadata: Metadata; 9 | } 10 | 11 | export const guardrailFailedResult: InputGuardrailResult = { 12 | rejected: true, 13 | reason: "Guardrail failed", 14 | metadata: {}, 15 | }; 16 | 17 | export type InputGuardrail< 18 | Metadata extends Record | undefined = Record 19 | > = ( 20 | generateResponseParams: GenerateResponseParams 21 | ) => Promise>; 22 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/.env.example: -------------------------------------------------------------------------------- 1 | PORT=3000 2 | OPENAI_ENDPOINT=https://.openai.azure.com/ 3 | OPENAI_API_KEY= 4 | OPENAI_API_VERSION="2024-06-01" 5 | OPENAI_VERIFIED_ANSWER_EMBEDDING_DEPLOYMENT="docs-chatbot-embedding-ada-002" 6 | OPENAI_RETRIEVAL_EMBEDDING_DEPLOYMENT="text-embedding-3-small" 7 | OPENAI_EMBEDDING_MODEL=text-embedding-ada-002 8 | OPENAI_EMBEDDING_MODEL_VERSION=2023-03-15-preview 9 | OPENAI_CHAT_COMPLETION_DEPLOYMENT= 10 | OPENAI_CHAT_COMPLETION_MODEL_VERSION=2023-05-15 11 | MONGODB_CONNECTION_URI= 12 | MONGODB_DATABASE_NAME= 13 | VECTOR_SEARCH_INDEX_NAME= 14 | NODE_ENV=development 15 | ALLOWED_ORIGINS=http://example.com,http://localhost:5173 16 | -------------------------------------------------------------------------------- /packages/scripts/src/findCentroid.ts: -------------------------------------------------------------------------------- 1 | import { strict as assert } from "assert"; 2 | 3 | /** 4 | Finds the centroid (mean) of a set of n-dimensional vectors. 5 | */ 6 | export const findCentroid = (vectors: number[][]) => { 7 | assert(vectors.length > 0, "Cannot find centroid of 0 vectors!"); 8 | assert( 9 | vectors.find((vector) => vectors[0].length !== vector.length) === undefined, 10 | "Vector dimensionality mismatch!" 11 | ); 12 | return ( 13 | vectors 14 | // Add all vectors together into one vector (add each dimension together) 15 | .reduce((acc, cur) => cur.map((d, i) => (acc[i] ?? 0) + d), []) 16 | // Average each dimension of the combined vector 17 | .map((d) => d / vectors.length) 18 | ); 19 | }; 20 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/release-notes/ReleaseInfo.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { GitHubReleaseInfo } from "./github"; 3 | import { JiraReleaseInfo } from "./jira"; 4 | import { stripIndents } from "common-tags"; 5 | 6 | export const ReleaseInfo = z.object({ 7 | github: GitHubReleaseInfo.optional(), 8 | jira: JiraReleaseInfo.optional(), 9 | projectDescription: z.string().describe(stripIndents` 10 | A contextual description of the software project. This should 11 | describe the purpose, use cases, and goals of the project. In 12 | particular, it should describe the project's users and the 13 | way they interact with the project. 14 | `), 15 | }); 16 | 17 | export type ReleaseInfo = z.infer; 18 | -------------------------------------------------------------------------------- /packages/benchmarks/src/textToDriver/generateDriverCode/languagePrompts/listExamplesInPrompt.test.ts: -------------------------------------------------------------------------------- 1 | import { listExamplesInPrompt } from "./listExamplesInPrompt"; 2 | 3 | describe("listExamplesInPrompt", () => { 4 | it("formats an empty list correctly", () => { 5 | const result = listExamplesInPrompt([]); 6 | expect(result).toBe(""); 7 | }); 8 | 9 | it("formats a single item list correctly", () => { 10 | const result = listExamplesInPrompt(["Test item"]); 11 | expect(result).toBe("1. Test item"); 12 | }); 13 | 14 | it("formats multiple items correctly", () => { 15 | const result = listExamplesInPrompt(["Item one", "Item two", "Item three"]); 16 | expect(result).toBe("1. Item one\n2. Item two\n3. Item three"); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/.env.example: -------------------------------------------------------------------------------- 1 | PORT=3000 2 | OPENAI_ENDPOINT=https://.openai.azure.com/ 3 | OPENAI_API_KEY= 4 | OPENAI_RETRIEVAL_EMBEDDING_DEPLOYMENT="text-embedding-3-small" 5 | OPENAI_VERIFIED_ANSWER_EMBEDDING_DEPLOYMENT="docs-chatbot-embedding-ada-002" 6 | OPENAI_EMBEDDING_MODEL=text-embedding-ada-002 7 | OPENAI_EMBEDDING_MODEL_VERSION=2023-03-15-preview 8 | OPENAI_CHAT_COMPLETION_DEPLOYMENT= 9 | OPENAI_CHAT_COMPLETION_MODEL_VERSION= 10 | MONGODB_CONNECTION_URI= 11 | MONGODB_DATABASE_NAME= 12 | VECTOR_SEARCH_INDEX_NAME= 13 | OPENAI_API_VERSION="2024-06-01" 14 | BRAINTRUST_ENDPOINT="https://api.braintrust.dev/v1/proxy" 15 | BRAINTRUST_API_KEY="" 16 | -------------------------------------------------------------------------------- /packages/benchmarks/src/reporting/getBraintrustExperimentSummary.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEnvVars, BRAINTRUST_ENV_VARS } from "mongodb-rag-core"; 2 | import { getBraintrustExperimentSummary } from "./getBraintrustExperimentSummary"; 3 | 4 | describe.skip("getBraintrustExperimentSummary", () => { 5 | it("should return the experiment summary", async () => { 6 | const { BRAINTRUST_API_KEY } = assertEnvVars(BRAINTRUST_ENV_VARS); 7 | const result = await getBraintrustExperimentSummary({ 8 | experimentName: 9 | "mongosh-benchmark-official?experimentType=agentic&model=gemini-2.5-pro-preview-03-25", 10 | projectName: "natural-language-to-mongosh", 11 | apiKey: BRAINTRUST_API_KEY, 12 | }); 13 | expect(result).toBeDefined(); 14 | }); 15 | }); 16 | -------------------------------------------------------------------------------- /packages/datasets/src/uploadDatasetToBraintrust.ts: -------------------------------------------------------------------------------- 1 | import { initDataset, DatasetRecord } from "mongodb-rag-core/braintrust"; 2 | 3 | export async function uploadDatasetToBraintrust({ 4 | apiKey, 5 | datasetName, 6 | projectName, 7 | description, 8 | dataset, 9 | metadata, 10 | }: { 11 | apiKey: string; 12 | datasetName: string; 13 | projectName: string; 14 | description: string; 15 | dataset: Partial[]; 16 | metadata?: Record; 17 | }) { 18 | const btDataset = initDataset({ 19 | apiKey, 20 | dataset: datasetName, 21 | description, 22 | project: projectName, 23 | metadata, 24 | }); 25 | dataset.forEach((d) => btDataset.insert(d)); 26 | const res = await btDataset.summarize(); 27 | return res; 28 | } 29 | -------------------------------------------------------------------------------- /packages/benchmarks/src/textToDriver/makeDatabases/insertCollectionDocuments.ts: -------------------------------------------------------------------------------- 1 | import { MongoClient, Document } from "mongodb-rag-core/mongodb"; 2 | 3 | export async function insertCollectionDocuments({ 4 | client, 5 | documents, 6 | collectionName, 7 | databaseName, 8 | reset = false, 9 | }: { 10 | client: MongoClient; 11 | documents: Document[]; 12 | collectionName: string; 13 | databaseName: string; 14 | reset?: boolean; 15 | }) { 16 | const collection = client.db(databaseName).collection(collectionName); 17 | if (reset) { 18 | console.log( 19 | `Deleting all existing documents from ${databaseName}.${collectionName} before insert...` 20 | ); 21 | await collection.deleteMany({}); 22 | } 23 | await collection.insertMany(documents); 24 | } 25 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/.release-it.json: -------------------------------------------------------------------------------- 1 | { 2 | "git": { 3 | "commitArgs": ["-a"], 4 | "commitMessage": "Release ${npm.name} v${version}", 5 | "changelog": "git log --pretty=format:'* %s (%h)' ${latestTag}..HEAD -- .", 6 | "tagMatch": "${npm.name}-*", 7 | "getLatestTagFromAllRefs": true, 8 | "tag": true, 9 | "push": true, 10 | "pushRepo": "upstream", 11 | "tagName": "${npm.name}-v${version}" 12 | }, 13 | "npm": { 14 | "publish": false 15 | }, 16 | "github": { 17 | "releaseName": "${npm.name}-v${version}", 18 | "draft": true, 19 | "release": true 20 | }, 21 | "hooks": { 22 | "before:init": [ 23 | "pushd ../mongodb-rag-core && npm i && npm run build && popd", 24 | "npm run build" 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/src/main.ts: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | import yargs from "yargs"; 3 | 4 | import "dotenv/config"; 5 | 6 | function commandDir( 7 | argv: yargs.Argv, 8 | directory: string, 9 | options?: yargs.RequireDirectoryOptions 10 | ): yargs.Argv { 11 | return argv.commandDir(directory, { 12 | extensions: ["js"], 13 | exclude: /^(?:index|.*\.test)\.js$/, // .map, .d.ts excluded by 'extensions' property 14 | visit(commandModule) { 15 | return commandModule.default; 16 | }, 17 | ...options, 18 | }); 19 | } 20 | 21 | async function main() { 22 | const argv = commandDir(yargs.help(), "commands").demandCommand(); 23 | 24 | // Accessing this property executes CLI 25 | argv.argv; 26 | } 27 | 28 | main().catch(console.error); 29 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/.release-it.json: -------------------------------------------------------------------------------- 1 | { 2 | "git": { 3 | "commitArgs": ["-a"], 4 | "commitMessage": "Release ${npm.name} v${version}", 5 | "changelog": "git log --pretty=format:'* %s (%h)' ${latestTag}..HEAD -- .", 6 | "tagMatch": "${npm.name}-*", 7 | "getLatestTagFromAllRefs": true, 8 | "tag": true, 9 | "push": true, 10 | "pushRepo": "upstream", 11 | "tagName": "${npm.name}-v${version}" 12 | }, 13 | "npm": { 14 | "publish": false 15 | }, 16 | "github": { 17 | "releaseName": "${npm.name}-v${version}", 18 | "draft": true, 19 | "release": true 20 | }, 21 | "hooks": { 22 | "before:init": [ 23 | "pushd ../mongodb-rag-core && npm i && npm run build && popd", 24 | "npm run build" 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/main.ts: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | import yargs from "yargs"; 3 | 4 | import "dotenv/config"; 5 | 6 | function commandDir( 7 | argv: yargs.Argv, 8 | directory: string, 9 | options?: yargs.RequireDirectoryOptions 10 | ): yargs.Argv { 11 | return argv.commandDir(directory, { 12 | extensions: ["js"], 13 | exclude: /^(?:index|.*\.test)\.js$/, // .map, .d.ts excluded by 'extensions' property 14 | visit(commandModule) { 15 | return commandModule.default; 16 | }, 17 | ...options, 18 | }); 19 | } 20 | 21 | async function main() { 22 | const argv = commandDir(yargs.help(), "commands").demandCommand(); 23 | 24 | // Accessing this property executes CLI 25 | argv.argv; 26 | } 27 | 28 | main().catch(console.error); 29 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/vitest.setup.ts: -------------------------------------------------------------------------------- 1 | import "@testing-library/jest-dom"; 2 | import { cleanup } from "@testing-library/react"; 3 | import { afterEach } from "vitest"; 4 | 5 | // We overwrite console methods to suppress certain errors that are expected & distracting 6 | const originalConsoleError = console.error; 7 | console.error = function (...data) { 8 | // JSDom uses an outdated CSS parser that throws errors on valid modern CSS 9 | // See: https://github.com/jsdom/jsdom/issues/2177#issuecomment-376139329 10 | if ( 11 | typeof data[0]?.toString === "function" && 12 | data[0].toString().startsWith("Error: Could not parse CSS stylesheet") 13 | ) { 14 | return; 15 | } 16 | originalConsoleError(...data); 17 | }; 18 | 19 | afterEach(() => { 20 | cleanup(); 21 | }); 22 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/executeCode/extractCodeFromMarkdown.ts: -------------------------------------------------------------------------------- 1 | /** 2 | Extracts code from markdown by removing all code blocks and returning the rest of the markdown. 3 | If there are no markdown code blocks, the original text is returned. 4 | */ 5 | export function extractCodeFromMarkdown(maybeMarkdown: string) { 6 | if (hasMarkdownCodeblock(maybeMarkdown)) { 7 | const codeBlocks = maybeMarkdown.match(/```[\w]*\n([\s\S]*?)```/g) || []; 8 | return codeBlocks 9 | .map((block) => 10 | block.replaceAll(/```[\w]*\n([\s\S]*?)```/gs, "$1").trim() 11 | ) 12 | .join("\n\n"); 13 | } 14 | return maybeMarkdown; 15 | } 16 | 17 | export function hasMarkdownCodeblock(maybeMarkdown: string) { 18 | return /```[\w]*\n([\s\S]*?)```/g.test(maybeMarkdown); 19 | } 20 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-verified-answers/.release-it.json: -------------------------------------------------------------------------------- 1 | { 2 | "git": { 3 | "commitArgs": ["-a"], 4 | "commitMessage": "Release ${npm.name} v${version}", 5 | "changelog": "git log --pretty=format:'* %s (%h)' ${latestTag}..HEAD -- .", 6 | "tagMatch": "${npm.name}-*", 7 | "getLatestTagFromAllRefs": true, 8 | "tag": true, 9 | "push": true, 10 | "pushRepo": "upstream", 11 | "tagName": "${npm.name}-v${version}" 12 | }, 13 | "npm": { 14 | "publish": false 15 | }, 16 | "github": { 17 | "releaseName": "${npm.name}-v${version}", 18 | "draft": true, 19 | "release": true 20 | }, 21 | "hooks": { 22 | "before:init": [ 23 | "pushd ../mongodb-rag-core && npm i && npm run build && popd", 24 | "npm run build" 25 | ] 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-verified-answers/src/main.ts: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | import yargs from "yargs"; 3 | 4 | import "dotenv/config"; 5 | 6 | function commandDir( 7 | argv: yargs.Argv, 8 | directory: string, 9 | options?: yargs.RequireDirectoryOptions 10 | ): yargs.Argv { 11 | return argv.commandDir(directory, { 12 | extensions: ["js"], 13 | exclude: /^(?:index|.*\.test)\.js$/, // .map, .d.ts excluded by 'extensions' property 14 | visit(commandModule) { 15 | return commandModule.default; 16 | }, 17 | ...options, 18 | }); 19 | } 20 | 21 | async function main() { 22 | const argv = commandDir(yargs.help(), "commands").demandCommand(); 23 | 24 | // Accessing this property executes CLI 25 | argv.argv; 26 | } 27 | 28 | main().catch(console.error); 29 | -------------------------------------------------------------------------------- /packages/datasets/.release-it.json: -------------------------------------------------------------------------------- 1 | { 2 | "git": { 3 | "commitArgs": [ 4 | "-a" 5 | ], 6 | "commitMessage": "Release ${npm.name} v${version}", 7 | "changelog": "git log --pretty=format:'* %s (%h)' ${latestTag}..HEAD -- .", 8 | "tagMatch": "${npm.name}-*", 9 | "getLatestTagFromAllRefs": true, 10 | "tag": true, 11 | "push": true, 12 | "pushRepo": "upstream", 13 | "tagName": "${npm.name}-v${version}" 14 | }, 15 | "npm": { 16 | "publish": false 17 | }, 18 | "github": { 19 | "releaseName": "${npm.name}-v${version}", 20 | "draft": true, 21 | "release": true 22 | }, 23 | "hooks": { 24 | "before:init": [ 25 | "npm run build" 26 | ] 27 | } 28 | } -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/dataSources/ProjectBase.ts: -------------------------------------------------------------------------------- 1 | /** 2 | Base project type used in all project data sources (snooty sites, devcenter, etc.) 3 | */ 4 | export interface ProjectBase { 5 | /** 6 | Type of project 7 | @example "snooty" | "devcenter" 8 | */ 9 | type: string; 10 | /** 11 | Snooty project name 12 | @example "kotlin" 13 | */ 14 | name: string; 15 | /** 16 | Tags to include in all documents from the site in the embedded_content collection 17 | @example ["kotlin", "docs", "driver"] 18 | */ 19 | tags?: string[]; 20 | 21 | /** 22 | Name of the product. 23 | @example "MongoDB Atlas" 24 | */ 25 | productName?: string; 26 | 27 | /** 28 | The version of the project. 29 | @example "v4.4" 30 | */ 31 | version?: string; 32 | } 33 | -------------------------------------------------------------------------------- /packages/mercury-case-analysis/.release-it.json: -------------------------------------------------------------------------------- 1 | { 2 | "git": { 3 | "commitArgs": ["-a"], 4 | "commitMessage": "Release ${npm.name} v${version}", 5 | "changelog": "git log --pretty=format:'* %s (%h)' ${latestTag}..HEAD -- .", 6 | "tagMatch": "${npm.name}-*", 7 | "getLatestTagFromAllRefs": true, 8 | "tag": true, 9 | "push": true, 10 | "pushRepo": "upstream", 11 | "tagName": "${npm.name}-v${version}" 12 | }, 13 | "npm": { 14 | "publish": false 15 | }, 16 | "github": { 17 | "releaseName": "${npm.name}-v${version}", 18 | "draft": true, 19 | "release": true 20 | }, 21 | "hooks": { 22 | "before:init": [ 23 | "pushd ../mongodb-rag-core && npm i && npm run build && popd", 24 | "npm run build", 25 | "npm run lint" 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/.release-it.json: -------------------------------------------------------------------------------- 1 | { 2 | "git": { 3 | "commitArgs": ["-a"], 4 | "commitMessage": "Release ${npm.name} v${version}", 5 | "changelog": "git log --pretty=format:'* %s (%h)' ${latestTag}..HEAD -- .", 6 | "tagMatch": "${npm.name}-*", 7 | "getLatestTagFromAllRefs": true, 8 | "tag": true, 9 | "push": true, 10 | "pushRepo": "upstream", 11 | "tagName": "${npm.name}-v${version}" 12 | }, 13 | "npm": { 14 | "publish": false 15 | }, 16 | "github": { 17 | "releaseName": "${npm.name}-v${version}", 18 | "draft": true, 19 | "release": true 20 | }, 21 | "hooks": { 22 | "before:init": [ 23 | "pushd ../mongodb-rag-core && npm i && npm run build && popd", 24 | "npm run build", 25 | "npm run lint" 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /packages/benchmarks/.env.example: -------------------------------------------------------------------------------- 1 | MONGODB_CONNECTION_URI="mongodb+srv://..." 2 | MONGODB_DATABASE_NAME="..." 3 | # Note: The quiz questions are stored in staging 4 | MONGODB_QUIZ_QUESTIONS_CONNECTION_URI="mongodb+srv://:@docs-chatbot-staging.ngvo1.mongodb.net/?retryWrites=true&w=majority" 5 | MONGODB_QUIZ_QUESTIONS_DATABASE_NAME="datasets" 6 | MONGODB_QUIZ_QUESTIONS_COLLECTION_NAME="quiz_questions" 7 | MONGODB_TEXT_TO_DRIVER_CONNECTION_URI="" 8 | BRAINTRUST_ENDPOINT="https://api.braintrust.dev/v1/proxy" 9 | BRAINTRUST_API_KEY="" 10 | BRAINTRUST_TEXT_TO_DRIVER_PROJECT_NAME="" 11 | OPENAI_API_VERSION="2024-06-01" 12 | OPENAI_ENDPOINT="https://.openai.azure.com/" 13 | OPENAI_API_KEY="" 14 | MONGODB_TEXT_TO_DRIVER_CONNECTION_URI="" 15 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/EnvVars.ts: -------------------------------------------------------------------------------- 1 | const MODEL_JUDGE_ENV_VARS = { 2 | JUDGE_LLM: "", 3 | JUDGE_EMBEDDING_MODEL: "", 4 | }; 5 | 6 | export const EVAL_ENV_VARS = { 7 | ...MODEL_JUDGE_ENV_VARS, 8 | }; 9 | 10 | export const ANALYZER_ENV_VARS = { 11 | OPENAI_ANALYZER_CHAT_COMPLETION_DEPLOYMENT: "", 12 | }; 13 | 14 | export const TRACING_ENV_VARS = { 15 | ...MODEL_JUDGE_ENV_VARS, 16 | ...ANALYZER_ENV_VARS, 17 | }; 18 | 19 | export const AZURE_OPENAI_ENV_VARS = { 20 | OPENAI_API_KEY: "", 21 | OPENAI_API_VERSION: "", 22 | OPENAI_RESOURCE_NAME: "", 23 | }; 24 | 25 | export const PREPROCESSOR_ENV_VARS = { 26 | OPENAI_PREPROCESSOR_CHAT_COMPLETION_DEPLOYMENT: "", 27 | }; 28 | 29 | export const SLACK_ENV_VARS = { 30 | SLACK_BOT_TOKEN: "", 31 | SLACK_COMMENT_CONVERSATION_ID: "", 32 | }; 33 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | browser: true, 4 | es2020: true, 5 | }, 6 | root: true, 7 | parser: "@typescript-eslint/parser", 8 | extends: [ 9 | "eslint:recommended", 10 | "plugin:@typescript-eslint/recommended", 11 | "plugin:react-hooks/recommended", 12 | "plugin:storybook/recommended", 13 | "prettier", 14 | ], 15 | parserOptions: { 16 | ecmaVersion: "latest", 17 | sourceType: "module", 18 | }, 19 | plugins: ["react-refresh"], 20 | rules: { 21 | "react-refresh/only-export-components": "warn", 22 | "@typescript-eslint/no-unused-vars": [ 23 | "warn", 24 | { 25 | varsIgnorePattern: "^_", 26 | argsIgnorePattern: "^_", 27 | ignoreRestSiblings: true, 28 | }, 29 | ], 30 | }, 31 | }; 32 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-verified-answers/src/parseVerifiedAnswersYaml.ts: -------------------------------------------------------------------------------- 1 | import { parse } from "yaml"; 2 | import { z } from "zod"; 3 | import { Reference } from "mongodb-rag-core"; 4 | 5 | /** 6 | Verified answers as specified in the verified answer yaml. 7 | */ 8 | export const VerifiedAnswerSpec = z.object({ 9 | questions: z.array(z.string()), 10 | answer: z.string(), 11 | author_email: z.string(), 12 | hidden: z.optional(z.boolean()), 13 | references: z.array(Reference), 14 | }); 15 | 16 | export type VerifiedAnswerSpec = z.infer; 17 | 18 | export const parseVerifiedAnswerYaml = (yaml: string): VerifiedAnswerSpec[] => { 19 | if (yaml.trim() === "") { 20 | return []; 21 | } 22 | 23 | const parsed = parse(yaml); 24 | return z.array(VerifiedAnswerSpec).parse(parsed); 25 | }; 26 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/executeCode/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./computeNormalizedLogisticalExecutionTime"; 2 | export * from "./computeNormalizedLogarithmicQueryEfficiency"; 3 | export * from "./DatabaseExecutionResult"; 4 | export * from "./databaseMetadata"; 5 | export * from "./executeEjsonAggregation"; 6 | export * from "./executeNodeJsDriverCode"; 7 | export * from "./executeMongoshQuery"; 8 | export * from "./extractCodeFromMarkdown"; 9 | export * from "./extractMongoDbQueryMetadata"; 10 | export * from "./fuzzyMatchExecutionResults"; 11 | export * from "./redactMongoDbConnectionUri"; 12 | export * from "./isNonMutativeOperation"; 13 | export * from "./isReasonableResult"; 14 | export * from "./prettyPrintMongoDbDocument"; 15 | export * from "./profileMongoshQuery"; 16 | export * from "./truncateDbOperationOutputForLlm"; 17 | -------------------------------------------------------------------------------- /packages/benchmarks/src/textToDriver/generateDriverCode/tools/submitFinalSolution.ts: -------------------------------------------------------------------------------- 1 | import { Tool, tool } from "mongodb-rag-core/aiSdk"; 2 | import { z } from "zod"; 3 | 4 | const MongoDbAggregateOperationSchema = z.object({ 5 | databaseName: z.string(), 6 | collectionName: z.string(), 7 | pipeline: z 8 | .array(z.record(z.string(), z.any())) 9 | .describe("MongoDB aggregation pipeline"), 10 | }); 11 | 12 | export type MongoDbAggregateOperation = z.infer< 13 | typeof MongoDbAggregateOperationSchema 14 | >; 15 | 16 | export const submitFinalSolutionToolName = "submit-final-solution"; 17 | 18 | export const submitFinalSolutionTool: Tool = tool({ 19 | name: submitFinalSolutionToolName, 20 | description: "Submit the final solution of MongoDB operation", 21 | inputSchema: MongoDbAggregateOperationSchema, 22 | }); 23 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/dataSources/extractMarkdownH1.test.ts: -------------------------------------------------------------------------------- 1 | import { extractMarkdownH1 } from "./extractMarkdownH1"; 2 | 3 | describe("extractMarkdownH1()", () => { 4 | it("extracts the H1 from a markdown page", () => { 5 | const page = `# My Page Title 6 | 7 | Some content 8 | 9 | ## H2`; 10 | const h1 = extractMarkdownH1(page); 11 | expect(h1).toEqual("My Page Title"); 12 | }); 13 | it("returns null if no H1 exists", () => { 14 | const page = `## H2 15 | 16 | Some content`; 17 | const h1 = extractMarkdownH1(page); 18 | expect(h1).toBeUndefined(); 19 | }); 20 | it("finds H1 not at top of page", () => { 21 | const page = `## H2 22 | 23 | # My Page Title 24 | 25 | Some content`; 26 | const h1 = extractMarkdownH1(page); 27 | expect(h1).toEqual("My Page Title"); 28 | }); 29 | }); 30 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/test/mockOpenAi.ts: -------------------------------------------------------------------------------- 1 | export const makeMockOpenAIToolCall = (funcRes: Record) => { 2 | return { 3 | OpenAI: jest.fn().mockImplementation(() => { 4 | return { 5 | chat: { 6 | completions: { 7 | create: jest.fn().mockResolvedValue({ 8 | choices: [ 9 | { 10 | message: { 11 | tool_calls: [ 12 | { 13 | function: { 14 | arguments: JSON.stringify(funcRes), 15 | }, 16 | }, 17 | ], 18 | }, 19 | }, 20 | ], 21 | }), 22 | }, 23 | }, 24 | }; 25 | }), 26 | }; 27 | }; 28 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/References.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | 3 | /** 4 | A formatted reference for an assistant message. 5 | 6 | For example, a Reference might be a docs page, dev center article, or 7 | a MongoDB University module. 8 | */ 9 | export type Reference = z.infer; 10 | export const Reference = z.object({ 11 | url: z.string(), 12 | title: z.string(), 13 | metadata: z 14 | .object({ 15 | sourceName: z.string().optional().describe("The name of the source."), 16 | sourceType: z.string().optional(), 17 | tags: z.array(z.string()).optional(), 18 | }) 19 | .passthrough() // We accept additional unknown metadata fields 20 | .optional(), 21 | }); 22 | 23 | export type References = z.infer; 24 | export const References = z.array(Reference); 25 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/dataSources/removeMarkdownImagesAndLinks.ts: -------------------------------------------------------------------------------- 1 | /** 2 | Utility function to remove markdown images and links from a string. 3 | Useful if you do not want to include images and links in content, 4 | which can add significantly add to the token count when creating embeddings 5 | while also diluting the semantic meaning of the content. 6 | */ 7 | export function removeMarkdownImagesAndLinks(content: string) { 8 | const mdLink = /!?\[([\s\S]*?)\]\([\s\S]*?\)/g; 9 | 10 | let cleanedContent = content.replaceAll(mdLink, (match, text) => { 11 | // remove images 12 | if (match.startsWith("!")) { 13 | return ""; 14 | } else return text; 15 | }); 16 | // remove unnecessary new lines 17 | cleanedContent = cleanedContent.replaceAll(/\n{3,}/g, "\n\n"); 18 | 19 | return cleanedContent; 20 | } 21 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/context/tdbx-content-types/whats-new.yaml: -------------------------------------------------------------------------------- 1 | type: whats-new 2 | name: What’s New 3 | objectives: 4 | - Inform users of new features/functionality 5 | - Inform users of behavior changes 6 | - Inform users of deprecated/replaced functionality 7 | pageStructure: | 8 | ========== 9 | What’s New 10 | ========== 11 | 12 | 13 | 14 | What's New in x.xx 15 | ------------------ 16 | 17 | List the features, behavior changes, and deprecated/replaced functionality new to this version 18 | 19 | Link to release notes if they provide more insight. 20 | if useful. 21 | examples: 22 | - https://mongodb.com/docs/drivers/go/current/whats-new/ 23 | - https://mongodb.com/docs/drivers/node/current/whats-new/ 24 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/context/tdbx-content-types/migrate-legacy-version.yaml: -------------------------------------------------------------------------------- 1 | type: migrate-legacy-version 2 | name: Migration from Legacy Version 3 | objectives: 4 | - Provide guidance on how to migrate from a legacy driver/library to 5 | the supported driver versions. 6 | pageStructure: | 7 | =========================== 8 | Migrate from the Legacy API 9 | =========================== 10 | 11 | Overview 12 | ---------- 13 | 14 | Provide instructions on how to identify whether the reader is using the legacy versions or API. 15 | Describe a common solution to making changes 16 | 17 | API Changes 18 | ----------- 19 | 20 | Provide a list of changes that explains the legacy implementation and corresponding current implementation 21 | examples: 22 | - https://mongodb.com/docs/drivers/java/sync/current/legacy/ 23 | ... 24 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/assertEnvVars.ts: -------------------------------------------------------------------------------- 1 | export const assertEnvVars = >( 2 | ENV_VARS: T 3 | ): T => { 4 | const vars: Record = { ...ENV_VARS }; 5 | const anyVarMissing = Object.keys(vars).reduce( 6 | (anyVarMissing, currentVar) => { 7 | // Get this var value from process.env - either valid string or undefined 8 | vars[currentVar] = process.env[currentVar]; 9 | 10 | // Return true if any var was missing so far or if this var was not set 11 | return anyVarMissing || !vars[currentVar]; 12 | }, 13 | false 14 | ); 15 | if (anyVarMissing) { 16 | throw new Error(`Missing env var(s): 17 | ${Object.entries(vars) 18 | .filter(([, value]) => !value) 19 | .map(([key]) => `- ${key}`) 20 | .join("\n")}`); 21 | } 22 | return vars as T; 23 | }; 24 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/dataSources/extractMarkdownH1.ts: -------------------------------------------------------------------------------- 1 | /** 2 | Extract the first H1 from a Markdown string 3 | 4 | @param mdContent - some Markdown content 5 | @returns page H1 `string` if it exists, otherwise `null` 6 | @example 7 | ```ts 8 | const mdContent = "# My Page Title"; 9 | const pageTitle = extractMarkdownH1(mdContent); 10 | console.log(pageTitle); // "My Page Title" 11 | 12 | const noTitleMdContent = "content without a h1"; 13 | const noTitle = extractMarkdownH1(noTitleMdContent); 14 | console.log(noTitle); // null 15 | ``` 16 | */ 17 | export function extractMarkdownH1(mdContent: string) { 18 | const h1Regex = /^#\s(.+)$/gm; 19 | const matches = mdContent.match(h1Regex); 20 | if (matches && matches.length > 0) { 21 | const title = matches[0].replace("# ", ""); 22 | return title; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /packages/scripts/.release-it.json: -------------------------------------------------------------------------------- 1 | { 2 | "git": { 3 | "commitArgs": ["-a"], 4 | "commitMessage": "Release ${npm.name} v${version}", 5 | "changelog": "git log --pretty=format:'* %s (%h)' ${latestTag}..HEAD -- .", 6 | "tagMatch": "${npm.name}-*", 7 | "getLatestTagFromAllRefs": true, 8 | "tag": true, 9 | "push": true, 10 | "pushRepo": "upstream", 11 | "tagName": "${npm.name}-v${version}" 12 | }, 13 | "npm": { 14 | "publish": false 15 | }, 16 | "github": { 17 | "releaseName": "${npm.name}-v${version}", 18 | "draft": true, 19 | "release": true 20 | }, 21 | "hooks": { 22 | "before:init": [ 23 | "pushd ../mongodb-rag-core && npm i && npm run build && popd", 24 | "pushd ../mongodb-chatbot-server && npm i && npm run build && popd", 25 | "npm run build" 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/testData/sitemap-pages.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | https://www.mongodb.com/blog/post/supercharge-ai-data-management-knowledge-graphs 5 | 6 | 7 | https://www.mongodb.com/blog/post/reintroducing-versioned-mongodb-atlas-administration-api 8 | 9 | 10 | https://www.mongodb.com/blog/post/building-gen-ai-mongodb-ai-partners-january-2025 11 | 12 | 13 | https://www.mongodb.com/blog/post/mongodb-empowers-isvs-to-drive-saas-innovation-india 14 | 15 | 16 | https://www.mongodb.com/blog/post/simplify-security-at-scale-resource-policies-mongodb-atlas 17 | 18 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/chat/index.ts: -------------------------------------------------------------------------------- 1 | import { OpenAI } from "mongodb-rag-core/openai"; 2 | 3 | export function chatMessage(t: T) { 4 | return t; 5 | } 6 | 7 | export const systemMessage = ( 8 | args: Omit 9 | ): OpenAI.ChatCompletionSystemMessageParam => 10 | chatMessage({ role: "system", ...args }); 11 | 12 | export const userMessage = ( 13 | args: Omit 14 | ): OpenAI.ChatCompletionUserMessageParam => 15 | chatMessage({ role: "user", ...args }); 16 | 17 | export const assistantMessage = ( 18 | args: Omit 19 | ): OpenAI.ChatCompletionAssistantMessageParam => 20 | chatMessage({ role: "assistant", ...args }); 21 | 22 | export * from "./makeGenerateChatCompletion"; 23 | -------------------------------------------------------------------------------- /packages/datasets/src/openAi.ts: -------------------------------------------------------------------------------- 1 | import { assertEnvVars, BRAINTRUST_ENV_VARS } from "mongodb-rag-core"; 2 | import { OpenAI } from "mongodb-rag-core/openai"; 3 | import { wrapOpenAI } from "mongodb-rag-core/braintrust"; 4 | import { createOpenAI } from "mongodb-rag-core/aiSdk"; 5 | 6 | export const makeOpenAiClient = () => { 7 | const { BRAINTRUST_API_KEY, BRAINTRUST_ENDPOINT } = 8 | assertEnvVars(BRAINTRUST_ENV_VARS); 9 | return wrapOpenAI( 10 | new OpenAI({ 11 | apiKey: BRAINTRUST_API_KEY, 12 | baseURL: BRAINTRUST_ENDPOINT, 13 | }) 14 | ); 15 | }; 16 | 17 | export const makeOpenAiProvider = () => { 18 | const { BRAINTRUST_API_KEY, BRAINTRUST_ENDPOINT } = 19 | assertEnvVars(BRAINTRUST_ENV_VARS); 20 | return createOpenAI({ 21 | apiKey: BRAINTRUST_API_KEY, 22 | baseURL: BRAINTRUST_ENDPOINT, 23 | }); 24 | }; 25 | 26 | export const model = "gpt-4o"; 27 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/.release-it.json: -------------------------------------------------------------------------------- 1 | { 2 | "git": { 3 | "commitArgs": ["-a"], 4 | "commitMessage": "Release ${npm.name} v${version}", 5 | "changelog": "git log --pretty=format:'* %s (%h)' ${latestTag}..HEAD -- .", 6 | "tagMatch": "${npm.name}-*", 7 | "getLatestTagFromAllRefs": true, 8 | "tag": true, 9 | "push": true, 10 | "pushRepo": "upstream", 11 | "tagName": "${npm.name}-v${version}" 12 | }, 13 | "npm": { 14 | "publish": false 15 | }, 16 | "github": { 17 | "releaseName": "${npm.name}-v${version}", 18 | "draft": true, 19 | "release": true 20 | }, 21 | "hooks": { 22 | "before:init": [ 23 | "pushd ../mongodb-rag-core && npm i && npm run build && popd", 24 | "pushd ../mongodb-chatbot-server && npm i && npm run build && popd", 25 | "npm run build", 26 | "npm run lint" 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /packages/datasets/src/bin/countExamplesByLength.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | import yaml from "yaml"; 4 | import { AugmentedAstExtractedCodeblockWithUtility } from "../codeExampleDataset/AstExtractedCodeBlock.js"; 5 | function main() { 6 | const LEN = 150; 7 | const basePath = path.resolve("data"); 8 | const pathIn = path.resolve( 9 | basePath, 10 | "docs-chatbot.code-examples-with-prompts-with-utility-filtered-for-export-1723650460861.yaml" 11 | ); 12 | const examples = yaml.parse( 13 | fs.readFileSync(pathIn, "utf-8") 14 | ) as AugmentedAstExtractedCodeblockWithUtility[]; 15 | console.log("Tot num examples:", examples.length); 16 | const usableExamples = examples.filter( 17 | (example) => example.code.length >= LEN && example.isUseful 18 | ); 19 | console.log(`Num examples of length >= ${LEN}:`, usableExamples.length); 20 | } 21 | main(); 22 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/src/PublicIngestEnvVars.ts: -------------------------------------------------------------------------------- 1 | import { 2 | CORE_CHATBOT_APP_ENV_VARS, 3 | CORE_OPENAI_RETRIEVAL_ENV_VARS, 4 | } from "mongodb-rag-core"; 5 | import { INGEST_ENV_VARS } from "mongodb-rag-ingest"; 6 | 7 | export const PUBLIC_INGEST_MONGODB_DOT_COM_ENV_VARS = { 8 | MONGODB_DOT_COM_CONNECTION_URI: "", 9 | MONGODB_DOT_COM_DB_NAME: "", 10 | }; 11 | 12 | export const PUBLIC_INGEST_MONGODB_DOCS_META_ENV_VARS = { 13 | MONGODB_META_DATABASE_NAME: "", 14 | }; 15 | 16 | export const INGEST_DEVCENTER_ENV_VARS = { 17 | DEVCENTER_CONNECTION_URI: "", 18 | }; 19 | 20 | export const INGEST_MDBU_ENV_VARS = { 21 | UNIVERSITY_DATA_API_KEY: "", 22 | }; 23 | 24 | export const PUBLIC_INGEST_ENV_VARS = { 25 | ...CORE_CHATBOT_APP_ENV_VARS, 26 | ...CORE_OPENAI_RETRIEVAL_ENV_VARS, 27 | ...INGEST_ENV_VARS, 28 | ...INGEST_DEVCENTER_ENV_VARS, 29 | ...INGEST_MDBU_ENV_VARS, 30 | }; 31 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/App.module.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | } 4 | 5 | .app_background { 6 | box-sizing: border-box; 7 | display: flex; 8 | flex-direction: column; 9 | align-items: center; 10 | gap: 20px; 11 | background: #001e2b; 12 | padding: 20px; 13 | height: 100vh; 14 | } 15 | 16 | .background_light { 17 | /* palette.gray.light3 */ 18 | background: #f9fbfa; 19 | } 20 | 21 | .background_dark { 22 | /* palette.black */ 23 | background: #001e2b; 24 | } 25 | 26 | .controls_container { 27 | display: flex; 28 | flex-direction: column; 29 | align-items: center; 30 | gap: 1rem; 31 | max-width: 450px; 32 | } 33 | 34 | .main_content { 35 | max-width: 880px; 36 | min-width: 315px; 37 | width: 100%; 38 | flex-grow: 1; 39 | } 40 | 41 | .streaming_toggle { 42 | display: flex; 43 | align-items: center; 44 | gap: 0.5rem; 45 | width: 100%; 46 | } 47 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/corsOptions.ts: -------------------------------------------------------------------------------- 1 | import { CorsOptions } from "cors"; 2 | 3 | export const makeCorsOptions = ( 4 | isProduction: boolean, 5 | allowedOrigins: string[] 6 | ) => 7 | ({ 8 | origin: !isProduction 9 | ? (origin, callback) => { 10 | // Allow all localhost origins in non-production 11 | if ( 12 | !origin || 13 | origin.includes("localhost") || 14 | origin.includes("127.0.0.1") 15 | ) { 16 | callback(null, true); 17 | } else if (allowedOrigins.includes(origin)) { 18 | callback(null, true); 19 | } else { 20 | callback(new Error("Not allowed by CORS")); 21 | } 22 | } 23 | : allowedOrigins, 24 | // Allow cookies from different origins to be sent to the server. 25 | credentials: true, 26 | } satisfies CorsOptions); 27 | -------------------------------------------------------------------------------- /packages/datasets/src/treeGeneration/databaseNlQueries/databaseNodes/generateUseCases.test.ts: -------------------------------------------------------------------------------- 1 | import { generateDatabaseUseCases } from "./generateUseCases"; 2 | import { makeSampleLlmOptions, userNodes } from "./sampleData/mql"; 3 | 4 | // Skipping LLM call tests 5 | describe.skip("generateDatabaseUseCases", () => { 6 | jest.setTimeout(300000); // Increase timeout for OpenAI API calls 7 | 8 | it("should generate use cases for a database user", async () => { 9 | for (const userNode of userNodes) { 10 | const useCases = await generateDatabaseUseCases( 11 | userNode, 12 | makeSampleLlmOptions(), 13 | 1 14 | ); 15 | console.log("Use cases for user:", userNode.data.name); 16 | console.log( 17 | JSON.stringify( 18 | useCases.map((useCase) => useCase.data), 19 | null, 20 | 2 21 | ) 22 | ); 23 | } 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/docs-metadata/generateMetadata.ts: -------------------------------------------------------------------------------- 1 | import { OpenAI } from "mongodb-rag-core/openai"; 2 | import { RunLogger } from "../runlogger"; 3 | import { makeGenerateMetaDescription } from "./generateMetaDescription"; 4 | 5 | export type DocsMetadata = { 6 | description: string; 7 | }; 8 | 9 | export function makeGenerateDocsMetadata({ 10 | openAiClient, 11 | logger, 12 | }: { 13 | openAiClient: OpenAI; 14 | logger?: RunLogger; 15 | }) { 16 | const generateMetaDescription = makeGenerateMetaDescription({ 17 | openAiClient, 18 | logger, 19 | }); 20 | 21 | return async function generateDocsMetadata({ 22 | url, 23 | text, 24 | }: { 25 | url: string; 26 | text: string; 27 | }): Promise { 28 | const description = await generateMetaDescription({ url, text }); 29 | 30 | return { 31 | description, 32 | }; 33 | }; 34 | } 35 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/frontMatter/updateFrontMatter.test.ts: -------------------------------------------------------------------------------- 1 | import { updateFrontMatter } from "./updateFrontMatter"; 2 | describe("updateFrontMatter()", () => { 3 | test("should add frontmatter", () => { 4 | const text = "foo"; 5 | const metadata = { bar: "baz", a: ["b", "c"] }; 6 | const output = updateFrontMatter(text, metadata); 7 | const expected = `--- 8 | bar: baz 9 | a: 10 | - b 11 | - c 12 | --- 13 | 14 | foo`; 15 | expect(output).toBe(expected); 16 | }); 17 | test("should update frontmatter", () => { 18 | const text = `--- 19 | bar: baz 20 | lorem: ipsum 21 | --- 22 | 23 | foo`; 24 | const metadata = { bar: "qux", a: ["b", "c"] }; 25 | const output = updateFrontMatter(text, metadata); 26 | const expected = `--- 27 | bar: qux 28 | lorem: ipsum 29 | a: 30 | - b 31 | - c 32 | --- 33 | 34 | foo`; 35 | expect(output).toBe(expected); 36 | }); 37 | }); 38 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/.env.example: -------------------------------------------------------------------------------- 1 | # Chatbot App 2 | MONGODB_CONNECTION_URI= 3 | MONGODB_DATABASE_NAME= 4 | OPENAI_ENDPOINT=https://.openai.azure.com/ 5 | OPENAI_API_KEY= 6 | OPENAI_VERIFIED_ANSWER_EMBEDDING_DEPLOYMENT="docs-chatbot-embedding-ada-002" 7 | OPENAI_RETRIEVAL_EMBEDDING_DEPLOYMENT="text-embedding-3-small" 8 | OPENAI_API_VERSION="2024-06-01" 9 | # Data Sources 10 | ## MongoDB University 11 | UNIVERSITY_DATA_API_KEY= 12 | ## MongoDB Developer Center 13 | DEVCENTER_CONNECTION_URI="mongodb+srv://:@devhub-cluster.sewho.mongodb.net/?retryWrites=true&w=majority" 14 | ## MongoDB Marketing CMS (mongodb.com) 15 | MONGODB_DOT_COM_CONNECTION_URI="mongodb+srv://..." 16 | MONGODB_DOT_COM_DB_NAME= 17 | ## Docs style guide, meta reference (mongodb.com/docs/meta) 18 | MONGODB_META_DATABASE_NAME="docs-meta" 19 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/context/tdbx-content-types/upgrade-version.yaml: -------------------------------------------------------------------------------- 1 | type: upgrade-version 2 | name: Upgrading Versions 3 | objectives: 4 | - Help a reader identify what changes they need to make to their 5 | application or build/deployment environment when upgrading to a new 6 | version. 7 | pageStructure: | 8 | ================== 9 | 10 | ================== 11 | 12 | Overview 13 | -------- 14 | 15 | Identify Link to required products for upgrading 16 | 17 | Breaking Changes 18 | ---------------- 19 | 20 | Provide lists of breaking changes for each version to help the reader understand what they need to change when upgrading to a newer version of the driver. 21 | 22 | For drivers, create a separate section to list driver versions that drop support for which server versions. 23 | examples: 24 | - https://mongodb.com/docs/drivers/java/sync/current/upgrade/ 25 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/processors/QueryPreprocessorFunc.ts: -------------------------------------------------------------------------------- 1 | import { Message } from "mongodb-rag-core"; 2 | 3 | export type QueryProcessorFuncParams = { 4 | query: string; 5 | messages?: Message[]; 6 | }; 7 | 8 | /** 9 | Query preprocessors transform an input query to a new query based on your 10 | business logic. 11 | 12 | If the preprocessor can't transform the query, it may return undefined. The 13 | preprocessor may also suggest not answering with the rejectQuery field in the 14 | return value (for example, if the query disparages your company, you might 15 | want to send a canned response). 16 | */ 17 | export type QueryPreprocessorFunc< 18 | ReturnType extends QueryPreprocessorResult = QueryPreprocessorResult 19 | > = (params: QueryProcessorFuncParams) => Promise; 20 | 21 | export type QueryPreprocessorResult = { 22 | query?: string; 23 | rejectQuery?: boolean; 24 | }; 25 | -------------------------------------------------------------------------------- /packages/datasets/src/codeExampleDataset/AstExtractedCodeBlock.ts: -------------------------------------------------------------------------------- 1 | import { Code } from "mdast"; 2 | export interface AstExtractedCodeblock { 3 | code: string; 4 | programmingLanguage: string | null; 5 | metadata: { 6 | pageUrl: string; 7 | sourceName: string; 8 | tags?: string[]; 9 | pageTitle?: string; 10 | mdastNode?: Code; 11 | parentHeadings?: { 12 | h1?: string; 13 | h2?: string; 14 | h3?: string; 15 | h4?: string; 16 | h5?: string; 17 | h6?: string; 18 | }; 19 | }; 20 | } 21 | 22 | export type AugmentedAstExtractedCodeblock = AstExtractedCodeblock & { 23 | prompts: string[]; 24 | classification: string; 25 | }; 26 | 27 | export interface CodeExampleUtility { 28 | isUseful: boolean; 29 | usefulnessReasoning: string; 30 | } 31 | export type AugmentedAstExtractedCodeblockWithUtility = 32 | AugmentedAstExtractedCodeblock & CodeExampleUtility; 33 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/FloatingActionButtonTrigger.tsx: -------------------------------------------------------------------------------- 1 | import { css, cx } from "@emotion/css"; 2 | import { 3 | ActionButtonTrigger, 4 | type ActionButtonTriggerProps, 5 | } from "./ActionButtonTrigger"; 6 | 7 | const styles = { 8 | chat_trigger: css` 9 | position: fixed; 10 | bottom: 24px; 11 | right: 24px; 12 | 13 | @media screen and (min-width: 768px) { 14 | bottom: 32px; 15 | right: 24px; 16 | } 17 | @media screen and (min-width: 1024px) { 18 | bottom: 32px; 19 | right: 49px; 20 | } 21 | `, 22 | }; 23 | 24 | export type FloatingActionButtonTriggerProps = ActionButtonTriggerProps; 25 | 26 | export function FloatingActionButtonTrigger({ 27 | className, 28 | ...props 29 | }: FloatingActionButtonTriggerProps) { 30 | return ( 31 | 35 | ); 36 | } 37 | -------------------------------------------------------------------------------- /packages/datasets/src/bin/countUsableExamples.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | import yaml from "yaml"; 4 | import { AugmentedAstExtractedCodeblock } from "../codeExampleDataset/AstExtractedCodeBlock.js"; 5 | function main() { 6 | const basePath = path.resolve("data"); 7 | const pathIn = path.resolve( 8 | basePath, 9 | "docs-chatbot.code-examples-with-prompts-all-1713802102876.yaml" 10 | ); 11 | const examples = yaml.parse( 12 | fs.readFileSync(pathIn, "utf-8") 13 | ) as AugmentedAstExtractedCodeblock[]; 14 | const usableExamples = examples.filter( 15 | (example) => 16 | example.metadata.sourceName.includes("snooty") && 17 | ![ 18 | "unknown", 19 | "error_message", 20 | "execution_output", 21 | "example_data", 22 | ].includes(example.classification) 23 | ); 24 | console.log("Num usable examples:", usableExamples.length); 25 | } 26 | main(); 27 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/context/tdbx-content-types/issue-help.yaml: -------------------------------------------------------------------------------- 1 | type: issue-help 2 | name: Issues & Help 3 | objectives: 4 | - Provide information on how to get support for general questions, how to report bugs, how to request new features, and how to contribute to the driver. 5 | pageStructure: | 6 | ============= 7 | Issues & Help 8 | ============= 9 | 10 | Link to MongoDB community forum. 11 | 12 | Bugs / Feature Requests 13 | ----------------------- 14 | 15 | Provide the user with instructions on how to open a ticket on JIRA. 16 | 17 | Pull Requests 18 | ------------- 19 | 20 | Provide the user with instructions on how to open a PR against the official driver repository. 21 | examples: 22 | - https://mongodb.com/docs/kafka-connector/current/issues-and-help/ 23 | - https://mongodb.com/docs/drivers/go/current/issues-and-help/ 24 | - https://mongodb.com/docs/drivers/node/current/issues-and-help/ 25 | -------------------------------------------------------------------------------- /packages/datasets/src/uploadDatasetToHuggingFace.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEnvVars } from "mongodb-rag-core"; 2 | import { uploadDatasetToHuggingFace } from "./uploadDatasetToHuggingFace"; 3 | import { HUGGINGFACE } from "./EnvVars"; 4 | 5 | describe.skip("UploadDatasetToHuggingFaceParams", () => { 6 | jest.setTimeout(60000); 7 | it("should upload data to a HF repository", async () => { 8 | const { HUGGINGFACE_ACCESS_TOKEN } = assertEnvVars(HUGGINGFACE); 9 | await uploadDatasetToHuggingFace({ 10 | huggingFace: { 11 | repoName: "bpmutter/test_integration", 12 | accessToken: HUGGINGFACE_ACCESS_TOKEN, 13 | }, 14 | commit: { 15 | title: "Add dataset", 16 | description: "My dataset...", 17 | }, 18 | data: [ 19 | { 20 | path: "data.json", 21 | content: new Blob([JSON.stringify({ foo: "bar" })]), 22 | }, 23 | ], 24 | }); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/tracing/scrubbedMessages/ScrubbedMessageStore.ts: -------------------------------------------------------------------------------- 1 | import { ObjectId } from "mongodb-rag-core/mongodb"; 2 | import { ScrubbedMessage } from "./ScrubbedMessage"; 3 | 4 | export interface ScrubbedMessageStore< 5 | SmAnalysis extends Record | undefined = undefined 6 | > { 7 | insertScrubbedMessage(args: { 8 | message: ScrubbedMessage; 9 | }): Promise; 10 | insertScrubbedMessages(args: { 11 | messages: ScrubbedMessage[]; 12 | }): Promise; 13 | updateScrubbedMessage(args: { 14 | id: ObjectId; 15 | message: Partial, "_id">>; 16 | }): Promise; 17 | findScrubbedMessageById(args: { 18 | id: ObjectId; 19 | }): Promise | null>; 20 | findScrubbedMessagesByConversationId(args: { 21 | conversationId: ObjectId; 22 | }): Promise[]>; 23 | } 24 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/MongoDbDatabaseConnection.ts: -------------------------------------------------------------------------------- 1 | import { DatabaseConnection } from "./DatabaseConnection"; 2 | import { MongoClient, Db } from "mongodb"; 3 | 4 | export interface MakeMongoDbDatabaseConnectionParams { 5 | connectionUri: string; 6 | databaseName: string; 7 | } 8 | 9 | /** 10 | Constructs connection to MongoDB database. 11 | */ 12 | export function makeMongoDbDatabaseConnection({ 13 | connectionUri, 14 | databaseName, 15 | }: MakeMongoDbDatabaseConnectionParams): DatabaseConnection & { 16 | mongoClient: MongoClient; 17 | db: Db; 18 | } { 19 | const mongoClient = new MongoClient(connectionUri, { 20 | serverSelectionTimeoutMS: 30000, 21 | }); 22 | const db = mongoClient.db(databaseName); 23 | return { 24 | mongoClient, 25 | db, 26 | async drop() { 27 | await db.dropDatabase(); 28 | }, 29 | 30 | async close(force) { 31 | mongoClient.close(force); 32 | }, 33 | }; 34 | } 35 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Website 2 | 3 | This website is built using [Docusaurus 2](https://docusaurus.io/), a modern static website generator. 4 | 5 | ### Installation 6 | 7 | ``` 8 | $ yarn 9 | ``` 10 | 11 | ### Local Development 12 | 13 | ``` 14 | $ yarn start 15 | ``` 16 | 17 | This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server. 18 | 19 | ### Build 20 | 21 | ``` 22 | $ yarn build 23 | ``` 24 | 25 | This command generates static content into the `build` directory and can be served using any static contents hosting service. 26 | 27 | ### Deployment 28 | 29 | Using SSH: 30 | 31 | ``` 32 | $ USE_SSH=true yarn deploy 33 | ``` 34 | 35 | Not using SSH: 36 | 37 | ``` 38 | $ GIT_USER= yarn deploy 39 | ``` 40 | 41 | If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch. 42 | -------------------------------------------------------------------------------- /packages/datasets/src/test/globalSetup.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | import { IP, REPLICA_SET_PORT, SERVER_PORT } from "./constants"; 3 | import "./globals"; 4 | export default async function () { 5 | try { 6 | const mongoMemoryServerInstance = await MongoMemoryServer.create({ 7 | instance: { 8 | port: SERVER_PORT, 9 | ip: IP, 10 | }, 11 | }); 12 | global.__MONGO_MEMORY_SERVER_INSTANCE = mongoMemoryServerInstance; 13 | 14 | const mongoMemoryServerReplicaSet = await MongoMemoryReplSet.create({ 15 | instanceOpts: [ 16 | { 17 | port: REPLICA_SET_PORT, 18 | }, 19 | ], 20 | replSet: { 21 | ip: IP, 22 | }, 23 | }); 24 | global.__MONGO_MEMORY_REPLICA_SET = mongoMemoryServerReplicaSet; 25 | } catch (error) { 26 | console.error("Error in global setup:", error); 27 | throw error; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/createMessage.ts: -------------------------------------------------------------------------------- 1 | import { type References } from "./references"; 2 | import { 3 | AssistantMessageMetadata, 4 | MessageData, 5 | } from "./services/conversations"; 6 | 7 | export function createMessageId() { 8 | const now = Date.now(); 9 | const nonce = Math.floor(Math.random() * 100000); 10 | return String(now + nonce); 11 | } 12 | 13 | export type CreateMessageArgs = 14 | | { 15 | id?: string; 16 | role: "assistant"; 17 | content: string; 18 | references?: References; 19 | metadata?: AssistantMessageMetadata; 20 | } 21 | | { 22 | id?: string; 23 | role: "user"; 24 | content: string; 25 | }; 26 | 27 | export default function createMessage(args: CreateMessageArgs): MessageData { 28 | const message: MessageData = { 29 | id: args.id ?? createMessageId(), 30 | createdAt: new Date().toISOString(), 31 | ...args, 32 | }; 33 | return message; 34 | } 35 | -------------------------------------------------------------------------------- /packages/benchmarks/src/test/globalSetup.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | import { IP, REPLICA_SET_PORT, SERVER_PORT } from "./constants"; 3 | import "./globals"; 4 | 5 | export default async function () { 6 | try { 7 | const mongoMemoryServerInstance = await MongoMemoryServer.create({ 8 | instance: { 9 | port: SERVER_PORT, 10 | ip: IP, 11 | }, 12 | }); 13 | global.__MONGO_MEMORY_SERVER_INSTANCE = mongoMemoryServerInstance; 14 | 15 | const mongoMemoryServerReplicaSet = await MongoMemoryReplSet.create({ 16 | instanceOpts: [ 17 | { 18 | port: REPLICA_SET_PORT, 19 | }, 20 | ], 21 | replSet: { 22 | ip: IP, 23 | }, 24 | }); 25 | global.__MONGO_MEMORY_REPLICA_SET = mongoMemoryServerReplicaSet; 26 | } catch (error) { 27 | console.error("Error in global setup:", error); 28 | throw error; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-ui/src/ActionButtonTrigger.tsx: -------------------------------------------------------------------------------- 1 | import { useDarkMode } from "@leafygreen-ui/leafygreen-provider"; 2 | import { ChatTrigger } from "@lg-chat/fixed-chat-window"; 3 | import { ChatbotTriggerProps } from "./ChatbotTrigger"; 4 | import { useChatbotContext } from "./useChatbotContext"; 5 | import { cx } from "@emotion/css"; 6 | 7 | export type ActionButtonTriggerProps = ChatbotTriggerProps & { 8 | text?: string; 9 | }; 10 | 11 | export function ActionButtonTrigger(props: ActionButtonTriggerProps) { 12 | const { darkMode } = useDarkMode(props.darkMode); 13 | const { chatbotName, openChat } = useChatbotContext(); 14 | const { className, text } = props; 15 | 16 | const buttonText = text ?? `Ask ${chatbotName ?? "the Chatbot"}`; 17 | 18 | return ( 19 | 24 | {buttonText} 25 | 26 | ); 27 | } 28 | -------------------------------------------------------------------------------- /packages/scripts/src/test/globalSetup.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | import { IP, REPLICA_SET_PORT, SERVER_PORT } from "./constants"; 3 | import "./globals"; 4 | 5 | export default async function () { 6 | try { 7 | const mongoMemoryServerInstance = await MongoMemoryServer.create({ 8 | instance: { 9 | port: SERVER_PORT, 10 | ip: IP, 11 | }, 12 | }); 13 | global.__MONGO_MEMORY_SERVER_INSTANCE = mongoMemoryServerInstance; 14 | 15 | const mongoMemoryServerReplicaSet = await MongoMemoryReplSet.create({ 16 | instanceOpts: [ 17 | { 18 | port: REPLICA_SET_PORT, 19 | }, 20 | ], 21 | replSet: { 22 | ip: IP, 23 | }, 24 | }); 25 | global.__MONGO_MEMORY_REPLICA_SET = mongoMemoryServerReplicaSet; 26 | } catch (error) { 27 | console.error("Error in global setup:", error); 28 | throw error; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/yaml.ts: -------------------------------------------------------------------------------- 1 | import { promises as fs } from "fs"; 2 | import path from "path"; 3 | import YAML from "yaml"; 4 | 5 | export async function readYamlDir( 6 | dir: string 7 | ): Promise<[string, Record][]> { 8 | try { 9 | const files = await fs.readdir(dir); 10 | const yamlFiles = files.filter( 11 | (file) => path.extname(file) === ".yml" || path.extname(file) === ".yaml" 12 | ); 13 | 14 | const yamlDocuments = await Promise.all( 15 | yamlFiles.map(async (file) => { 16 | const filePath = path.join(dir, file); 17 | const fileContents = await fs.readFile(filePath, "utf8"); 18 | const yaml = YAML.parse(fileContents); 19 | return [filePath, yaml] as [string, Record]; 20 | }) 21 | ); 22 | 23 | return yamlDocuments; 24 | } catch (error) { 25 | console.error("Error parsing YAML files:", error); 26 | return []; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/test/globalSetup.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | import { IP, REPLICA_SET_PORT, SERVER_PORT } from "./constants"; 3 | import "./globals"; 4 | 5 | export default async function () { 6 | try { 7 | const mongoMemoryServerInstance = await MongoMemoryServer.create({ 8 | instance: { 9 | port: SERVER_PORT, 10 | ip: IP, 11 | }, 12 | }); 13 | global.__MONGO_MEMORY_SERVER_INSTANCE = mongoMemoryServerInstance; 14 | 15 | const mongoMemoryServerReplicaSet = await MongoMemoryReplSet.create({ 16 | instanceOpts: [ 17 | { 18 | port: REPLICA_SET_PORT, 19 | }, 20 | ], 21 | replSet: { 22 | ip: IP, 23 | }, 24 | }); 25 | global.__MONGO_MEMORY_REPLICA_SET = mongoMemoryServerReplicaSet; 26 | } catch (error) { 27 | console.error("Error in global setup:", error); 28 | throw error; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/context/tdbx-content-types/troubleshooting.yaml: -------------------------------------------------------------------------------- 1 | type: troubleshooting 2 | name: Troubleshooting 3 | notes: 4 | - Currently an experimental type that overlaps objectives of the FAQ 5 | and the Fundamental types. Requested by Kafka Connector PM to 6 | present content that customers frequently ask about. 7 | objectives: 8 | - Provide detailed answers on how to resolve a common issue including 9 | implementation examples. 10 | pageStructure: | 11 | ======================= 12 | 13 | ======================= 14 | 15 | Overview 16 | -------- 17 | 18 | Describe the topic and provide information that helps the user identify whether their situation matches. 19 | 20 | Identify common causes 21 | 22 | Provide potential solutions 23 | 24 | Recommend preventative measures 25 | examples: 26 | - https://mongodb.com/docs/kafka-connector/current/troubleshooting/recover-from-invalid-resume-token/ 27 | -------------------------------------------------------------------------------- /packages/mongodb-rag-ingest/src/test/globalSetup.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | import { IP, REPLICA_SET_PORT, SERVER_PORT } from "./constants"; 3 | import "./globals"; 4 | 5 | export default async function () { 6 | try { 7 | const mongoMemoryServerInstance = await MongoMemoryServer.create({ 8 | instance: { 9 | port: SERVER_PORT, 10 | ip: IP, 11 | }, 12 | }); 13 | global.__MONGO_MEMORY_SERVER_INSTANCE = mongoMemoryServerInstance; 14 | 15 | const mongoMemoryServerReplicaSet = await MongoMemoryReplSet.create({ 16 | instanceOpts: [ 17 | { 18 | port: REPLICA_SET_PORT, 19 | }, 20 | ], 21 | replSet: { 22 | ip: IP, 23 | }, 24 | }); 25 | global.__MONGO_MEMORY_REPLICA_SET = mongoMemoryServerReplicaSet; 26 | } catch (error) { 27 | console.error("Error in global setup:", error); 28 | throw error; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/test/globalSetup.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | import { IP, REPLICA_SET_PORT, SERVER_PORT } from "./constants"; 3 | import "./globals"; 4 | 5 | export default async function () { 6 | try { 7 | const mongoMemoryServerInstance = await MongoMemoryServer.create({ 8 | instance: { 9 | port: SERVER_PORT, 10 | ip: IP, 11 | }, 12 | }); 13 | global.__MONGO_MEMORY_SERVER_INSTANCE = mongoMemoryServerInstance; 14 | 15 | const mongoMemoryServerReplicaSet = await MongoMemoryReplSet.create({ 16 | instanceOpts: [ 17 | { 18 | port: REPLICA_SET_PORT, 19 | }, 20 | ], 21 | replSet: { 22 | ip: IP, 23 | }, 24 | }); 25 | global.__MONGO_MEMORY_REPLICA_SET = mongoMemoryServerReplicaSet; 26 | } catch (error) { 27 | console.error("Error in global setup:", error); 28 | throw error; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/src/test/globalSetup.ts: -------------------------------------------------------------------------------- 1 | import { MongoMemoryServer, MongoMemoryReplSet } from "mongodb-memory-server"; 2 | import { IP, REPLICA_SET_PORT, SERVER_PORT } from "./constants"; 3 | import "./globals"; 4 | 5 | export default async function () { 6 | try { 7 | const mongoMemoryServerInstance = await MongoMemoryServer.create({ 8 | instance: { 9 | port: SERVER_PORT, 10 | ip: IP, 11 | }, 12 | }); 13 | global.__MONGO_MEMORY_SERVER_INSTANCE = mongoMemoryServerInstance; 14 | 15 | const mongoMemoryServerReplicaSet = await MongoMemoryReplSet.create({ 16 | instanceOpts: [ 17 | { 18 | port: REPLICA_SET_PORT, 19 | }, 20 | ], 21 | replSet: { 22 | ip: IP, 23 | }, 24 | }); 25 | global.__MONGO_MEMORY_REPLICA_SET = mongoMemoryServerReplicaSet; 26 | } catch (error) { 27 | console.error("Error in global setup:", error); 28 | throw error; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /packages/datasets/src/treeGeneration/mongoDbDatabases/README.md: -------------------------------------------------------------------------------- 1 | # Generate MongoDB Databases 2 | 3 | This directory contains the logic for generating MongoDB databases. 4 | 5 | The generation pipeline should use the following flow: 6 | 7 | 1. create a business + its use case 8 | - maybe we help out by providing sector or something in prompt 9 | - can get inspiration from https://www.mongodb.com/solutions/customer-case-studies or other docs. doing napkin math i see ~300 case studies 10 | 2. Generate schema + indexes 11 | - prompt includes condensed version of schema design docs to push for best practices 12 | 3. Generate zod types to validate collection document schemas 13 | - want to have this to guarantee consistency of any documents we input 14 | 4. have AI write script to input data into schema 15 | - i think this is the hardest part 16 | - i think using the Claude Code TypeScript SDK (https://docs.anthropic.com/en/docs/claude-code/sdk#typescript) could be promising here -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/environments/qa.yml: -------------------------------------------------------------------------------- 1 | service: 2 | targetPort: 3000 3 | 4 | env: 5 | PORT: 3000 6 | MONGODB_DATABASE_NAME: docs-chatbot-qa-2023-10-19 7 | VECTOR_SEARCH_INDEX_NAME: vector_index 8 | OPENAI_CHAT_COMPLETION_MODEL_VERSION: 2023-06-01-preview 9 | ENVIRONMENT: qa 10 | NODE_ENV: qa 11 | OPENAI_PREPROCESSOR_CHAT_COMPLETION_DEPLOYMENT: gpt-4o-mini 12 | OPENAI_API_VERSION: "2024-06-01" 13 | OPENAI_CHAT_COMPLETION_DEPLOYMENT: gpt-4o 14 | OPENAI_VERIFIED_ANSWER_EMBEDDING_DEPLOYMENT: "docs-chatbot-embedding-ada-002" 15 | OPENAI_RETRIEVAL_EMBEDDING_DEPLOYMENT: "text-embedding-3-small" 16 | 17 | envSecrets: 18 | MONGODB_CONNECTION_URI: docs-chatbot-staging 19 | OPENAI_ENDPOINT: docs-chatbot-staging 20 | OPENAI_API_KEY: docs-chatbot-staging 21 | 22 | ingress: 23 | enabled: true 24 | hosts: 25 | - chat-qa.docs.staging.corp.mongodb.com 26 | 27 | resources: 28 | limits: 29 | memory: 1300Mi 30 | requests: 31 | memory: 1100Mi 32 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/context/tdbx-content-types/faq.yaml: -------------------------------------------------------------------------------- 1 | type: faq 2 | name: FAQ 3 | objectives: 4 | - Show commonly encountered issues/questions and provide answers and 5 | guidance on troubleshooting, usually driven by SEO findings. Prefer 6 | explaining how to correctly implement and configure the driver to 7 | accomplish tasks and avoid errors in other sections of the 8 | documentation. 9 | pageStructure: | 10 | === 11 | FAQ 12 | === 13 | 14 | List frequently asked questions, each with its own header as shown below: 15 | 16 | 17 | ---------- 18 | 19 | Present one or more brief answers to the question, including any copy and paste code examples as well as links to internal or external documentation that provide more information. 20 | examples: 21 | - https://mongodb.com/docs/drivers/node/current/faq/ 22 | - https://mongodb.com/docs/drivers/go/current/faq/ 23 | - https://mongodb.com/docs/drivers/java/sync/current/faq/ 24 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/test/middlewareTestHelpers.ts: -------------------------------------------------------------------------------- 1 | import { Request } from "express"; 2 | import { ParamsDictionary } from "express-serve-static-core"; 3 | import { createRequest, createResponse } from "node-mocks-http"; 4 | import { ConversationsService } from "mongodb-rag-core"; 5 | import { ParsedQs } from "qs"; 6 | import { 7 | ConversationsRouterLocals, 8 | ConversationsRouterResponse, 9 | } from "../routes/conversations/conversationsRouter"; 10 | 11 | export const createConversationsMiddlewareReq = () => 12 | createRequest< 13 | Request< 14 | ParamsDictionary, 15 | unknown, 16 | unknown, 17 | ParsedQs, 18 | ConversationsRouterLocals 19 | > 20 | >(); 21 | 22 | export const createConversationsMiddlewareRes = () => { 23 | const res = createResponse(); 24 | res.locals = { 25 | conversations: {} as unknown as ConversationsService, 26 | customData: {}, 27 | }; 28 | return res; 29 | }; 30 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/dataSources/GitDataSource.test.ts: -------------------------------------------------------------------------------- 1 | import { makeGitDataSource } from "./GitDataSource"; 2 | jest.setTimeout(90000); 3 | describe("GitDataSource", () => { 4 | it("should load and process a real repo", async () => { 5 | const dataSource = makeGitDataSource({ 6 | name: "sample", 7 | repoUri: "https://github.com/mongodb/chatbot.git", 8 | repoOptions: { 9 | "--depth": 1, 10 | "--branch": "main", 11 | }, 12 | filter: (path: string) => 13 | path.endsWith(".md") && path.includes("docs/docs"), 14 | metadata: { 15 | foo: "bar", 16 | }, 17 | handlePage: async (path) => [ 18 | { 19 | url: "https://example.com/" + path, 20 | title: "sample", 21 | body: "sample", 22 | format: "md", 23 | }, 24 | ], 25 | }); 26 | const pages = await dataSource.fetchPages(); 27 | expect(pages.length).toBeGreaterThanOrEqual(1); 28 | }); 29 | }); 30 | -------------------------------------------------------------------------------- /packages/chatbot-server-mongodb-public/environments/dev-rel.yml: -------------------------------------------------------------------------------- 1 | service: 2 | targetPort: 3000 3 | 4 | env: 5 | PORT: 3000 6 | MONGODB_DATABASE_NAME: dev-rel-generator 7 | VECTOR_SEARCH_INDEX_NAME: vector_index 8 | OPENAI_CHAT_COMPLETION_MODEL_VERSION: 2023-06-01-preview 9 | ENVIRONMENT: qa 10 | NODE_ENV: qa 11 | OPENAI_PREPROCESSOR_CHAT_COMPLETION_DEPLOYMENT: gpt-4o-mini 12 | OPENAI_API_VERSION: "2024-06-01" 13 | OPENAI_CHAT_COMPLETION_DEPLOYMENT: gpt-4o 14 | OPENAI_VERIFIED_ANSWER_EMBEDDING_DEPLOYMENT: "docs-chatbot-embedding-ada-002" 15 | OPENAI_RETRIEVAL_EMBEDDING_DEPLOYMENT: "text-embedding-3-small" 16 | 17 | envSecrets: 18 | MONGODB_CONNECTION_URI: docs-chatbot-staging 19 | OPENAI_ENDPOINT: docs-chatbot-staging 20 | OPENAI_API_KEY: docs-chatbot-staging 21 | 22 | ingress: 23 | enabled: true 24 | hosts: 25 | - dev-rel-generator.docs.staging.corp.mongodb.com 26 | 27 | resources: 28 | limits: 29 | memory: 1300Mi 30 | requests: 31 | memory: 1100Mi 32 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/logger.ts: -------------------------------------------------------------------------------- 1 | import { createLogger, transports, format } from "winston"; 2 | 3 | // Helper function to format JSON message 4 | interface CreateMessageParams { 5 | message: string; 6 | requestBody?: unknown; 7 | requestId?: string; 8 | ipAddress?: string; 9 | } 10 | export const createLoggerMessage = ({ 11 | message, 12 | requestBody, 13 | requestId, 14 | ipAddress, 15 | }: CreateMessageParams) => { 16 | return { 17 | message, 18 | requestBody, 19 | requestId, 20 | ipAddress, 21 | }; 22 | }; 23 | 24 | /** 25 | Returns a reusable global logger instance. If no logger exists yet, then 26 | a new logger is created 27 | 28 | @returns logger 29 | */ 30 | const initiateLogger = () => { 31 | const logger = createLogger({ 32 | transports: [ 33 | new transports.Console({ 34 | format: format.json(), 35 | }), 36 | ], 37 | }); 38 | 39 | return logger; 40 | }; 41 | 42 | export const logger = initiateLogger(); 43 | -------------------------------------------------------------------------------- /packages/scripts/src/findCentroid.test.ts: -------------------------------------------------------------------------------- 1 | import { findCentroid } from "./findCentroid"; 2 | 3 | describe("findCentroid", () => { 4 | it("finds centroid (mean) of M n-dimensional vectors", () => { 5 | expect( 6 | findCentroid([ 7 | [0, 0], 8 | [1, 1], 9 | ]) 10 | ).toStrictEqual([0.5, 0.5]); 11 | 12 | expect( 13 | findCentroid([ 14 | [4, 0, 0], 15 | [0, 4, 4], 16 | [4, 0, 4], 17 | [0, 0, 4], 18 | ]) 19 | ).toStrictEqual([2, 1, 3]); 20 | 21 | expect( 22 | findCentroid([ 23 | [1, 1, 1], 24 | [1, 1, 1], 25 | [1, 1, 1], 26 | ]) 27 | ).toStrictEqual([1, 1, 1]); 28 | }); 29 | 30 | it("rejects mixed-dimensional vectors", () => { 31 | expect(() => { 32 | findCentroid([[1], [1, 1]]); 33 | }).toThrow(); 34 | }); 35 | 36 | it("rejects empty vector set", () => { 37 | expect(() => { 38 | findCentroid([]); 39 | }).toThrow(); 40 | }); 41 | }); 42 | -------------------------------------------------------------------------------- /packages/benchmarks/src/reporting/bin/reportTechSupportBenchmarkResults.ts: -------------------------------------------------------------------------------- 1 | import { reportBenchmarkResults } from "../reportBenchmarkResults"; 2 | import { strict as assert } from "assert"; 3 | import fs from "fs"; 4 | import path from "path"; 5 | import "dotenv/config"; 6 | import { BSON } from "mongodb-rag-core/mongodb"; 7 | const { EJSON } = BSON; 8 | 9 | async function main() { 10 | const pathOut = path.join("testData", "tech_support_results.json"); 11 | 12 | console.log(`Reporting tech support benchmark results to ${pathOut}`); 13 | 14 | const apiKey = process.env.BRAINTRUST_API_KEY; 15 | assert(apiKey, "must have BRAINTRUST_API_KEY set"); 16 | 17 | const projectName = "tech-support-prompt-completion"; 18 | 19 | const cases = await reportBenchmarkResults({ 20 | apiKey, 21 | projectName, 22 | experimentType: "prompt_response", 23 | }); 24 | console.log(`Reported ${cases.length} cases`); 25 | fs.writeFileSync(pathOut, EJSON.stringify(cases)); 26 | } 27 | 28 | main(); 29 | -------------------------------------------------------------------------------- /packages/datasets/src/bin/analyzeNlToMongoShDatasetBraintrust.ts: -------------------------------------------------------------------------------- 1 | import fs from "fs"; 2 | import path from "path"; 3 | import { 4 | DatabaseNlQueryDatasetEntryBraintrustSchema, 5 | DatabaseNlQueryDatasetEntryBraintrust, 6 | convertBraintrustDatabaseNlQueryDatasetEntryToFlat, 7 | } from "../treeGeneration/databaseNlQueries/DatabaseNlQueryDatasetEntry"; 8 | import { countAndLogUsage } from "../treeGeneration/databaseNlQueries/analyzeDataset"; 9 | 10 | async function main() { 11 | const dataOutDir = path.resolve(__dirname, "..", "..", "dataOut"); 12 | 13 | const dataset = fs.readFileSync( 14 | path.resolve(dataOutDir, "atlas_sample_dataset_manual_review.json"), 15 | "utf-8" 16 | ); 17 | const datasetObjs: DatabaseNlQueryDatasetEntryBraintrust[] = 18 | DatabaseNlQueryDatasetEntryBraintrustSchema.array().parse( 19 | JSON.parse(dataset) 20 | ); 21 | countAndLogUsage( 22 | datasetObjs.map(convertBraintrustDatabaseNlQueryDatasetEntryToFlat) 23 | ); 24 | } 25 | main(); 26 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/executeCode/DatabaseExecutionResult.ts: -------------------------------------------------------------------------------- 1 | import { Document } from "mongodb"; 2 | 3 | export interface ExecuteMongoDbQueryParams { 4 | query: string; 5 | databaseName: string; 6 | /** 7 | Required for some executors, but not others. 8 | */ 9 | collectionName?: string; 10 | } 11 | 12 | export type ExecuteMongoDbQuery = ( 13 | params: ExecuteMongoDbQueryParams 14 | ) => Promise; 15 | 16 | export interface DatabaseExecutionResult { 17 | /** 18 | The result of executing the generated driver code. 19 | `null` if no result was returned. 20 | */ 21 | result: Document | Document[] | number | null; 22 | /** 23 | An error message if an error occurred during execution. 24 | */ 25 | error?: { 26 | message: string; 27 | }; 28 | /** 29 | The time in milliseconds it took to execute the generated driver code. 30 | `null` if execution failed or not tracked. 31 | */ 32 | executionTimeMs: number | null; 33 | } 34 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | # Logs 4 | logs 5 | *.log 6 | npm-debug.log* 7 | yarn-debug.log* 8 | yarn-error.log* 9 | lerna-debug.log* 10 | 11 | # Diagnostic reports (https://nodejs.org/api/report.html) 12 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 13 | 14 | # Runtime data 15 | pids 16 | *.pid 17 | *.seed 18 | *.pid.lock 19 | 20 | # Dependency directories 21 | node_modules/ 22 | 23 | # TypeScript cache 24 | *.tsbuildinfo 25 | 26 | # Optional npm cache directory 27 | .npm 28 | 29 | # Optional eslint cache 30 | .eslintcache 31 | 32 | # Output of 'npm pack' 33 | *.tgz 34 | 35 | # Yarn Integrity file 36 | .yarn-integrity 37 | 38 | # dotenv environment variables file 39 | .env 40 | .env.test 41 | 42 | # yarn v2 43 | .yarn/cache 44 | .yarn/unplugged 45 | .yarn/build-state.yml 46 | .yarn/install-state.gz 47 | .pnp.* 48 | 49 | # TS build 50 | build/ 51 | 52 | # vscode 53 | .vscode/ 54 | 55 | # Generated Artifacts 56 | runlogs/* 57 | 58 | # Docs AI Pilot 59 | ai-pilot/* 60 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/context/tdbx-content-types/landing-page.yml: -------------------------------------------------------------------------------- 1 | type: landing-page 2 | name: Landing Page 3 | objectives: 4 | - Introduce the product and sections of the documentation. 5 | pageStructure: | 6 | ============== 7 | 8 | ============== 9 | 10 | Introduce the product and what it does. 11 | 12 |
13 | ----------------- 14 | 15 | Summarize the content the user can find in each section of this product’s documentation. 16 | 17 | 18 | --------------------------- 19 | 20 | For drivers that do not have a corresponding docs site on mongodb.com, provide instructions on how to get started using that driver. 21 | 22 | Learn 23 | ----- 24 | 25 | Link to the Developher Hub search results for the product. 26 | 27 | Link to the MongoDB University course, if one exists for the product. 28 | examples: 29 | - https://mongodb.com/docs/drivers/java/sync/current/ 30 | - https://mongodb.com/docs/drivers/c/ 31 | -------------------------------------------------------------------------------- /packages/benchmarks/src/reporting/convertTagStatsToFlatObject.ts: -------------------------------------------------------------------------------- 1 | import { TagStats } from "./aggregateScoresByTag"; 2 | 3 | /** 4 | Converts the tagStats Map to a flat object with headers formatted as .mean, .length, etc. 5 | This makes it easier to export the data to formats like CSV.flat object where keys are formatted as ".." 6 | */ 7 | export function convertTagStatsToFlatObject( 8 | tagStats: TagStats 9 | ): Record> { 10 | const result: Record> = {}; 11 | 12 | // Process each tag 13 | for (const [tag, scoreStats] of tagStats.entries()) { 14 | result[tag] = {}; 15 | 16 | // Process each score type (accuracy, relevance, etc.) 17 | for (const [scoreName, stats] of Object.entries(scoreStats)) { 18 | for (const [statName, value] of Object.entries(stats)) { 19 | result[tag][`${scoreName}.${statName}`] = value; 20 | } 21 | } 22 | } 23 | 24 | return result; 25 | } 26 | -------------------------------------------------------------------------------- /packages/mongodb-rag-core/src/frontMatter/removeFrontMatter.test.ts: -------------------------------------------------------------------------------- 1 | import { removeFrontMatter } from "./removeFrontMatter"; 2 | 3 | describe("removeFrontMatter", () => { 4 | test("removes front matter if it exists", () => { 5 | const contentWithFrontmatter = ` 6 | --- 7 | title: My Blog Post 8 | author: John Doe 9 | date: 2023-08-15 10 | --- 11 | 12 | This is the body of my blog post. Here you can write the actual content of the post.`; 13 | const contentWithoutFrontmatter = `This is the body of my blog post. Here you can write the actual content of the post.`; 14 | expect(removeFrontMatter(contentWithFrontmatter)).toBe( 15 | contentWithoutFrontmatter 16 | ); 17 | }); 18 | test("does not affect text without front matter", () => { 19 | const contentWithoutFrontmatter = `This is the body of my blog post. Here you can write the actual content of the post.`; 20 | expect(removeFrontMatter(contentWithoutFrontmatter)).toBe( 21 | contentWithoutFrontmatter 22 | ); 23 | }); 24 | }); 25 | -------------------------------------------------------------------------------- /packages/ingest-mongodb-public/src/test/jestSetUp.ts: -------------------------------------------------------------------------------- 1 | import { logger } from "mongodb-rag-core"; 2 | // silence logger for tests 3 | logger.transports.forEach((t) => (t.silent = true)); 4 | 5 | // Polyfill File API for Node.js environment if not available 6 | // This fixes issues with cheerio/undici in CI environments 7 | if (typeof globalThis.File === "undefined") { 8 | try { 9 | // Try to use Node.js built-in File from buffer module (available in Node 20+) 10 | const { File } = require("node:buffer"); 11 | (globalThis as any).File = File; 12 | } catch { 13 | // Fallback for older Node.js versions 14 | const { Blob } = require("node:buffer"); 15 | (globalThis as any).File = class File extends Blob { 16 | name: string; 17 | lastModified: number; 18 | 19 | constructor(bits: BlobPart[], name: string, options?: FilePropertyBag) { 20 | super(bits, options); 21 | this.name = name; 22 | this.lastModified = options?.lastModified ?? Date.now(); 23 | } 24 | }; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /packages/mongodb-artifact-generator/src/release-notes/jira.test.ts: -------------------------------------------------------------------------------- 1 | import "dotenv/config"; 2 | import { makeJiraReleaseArtifacts } from "./jira"; 3 | 4 | import { standardConfig } from "../standardConfig"; 5 | import { JiraIssueArtifact } from "./projects"; 6 | 7 | const jiraApi = standardConfig.jiraApi?.(); 8 | 9 | if (!jiraApi) { 10 | throw new Error( 11 | "githubApi is required. Make sure to define it in the config." 12 | ); 13 | } 14 | 15 | describe.skip("makeJiraReleaseArtifacts", () => { 16 | it("returns a handle that can fetch issues", async () => { 17 | const gitHubReleaseArtifacts = makeJiraReleaseArtifacts({ 18 | jiraApi, 19 | project: "CLOUDP", 20 | version: "atlascli-1.22.0", 21 | }); 22 | const issues = await gitHubReleaseArtifacts.getIssues(); 23 | expect(issues.length).toBeGreaterThan(0); 24 | const firstIssue = issues[0] as JiraIssueArtifact; 25 | expect(firstIssue.type).toEqual("jira-issue"); 26 | expect(firstIssue.data.key).toEqual("CLOUDP-247010"); 27 | }); 28 | }); 29 | -------------------------------------------------------------------------------- /packages/benchmarks/src/reporting/bin/reportMultipleChoiceBenchmarkResults.ts: -------------------------------------------------------------------------------- 1 | import { reportBenchmarkResults } from "../reportBenchmarkResults"; 2 | import { strict as assert } from "assert"; 3 | import fs from "fs"; 4 | import path from "path"; 5 | import "dotenv/config"; 6 | import { BSON } from "mongodb-rag-core/mongodb"; 7 | const { EJSON } = BSON; 8 | 9 | async function main() { 10 | const pathOut = path.join( 11 | __dirname, 12 | "testData", 13 | "multiple_choice_results.json" 14 | ); 15 | 16 | console.log(`Reporting multiple choice benchmark results to ${pathOut}`); 17 | 18 | const apiKey = process.env.BRAINTRUST_API_KEY; 19 | assert(apiKey, "must have BRAINTRUST_API_KEY set"); 20 | 21 | const projectName = "mongodb-multiple-choice"; 22 | 23 | const cases = await reportBenchmarkResults({ 24 | apiKey, 25 | projectName, 26 | experimentType: "multiple_choice", 27 | }); 28 | console.log(`Reported ${cases.length} cases`); 29 | fs.writeFileSync(pathOut, EJSON.stringify(cases)); 30 | } 31 | 32 | main(); 33 | -------------------------------------------------------------------------------- /packages/mongodb-chatbot-server/src/middleware/rateLimit.ts: -------------------------------------------------------------------------------- 1 | import { rateLimit, Options } from "express-rate-limit"; 2 | import { ipAddressKeyGenerator } from "./ipAddressKeyGenerator"; 3 | 4 | export const defaultRateLimitResponse = { 5 | error: "Too many requests, please try again later.", 6 | }; 7 | 8 | export type RateLimitOptions = Partial; 9 | 10 | export const defaultRateLimitOptions = { 11 | windowMs: 5 * 60 * 1000, // 5 minutes 12 | max: 500, // limit each IP to 500 requests per windowMs 13 | standardHeaders: "draft-7", // draft-6: RateLimit-* headers; draft-7: combined RateLimit header 14 | legacyHeaders: true, // X-RateLimit-* headers 15 | message: defaultRateLimitResponse, 16 | keyGenerator: ipAddressKeyGenerator, 17 | } satisfies RateLimitOptions; 18 | 19 | /** 20 | Rate limit middleware. Applies {@link defaultRateLimitOptions} by default. 21 | */ 22 | export function makeRateLimit(options?: RateLimitOptions) { 23 | return rateLimit({ 24 | ...defaultRateLimitOptions, 25 | ...options, 26 | }); 27 | } 28 | --------------------------------------------------------------------------------