├── .cache └── vector_store.json ├── .devcontainer └── devcontainer.json ├── .env.example ├── .eslintrc.json ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── ai-opsec-agent.yml │ ├── azure-dev.yml │ ├── build-test.yaml │ └── validate-infra.yaml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── SECURITY.md ├── SUPPORT.md ├── app ├── api │ ├── chat │ │ ├── config │ │ │ ├── llamacloud │ │ │ │ └── route.ts │ │ │ └── route.ts │ │ ├── engine │ │ │ ├── chat.ts │ │ │ ├── generate.ts │ │ │ ├── index.ts │ │ │ ├── loader.ts │ │ │ ├── provider.ts │ │ │ ├── queryFilter.ts │ │ │ ├── settings.ts │ │ │ └── tools │ │ │ │ ├── code-generator.ts │ │ │ │ ├── document-generator.ts │ │ │ │ ├── duckduckgo.ts │ │ │ │ ├── form-filling.ts │ │ │ │ ├── img-gen.ts │ │ │ │ ├── index.ts │ │ │ │ ├── interpreter.ts │ │ │ │ ├── openapi-action.ts │ │ │ │ ├── query-engine.ts │ │ │ │ ├── weather.ts │ │ │ │ └── wikipedia.ts │ │ ├── llamaindex │ │ │ ├── documents │ │ │ │ ├── helper.ts │ │ │ │ ├── pipeline.ts │ │ │ │ └── upload.ts │ │ │ └── streaming │ │ │ │ ├── annotations.ts │ │ │ │ ├── events.ts │ │ │ │ ├── file.ts │ │ │ │ └── suggestion.ts │ │ ├── route.ts │ │ └── upload │ │ │ └── route.ts │ ├── files │ │ └── [...slug] │ │ │ └── route.ts │ └── sandbox │ │ └── route.ts ├── components │ ├── chat-section.tsx │ ├── header.tsx │ └── ui │ │ ├── accordion.tsx │ │ ├── button.tsx │ │ ├── card.tsx │ │ ├── chat │ │ ├── chat-avatar.tsx │ │ ├── chat-input.tsx │ │ ├── chat-message-content.tsx │ │ ├── chat-messages.tsx │ │ ├── chat-starter.tsx │ │ ├── custom │ │ │ ├── deep-research-card.tsx │ │ │ ├── llama-cloud-selector.tsx │ │ │ └── markdown.tsx │ │ ├── hooks │ │ │ ├── use-config.ts │ │ │ └── use-copy-to-clipboard.tsx │ │ └── tools │ │ │ ├── artifact.tsx │ │ │ ├── chat-tools.tsx │ │ │ └── weather-card.tsx │ │ ├── collapsible.tsx │ │ ├── icons │ │ ├── docx.svg │ │ ├── pdf.svg │ │ ├── sheet.svg │ │ └── txt.svg │ │ ├── input.tsx │ │ ├── lib │ │ └── utils.ts │ │ ├── select.tsx │ │ ├── tabs.tsx │ │ └── textarea.tsx ├── favicon.ico ├── globals.css ├── layout.tsx ├── markdown.css ├── observability │ └── index.ts └── page.tsx ├── azure.yaml ├── cache ├── doc_store.json ├── index_store.json └── vector_store.json ├── data └── 101.pdf ├── docs ├── architecture-diagram-llama-index-javascript.png ├── azd-up.png ├── cost.md ├── llama-index-javascript.png └── readme.md ├── infra ├── abbreviations.json ├── main.bicep ├── main.parameters.json ├── modules │ └── fetch-container-image.bicep └── resources.bicep ├── next-steps.md ├── next.config.json ├── next.config.mjs ├── package-lock.json ├── package.json ├── postcss.config.js ├── prettier.config.js ├── public └── llama.png ├── tailwind.config.ts ├── tsconfig.json └── webpack.config.mjs /.cache/vector_store.json: -------------------------------------------------------------------------------- 1 | {"embeddingDict":{},"textIdToRefDocId":{},"metadataDict":{}} -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "image": "mcr.microsoft.com/vscode/devcontainers/typescript-node:dev-20-bullseye", 3 | "features": { 4 | "ghcr.io/devcontainers-contrib/features/turborepo-npm:1": {}, 5 | "ghcr.io/azure/azure-dev/azd:latest": {}, 6 | "ghcr.io/devcontainers/features/docker-in-docker:1": { 7 | "version": "20.10.23", 8 | "moby": "false", 9 | "dockerDashComposeVersion": "v2" 10 | }, 11 | "ghcr.io/devcontainers/features/azure-cli:1": { 12 | "version": "latest", 13 | "installBicep": true 14 | }, 15 | "ghcr.io/devcontainers/features/github-cli:1": { 16 | "version": "latest" 17 | }, 18 | "ghcr.io/devcontainers-contrib/features/typescript:2": {} 19 | }, 20 | "customizations": { 21 | "codespaces": { 22 | "openFiles": [ 23 | "README.md" 24 | ] 25 | }, 26 | "vscode": { 27 | "extensions": [ 28 | "ms-vscode.typescript-language-features", 29 | "esbenp.prettier-vscode", 30 | "ms-azuretools.vscode-bicep", 31 | "ms-azuretools.vscode-azurecontainerapps", 32 | "ms-azuretools.vscode-docker", 33 | "dbaeumer.vscode-eslint", 34 | "esbenp.prettier-vscode", 35 | "ms-azuretools.azure-dev", 36 | "GitHub.vscode-pull-request-github", 37 | "EditorConfig.EditorConfig", 38 | "GitHub.copilot" 39 | ] 40 | } 41 | }, 42 | "portsAttributes": { 43 | "3000": { 44 | "label": "Next.js", 45 | "onAutoForward": "notify" 46 | } 47 | }, 48 | "forwardPorts": [ 49 | 3000 50 | ], 51 | "postCreateCommand": "npm install" 52 | } -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | # The Llama Cloud API key. 2 | # LLAMA_CLOUD_API_KEY= 3 | 4 | # The provider for the AI models to use. 5 | MODEL_PROVIDER=ollama 6 | 7 | # The name of LLM model to use. 8 | MODEL=phi3:mini 9 | 10 | # Name of the embedding model to use. 11 | EMBEDDING_MODEL=nomic-embed-text 12 | 13 | # Dimension of the embedding model to use. 14 | EMBEDDING_DIM=1024 15 | 16 | # The OpenAI API key to use. 17 | # OPENAI_API_KEY= 18 | 19 | # Temperature for sampling from the model. 20 | # LLM_TEMPERATURE= 21 | 22 | # Maximum number of tokens to generate. 23 | # LLM_MAX_TOKENS= 24 | 25 | # The number of similar embeddings to return when retrieving documents. 26 | TOP_K=3 27 | 28 | # FILESERVER_URL_PREFIX is the URL prefix of the server storing the images generated by the interpreter. 29 | FILESERVER_URL_PREFIX=http://localhost:3000/api/files 30 | 31 | # The system prompt for the AI model. 32 | SYSTEM_PROMPT=You are a helpful assistant who helps users with their questions. 33 | 34 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["next/core-web-vitals", "prettier"], 3 | "rules": { 4 | "max-params": ["error", 4], 5 | "prefer-const": "error" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Purpose 2 | 3 | * ... 4 | 5 | ## Does this introduce a breaking change? 6 | 7 | ``` 8 | [ ] Yes 9 | [ ] No 10 | ``` 11 | 12 | ## Pull Request Type 13 | What kind of change does this Pull Request introduce? 14 | 15 | 16 | ``` 17 | [ ] Bugfix 18 | [ ] Feature 19 | [ ] Code style update (formatting, local variables) 20 | [ ] Refactoring (no functional changes, no api changes) 21 | [ ] Documentation content changes 22 | [ ] Other... Please describe: 23 | ``` 24 | 25 | ## How to Test 26 | * Get the code 27 | 28 | ``` 29 | git clone [repo-address] 30 | cd [repo-name] 31 | git checkout [branch-name] 32 | npm install 33 | ``` 34 | 35 | * Test the code 36 | 37 | ``` 38 | ``` 39 | 40 | ## What to Check 41 | Verify that the following are valid 42 | * ... 43 | 44 | ## Other Information 45 | -------------------------------------------------------------------------------- /.github/workflows/ai-opsec-agent.yml: -------------------------------------------------------------------------------- 1 | name: AI OpSec Agent 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | paths: 7 | - "**/*.py" 8 | - "**/*.js" 9 | - "**/*.ts" 10 | - "**/*.java" 11 | - "**/*.cs" 12 | - "**/*.bicep" 13 | - "**/*.yml" 14 | - "**/*.yaml" 15 | - "**/*.json" 16 | 17 | 18 | permissions: 19 | contents: read 20 | pull-requests: write # so we can comment on PRs 21 | 22 | jobs: 23 | audit: 24 | runs-on: ubuntu-latest 25 | name: "Audit Codebase" 26 | steps: 27 | - name: Checkout code 28 | uses: actions/checkout@v4 29 | - name: Audit Codebase 30 | uses: Azure-Samples/azure-ai-travel-agents/.github/actions/ai-opsec-agent@main 31 | with: 32 | AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} 33 | AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} 34 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 35 | output: security-audit.md 36 | 37 | - name: Upload as artifact 38 | uses: actions/upload-artifact@v4 39 | with: 40 | name: security-audit 41 | path: security-audit.md 42 | 43 | -------------------------------------------------------------------------------- /.github/workflows/azure-dev.yml: -------------------------------------------------------------------------------- 1 | name: Deploy to Azure 2 | on: 3 | workflow_dispatch: 4 | push: 5 | # Run when commits are pushed to mainline branch (main or master) 6 | # Set this to the mainline branch you are using 7 | branches: 8 | - main 9 | 10 | # GitHub Actions workflow to deploy to Azure using azd 11 | # To configure required secrets for connecting to Azure, simply run `azd pipeline config` 12 | 13 | # Set up permissions for deploying with secretless Azure federated credentials 14 | # https://learn.microsoft.com/en-us/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication 15 | permissions: 16 | id-token: write 17 | contents: read 18 | 19 | jobs: 20 | build: 21 | runs-on: ubuntu-latest 22 | env: 23 | AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} 24 | AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} 25 | AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} 26 | AZURE_ENV_NAME: ${{ vars.AZURE_ENV_NAME }} 27 | AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} 28 | steps: 29 | - name: Checkout 30 | uses: actions/checkout@v4 31 | 32 | - name: Setup Node.js v20 33 | uses: actions/setup-node@v4 34 | with: 35 | node-version: 20 36 | 37 | - name: Install azd 38 | uses: Azure/setup-azd@v2 39 | 40 | - name: Log in with Azure (Federated Credentials) 41 | if: ${{ env.AZURE_CLIENT_ID != '' }} 42 | run: | 43 | azd auth login ` 44 | --client-id "$Env:AZURE_CLIENT_ID" ` 45 | --federated-credential-provider "github" ` 46 | --tenant-id "$Env:AZURE_TENANT_ID" 47 | shell: pwsh 48 | 49 | - name: Log in with Azure (Client Credentials) 50 | if: ${{ env.AZURE_CREDENTIALS != '' }} 51 | run: | 52 | $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; 53 | Write-Host "::add-mask::$($info.clientSecret)" 54 | 55 | azd auth login ` 56 | --client-id "$($info.clientId)" ` 57 | --client-secret "$($info.clientSecret)" ` 58 | --tenant-id "$($info.tenantId)" 59 | shell: pwsh 60 | env: 61 | AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} 62 | 63 | - name: Provision Infrastructure 64 | run: azd provision --no-prompt 65 | env: 66 | AZURE_ENV_NAME: ${{ vars.AZURE_ENV_NAME }} 67 | AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} 68 | AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} 69 | 70 | - name: Deploy Application 71 | run: | 72 | npm ci 73 | azd deploy --no-prompt 74 | env: 75 | AZURE_ENV_NAME: ${{ vars.AZURE_ENV_NAME }} 76 | AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} 77 | AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} 78 | -------------------------------------------------------------------------------- /.github/workflows/build-test.yaml: -------------------------------------------------------------------------------- 1 | name: Build and test 2 | on: 3 | push: 4 | branches: [main] 5 | pull_request: 6 | branches: [main] 7 | 8 | jobs: 9 | build_test: 10 | strategy: 11 | matrix: 12 | platform: [ubuntu-latest, macos-latest, windows-latest] 13 | node-version: ['20'] 14 | 15 | name: ${{ matrix.platform }} / Node.js v${{ matrix.node-version }} 16 | runs-on: ${{ matrix.platform }} 17 | steps: 18 | - run: git config --global core.autocrlf false 19 | - uses: actions/checkout@v4 20 | - name: Setup Node.js v${{ matrix.node-version }} 21 | uses: actions/setup-node@v4 22 | with: 23 | node-version: ${{ matrix.node-version }} 24 | - name: Install dependencies 25 | run: npm cache verify && npm ci 26 | - name: Build packages 27 | run: npm run build 28 | - name: Test packages 29 | run: npm test --if-present || exit 0 30 | 31 | build_test_all: 32 | if: always() 33 | runs-on: ubuntu-latest 34 | needs: build_test 35 | steps: 36 | - name: Check build matrix status 37 | if: ${{ needs.build_test.result != 'success' }} 38 | run: exit 1 -------------------------------------------------------------------------------- /.github/workflows/validate-infra.yaml: -------------------------------------------------------------------------------- 1 | name: Validate AZD template 2 | on: 3 | workflow_dispatch: 4 | push: 5 | branches: [main] 6 | paths: 7 | - 'infra/**' 8 | pull_request: 9 | branches: [main] 10 | paths: 11 | - 'infra/**' 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout 18 | uses: actions/checkout@v4 19 | 20 | - name: Build Bicep for linting 21 | uses: azure/CLI@v2 22 | with: 23 | inlineScript: az config set bicep.use_binary_from_path=false && az bicep build -f infra/main.bicep --stdout 24 | 25 | - name: Run Microsoft Security DevOps Analysis 26 | uses: microsoft/security-devops-action@v1 27 | id: msdo 28 | continue-on-error: true 29 | with: 30 | tools: templateanalyzer 31 | 32 | - name: Upload alerts to Security tab 33 | if: github.repository_owner == 'Azure-Samples' 34 | uses: github/codeql-action/upload-sarif@v3 35 | with: 36 | sarif_file: ${{ steps.msdo.outputs.sarifFile }} 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # next.js 12 | /.next/ 13 | /out/ 14 | 15 | # production 16 | /build 17 | 18 | # misc 19 | .DS_Store 20 | *.pem 21 | 22 | # debug 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | 27 | # local env files 28 | .env*.local 29 | .env 30 | # vercel 31 | .vercel 32 | 33 | # typescript 34 | *.tsbuildinfo 35 | next-env.d.ts 36 | 37 | tool-output/ 38 | .azure -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to this project 2 | 3 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 4 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 5 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 6 | 7 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 8 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 9 | provided by the bot. You will only need to do this once across all repos using our CLA. 10 | 11 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 12 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 13 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 14 | 15 | - [Code of Conduct](#coc) 16 | - [Issues and Bugs](#issue) 17 | - [Feature Requests](#feature) 18 | - [Submission Guidelines](#submit) 19 | 20 | ## Code of Conduct 21 | Help us keep this project open and inclusive. Please read and follow our [Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 22 | 23 | ## Found an Issue? 24 | If you find a bug in the source code or a mistake in the documentation, you can help us by 25 | [submitting an issue](#submit-issue) to the GitHub Repository. Even better, you can 26 | [submit a Pull Request](#submit-pr) with a fix. 27 | 28 | ## Want a Feature? 29 | You can *request* a new feature by [submitting an issue](#submit-issue) to the GitHub 30 | Repository. If you would like to *implement* a new feature, please submit an issue with 31 | a proposal for your work first, to be sure that we can use it. 32 | 33 | * **Small Features** can be crafted and directly [submitted as a Pull Request](#submit-pr). 34 | 35 | ## Submission Guidelines 36 | 37 | ### Submitting an Issue 38 | Before you submit an issue, search the archive, maybe your question was already answered. 39 | 40 | If your issue appears to be a bug, and hasn't been reported, open a new issue. 41 | Help us to maximize the effort we can spend fixing issues and adding new 42 | features, by not reporting duplicate issues. Providing the following information will increase the 43 | chances of your issue being dealt with quickly: 44 | 45 | * **Overview of the Issue** - if an error is being thrown a non-minified stack trace helps 46 | * **Version** - what version is affected (e.g. 0.1.2) 47 | * **Motivation for or Use Case** - explain what are you trying to do and why the current behavior is a bug for you 48 | * **Browsers and Operating System** - is this a problem with all browsers? 49 | * **Reproduce the Error** - provide a live example or a unambiguous set of steps 50 | * **Related Issues** - has a similar issue been reported before? 51 | * **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be 52 | causing the problem (line of code or commit) 53 | 54 | You can file new issues by providing the above information at the corresponding repository's issues link: https://github.com/[organization-name]/[repository-name]/issues/new]. 55 | 56 | ### Submitting a Pull Request (PR) 57 | Before you submit your Pull Request (PR) consider the following guidelines: 58 | 59 | * Search the repository (https://github.com/[organization-name]/[repository-name]/pulls) for an open or closed PR 60 | that relates to your submission. You don't want to duplicate effort. 61 | 62 | * Make your changes in a new git fork: 63 | 64 | * Commit your changes using a descriptive commit message 65 | * Push your fork to GitHub: 66 | * In GitHub, create a pull request 67 | * If we suggest changes then: 68 | * Make the required updates. 69 | * Rebase your fork and force push to your GitHub repository (this will update your Pull Request): 70 | 71 | ```shell 72 | git rebase master -i 73 | git push -f 74 | ``` 75 | 76 | That's it! Thank you for your contribution! 77 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:20-alpine as build 2 | 3 | WORKDIR /app 4 | 5 | COPY package.json package-lock.* ./ 6 | RUN npm install 7 | 8 | # Build the application 9 | COPY . . 10 | RUN npm run build 11 | 12 | # ==================================== 13 | FROM build as release 14 | 15 | CMD ["npm", "run", "start"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | - Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | - Full paths of source file(s) related to the manifestation of the issue 23 | - The location of the affected source code (tag/branch/commit or direct URL) 24 | - Any special configuration required to reproduce the issue 25 | - Step-by-step instructions to reproduce the issue 26 | - Proof-of-concept or exploit code (if possible) 27 | - Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # Support 2 | 3 | ## How to file issues and get help 4 | 5 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 6 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 7 | feature request as a new Issue. 8 | 9 | For help and questions about using this project, please use GitHub Issues and tag them with the 10 | **question** label. 11 | 12 | ## Microsoft Support Policy 13 | 14 | Support for this **PROJECT or PRODUCT** is limited to the resources listed above. -------------------------------------------------------------------------------- /app/api/chat/config/llamacloud/route.ts: -------------------------------------------------------------------------------- 1 | import { LLamaCloudFileService } from "llamaindex"; 2 | import { NextResponse } from "next/server"; 3 | 4 | /** 5 | * This API is to get config from the backend envs and expose them to the frontend 6 | */ 7 | export async function GET() { 8 | if (!process.env.LLAMA_CLOUD_API_KEY) { 9 | return NextResponse.json( 10 | { 11 | error: "env variable LLAMA_CLOUD_API_KEY is required to use LlamaCloud", 12 | }, 13 | { status: 500 }, 14 | ); 15 | } 16 | const config = { 17 | projects: await LLamaCloudFileService.getAllProjectsWithPipelines(), 18 | pipeline: { 19 | pipeline: process.env.LLAMA_CLOUD_INDEX_NAME, 20 | project: process.env.LLAMA_CLOUD_PROJECT_NAME, 21 | }, 22 | }; 23 | return NextResponse.json(config, { status: 200 }); 24 | } 25 | -------------------------------------------------------------------------------- /app/api/chat/config/route.ts: -------------------------------------------------------------------------------- 1 | import { NextResponse } from "next/server"; 2 | 3 | /** 4 | * This API is to get config from the backend envs and expose them to the frontend 5 | */ 6 | export async function GET() { 7 | const config = { 8 | starterQuestions: process.env.CONVERSATION_STARTERS?.trim().split("\n"), 9 | }; 10 | return NextResponse.json(config, { status: 200 }); 11 | } 12 | -------------------------------------------------------------------------------- /app/api/chat/engine/chat.ts: -------------------------------------------------------------------------------- 1 | import { BaseChatEngine, BaseToolWithCall, LLMAgent } from "llamaindex"; 2 | import fs from "node:fs/promises"; 3 | import path from "node:path"; 4 | import { getDataSource } from "./index"; 5 | import { createTools } from "./tools"; 6 | import { createQueryEngineTool } from "./tools/query-engine"; 7 | 8 | export async function createChatEngine(documentIds?: string[], params?: any) { 9 | const tools: BaseToolWithCall[] = []; 10 | 11 | // Add a query engine tool if we have a data source 12 | // Delete this code if you don't have a data source 13 | const index = await getDataSource(params); 14 | if (index) { 15 | tools.push(createQueryEngineTool(index, { documentIds })); 16 | } 17 | 18 | const configFile = path.join("config", "tools.json"); 19 | let toolConfig: any; 20 | try { 21 | // add tools from config file if it exists 22 | toolConfig = JSON.parse(await fs.readFile(configFile, "utf8")); 23 | } catch (e) { 24 | console.info(`Could not read ${configFile} file. Using no tools.`); 25 | } 26 | if (toolConfig) { 27 | tools.push(...(await createTools(toolConfig))); 28 | } 29 | 30 | const agent = new LLMAgent({ 31 | tools, 32 | systemPrompt: process.env.SYSTEM_PROMPT, 33 | }) as unknown as BaseChatEngine; 34 | 35 | return agent; 36 | } 37 | -------------------------------------------------------------------------------- /app/api/chat/engine/generate.ts: -------------------------------------------------------------------------------- 1 | import { VectorStoreIndex } from "llamaindex"; 2 | import { storageContextFromDefaults } from "llamaindex/storage/StorageContext"; 3 | 4 | import * as dotenv from "dotenv"; 5 | 6 | import { getDocuments } from "./loader"; 7 | import { initSettings } from "./settings"; 8 | 9 | // Load environment variables from local .env file 10 | dotenv.config(); 11 | 12 | async function getRuntime(func: any) { 13 | const start = Date.now(); 14 | await func(); 15 | const end = Date.now(); 16 | return end - start; 17 | } 18 | 19 | async function generateDatasource() { 20 | console.log(`Generating storage context...`); 21 | // Split documents, create embeddings and store them in the storage context 22 | const persistDir = process.env.STORAGE_CACHE_DIR; 23 | if (!persistDir) { 24 | throw new Error("STORAGE_CACHE_DIR environment variable is required!"); 25 | } 26 | const ms = await getRuntime(async () => { 27 | const storageContext = await storageContextFromDefaults({ 28 | persistDir, 29 | }); 30 | const documents = await getDocuments(); 31 | 32 | await VectorStoreIndex.fromDocuments(documents, { 33 | storageContext, 34 | }); 35 | }); 36 | console.log(`Storage context successfully generated in ${ms / 1000}s.`); 37 | } 38 | 39 | (async () => { 40 | initSettings(); 41 | await generateDatasource(); 42 | console.log("Finished generating storage."); 43 | })(); 44 | -------------------------------------------------------------------------------- /app/api/chat/engine/index.ts: -------------------------------------------------------------------------------- 1 | import { SimpleDocumentStore, VectorStoreIndex } from "llamaindex"; 2 | import { storageContextFromDefaults } from "llamaindex/storage/StorageContext"; 3 | 4 | export async function getDataSource(params?: any) { 5 | const persistDir = process.env.STORAGE_CACHE_DIR; 6 | if (!persistDir) { 7 | throw new Error("STORAGE_CACHE_DIR environment variable is required!"); 8 | } 9 | const storageContext = await storageContextFromDefaults({ 10 | persistDir, 11 | }); 12 | 13 | const numberOfDocs = Object.keys( 14 | (storageContext.docStore as SimpleDocumentStore).toDict(), 15 | ).length; 16 | if (numberOfDocs === 0) { 17 | return null; 18 | } 19 | return await VectorStoreIndex.init({ 20 | storageContext, 21 | }); 22 | } 23 | -------------------------------------------------------------------------------- /app/api/chat/engine/loader.ts: -------------------------------------------------------------------------------- 1 | import { 2 | FILE_EXT_TO_READER, 3 | SimpleDirectoryReader, 4 | } from "@llamaindex/readers/directory"; 5 | 6 | export const DATA_DIR = "./data"; 7 | 8 | export function getExtractors() { 9 | return FILE_EXT_TO_READER; 10 | } 11 | 12 | export async function getDocuments() { 13 | const documents = await new SimpleDirectoryReader().loadData({ 14 | directoryPath: DATA_DIR, 15 | }); 16 | // Set private=false to mark the document as public (required for filtering) 17 | for (const document of documents) { 18 | document.metadata = { 19 | ...document.metadata, 20 | private: "false", 21 | }; 22 | } 23 | return documents; 24 | } 25 | -------------------------------------------------------------------------------- /app/api/chat/engine/provider.ts: -------------------------------------------------------------------------------- 1 | import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai"; 2 | import { Settings } from "llamaindex"; 3 | import { 4 | DefaultAzureCredential, 5 | getBearerTokenProvider, 6 | } from "@azure/identity"; 7 | 8 | const AZURE_COGNITIVE_SERVICES_SCOPE = 9 | "https://cognitiveservices.azure.com/.default"; 10 | 11 | export function setupProvider() { 12 | const credential = new DefaultAzureCredential(); 13 | const azureADTokenProvider = getBearerTokenProvider( 14 | credential, 15 | AZURE_COGNITIVE_SERVICES_SCOPE, 16 | ); 17 | 18 | const azure = { 19 | azureADTokenProvider, 20 | deployment: process.env.AZURE_DEPLOYMENT_NAME ?? "gpt-35-turbo", 21 | }; 22 | 23 | // configure LLM model 24 | Settings.llm = new OpenAI({ 25 | azure, 26 | }) as any; 27 | 28 | // configure embedding model 29 | azure.deployment = process.env.EMBEDDING_MODEL as string; 30 | Settings.embedModel = new OpenAIEmbedding({ 31 | azure, 32 | model: process.env.EMBEDDING_MODEL, 33 | dimensions: process.env.EMBEDDING_DIM 34 | ? parseInt(process.env.EMBEDDING_DIM) 35 | : undefined, 36 | }); 37 | } 38 | -------------------------------------------------------------------------------- /app/api/chat/engine/queryFilter.ts: -------------------------------------------------------------------------------- 1 | import { MetadataFilter, MetadataFilters } from "llamaindex"; 2 | 3 | export function generateFilters(documentIds: string[]): MetadataFilters { 4 | // filter all documents have the private metadata key set to true 5 | const publicDocumentsFilter: MetadataFilter = { 6 | key: "private", 7 | value: "true", 8 | operator: "!=", 9 | }; 10 | 11 | // if no documentIds are provided, only retrieve information from public documents 12 | if (!documentIds.length) return { filters: [publicDocumentsFilter] }; 13 | 14 | const privateDocumentsFilter: MetadataFilter = { 15 | key: "doc_id", 16 | value: documentIds, 17 | operator: "in", 18 | }; 19 | 20 | // if documentIds are provided, retrieve information from public and private documents 21 | return { 22 | filters: [publicDocumentsFilter, privateDocumentsFilter], 23 | condition: "or", 24 | }; 25 | } 26 | -------------------------------------------------------------------------------- /app/api/chat/engine/settings.ts: -------------------------------------------------------------------------------- 1 | import { Settings } from "llamaindex"; 2 | import { setupProvider } from "./provider"; 3 | 4 | const CHUNK_SIZE = 512; 5 | const CHUNK_OVERLAP = 20; 6 | 7 | export const initSettings = async () => { 8 | console.log(`Using '${process.env.MODEL_PROVIDER}' model provider`); 9 | 10 | if (!process.env.MODEL || !process.env.EMBEDDING_MODEL) { 11 | throw new Error("'MODEL' and 'EMBEDDING_MODEL' env variables must be set."); 12 | } 13 | 14 | Settings.chunkSize = CHUNK_SIZE; 15 | Settings.chunkOverlap = CHUNK_OVERLAP; 16 | 17 | setupProvider(); 18 | }; 19 | -------------------------------------------------------------------------------- /app/api/chat/engine/tools/code-generator.ts: -------------------------------------------------------------------------------- 1 | import type { JSONSchemaType } from "ajv"; 2 | import { 3 | BaseTool, 4 | ChatMessage, 5 | JSONValue, 6 | Settings, 7 | ToolMetadata, 8 | } from "llamaindex"; 9 | 10 | // prompt based on https://github.com/e2b-dev/ai-artifacts 11 | const CODE_GENERATION_PROMPT = `You are a skilled software engineer. You do not make mistakes. Generate an artifact. You can install additional dependencies. You can use one of the following templates:\n 12 | 13 | 1. code-interpreter-multilang: "Runs code as a Jupyter notebook cell. Strong data analysis angle. Can use complex visualisation to explain results.". File: script.py. Dependencies installed: python, jupyter, numpy, pandas, matplotlib, seaborn, plotly. Port: none. 14 | 15 | 2. nextjs-developer: "A Next.js 13+ app that reloads automatically. Using the pages router.". File: pages/index.tsx. Dependencies installed: nextjs@14.2.5, typescript, @types/node, @types/react, @types/react-dom, postcss, tailwindcss, shadcn. Port: 3000. 16 | 17 | 3. vue-developer: "A Vue.js 3+ app that reloads automatically. Only when asked specifically for a Vue app.". File: app.vue. Dependencies installed: vue@latest, nuxt@3.13.0, tailwindcss. Port: 3000. 18 | 19 | 4. streamlit-developer: "A streamlit app that reloads automatically.". File: app.py. Dependencies installed: streamlit, pandas, numpy, matplotlib, request, seaborn, plotly. Port: 8501. 20 | 21 | 5. gradio-developer: "A gradio app. Gradio Blocks/Interface should be called demo.". File: app.py. Dependencies installed: gradio, pandas, numpy, matplotlib, request, seaborn, plotly. Port: 7860. 22 | 23 | Provide detail information about the artifact you're about to generate in the following JSON format with the following keys: 24 | 25 | commentary: Describe what you're about to do and the steps you want to take for generating the artifact in great detail. 26 | template: Name of the template used to generate the artifact. 27 | title: Short title of the artifact. Max 3 words. 28 | description: Short description of the artifact. Max 1 sentence. 29 | additional_dependencies: Additional dependencies required by the artifact. Do not include dependencies that are already included in the template. 30 | has_additional_dependencies: Detect if additional dependencies that are not included in the template are required by the artifact. 31 | install_dependencies_command: Command to install additional dependencies required by the artifact. 32 | port: Port number used by the resulted artifact. Null when no ports are exposed. 33 | file_path: Relative path to the file, including the file name. 34 | code: Code generated by the artifact. Only runnable code is allowed. 35 | 36 | Make sure to use the correct syntax for the programming language you're using. Make sure to generate only one code file. If you need to use CSS, make sure to include the CSS in the code file using Tailwind CSS syntax. 37 | `; 38 | 39 | // detail information to execute code 40 | export type CodeArtifact = { 41 | commentary: string; 42 | template: string; 43 | title: string; 44 | description: string; 45 | additional_dependencies: string[]; 46 | has_additional_dependencies: boolean; 47 | install_dependencies_command: string; 48 | port: number | null; 49 | file_path: string; 50 | code: string; 51 | files?: string[]; 52 | }; 53 | 54 | export type CodeGeneratorParameter = { 55 | requirement: string; 56 | oldCode?: string; 57 | sandboxFiles?: string[]; 58 | }; 59 | 60 | export type CodeGeneratorToolParams = { 61 | metadata?: ToolMetadata>; 62 | }; 63 | 64 | const DEFAULT_META_DATA: ToolMetadata> = 65 | { 66 | name: "artifact", 67 | description: `Generate a code artifact based on the input. Don't call this tool if the user has not asked for code generation. E.g. if the user asks to write a description or specification, don't call this tool.`, 68 | parameters: { 69 | type: "object", 70 | properties: { 71 | requirement: { 72 | type: "string", 73 | description: "The description of the application you want to build.", 74 | }, 75 | oldCode: { 76 | type: "string", 77 | description: "The existing code to be modified", 78 | nullable: true, 79 | }, 80 | sandboxFiles: { 81 | type: "array", 82 | description: 83 | "A list of sandbox file paths. Include these files if the code requires them.", 84 | items: { 85 | type: "string", 86 | }, 87 | nullable: true, 88 | }, 89 | }, 90 | required: ["requirement"], 91 | }, 92 | }; 93 | 94 | export class CodeGeneratorTool implements BaseTool { 95 | metadata: ToolMetadata>; 96 | 97 | constructor(params?: CodeGeneratorToolParams) { 98 | this.metadata = params?.metadata || DEFAULT_META_DATA; 99 | } 100 | 101 | async call(input: CodeGeneratorParameter) { 102 | try { 103 | const artifact = await this.generateArtifact( 104 | input.requirement, 105 | input.oldCode, 106 | input.sandboxFiles, // help the generated code use exact files 107 | ); 108 | if (input.sandboxFiles) { 109 | artifact.files = input.sandboxFiles; 110 | } 111 | return artifact as JSONValue; 112 | } catch (error) { 113 | return { isError: true }; 114 | } 115 | } 116 | 117 | // Generate artifact (code, environment, dependencies, etc.) 118 | async generateArtifact( 119 | query: string, 120 | oldCode?: string, 121 | attachments?: string[], 122 | ): Promise { 123 | const userMessage = ` 124 | ${query} 125 | ${oldCode ? `The existing code is: \n\`\`\`${oldCode}\`\`\`` : ""} 126 | ${attachments ? `The attachments are: \n${attachments.join("\n")}` : ""} 127 | `; 128 | const messages: ChatMessage[] = [ 129 | { role: "system", content: CODE_GENERATION_PROMPT }, 130 | { role: "user", content: userMessage }, 131 | ]; 132 | try { 133 | const response = await Settings.llm.chat({ messages }); 134 | const content = response.message.content.toString(); 135 | const jsonContent = content 136 | .replace(/^```json\s*|\s*```$/g, "") 137 | .replace(/^`+|`+$/g, "") 138 | .trim(); 139 | const artifact = JSON.parse(jsonContent) as CodeArtifact; 140 | return artifact; 141 | } catch (error) { 142 | console.log("Failed to generate artifact", error); 143 | throw error; 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /app/api/chat/engine/tools/document-generator.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchemaType } from "ajv"; 2 | import { BaseTool, ToolMetadata } from "llamaindex"; 3 | import { marked } from "marked"; 4 | import path from "node:path"; 5 | import { saveDocument } from "../../llamaindex/documents/helper"; 6 | 7 | const OUTPUT_DIR = "output/tools"; 8 | 9 | type DocumentParameter = { 10 | originalContent: string; 11 | fileName: string; 12 | }; 13 | 14 | const DEFAULT_METADATA: ToolMetadata> = { 15 | name: "document_generator", 16 | description: 17 | "Generate HTML document from markdown content. Return a file url to the document", 18 | parameters: { 19 | type: "object", 20 | properties: { 21 | originalContent: { 22 | type: "string", 23 | description: "The original markdown content to convert.", 24 | }, 25 | fileName: { 26 | type: "string", 27 | description: "The name of the document file (without extension).", 28 | }, 29 | }, 30 | required: ["originalContent", "fileName"], 31 | }, 32 | }; 33 | 34 | const COMMON_STYLES = ` 35 | body { 36 | font-family: Arial, sans-serif; 37 | line-height: 1.3; 38 | color: #333; 39 | } 40 | h1, h2, h3, h4, h5, h6 { 41 | margin-top: 1em; 42 | margin-bottom: 0.5em; 43 | } 44 | p { 45 | margin-bottom: 0.7em; 46 | } 47 | code { 48 | background-color: #f4f4f4; 49 | padding: 2px 4px; 50 | border-radius: 4px; 51 | } 52 | pre { 53 | background-color: #f4f4f4; 54 | padding: 10px; 55 | border-radius: 4px; 56 | overflow-x: auto; 57 | } 58 | table { 59 | border-collapse: collapse; 60 | width: 100%; 61 | margin-bottom: 1em; 62 | } 63 | th, td { 64 | border: 1px solid #ddd; 65 | padding: 8px; 66 | text-align: left; 67 | } 68 | th { 69 | background-color: #f2f2f2; 70 | font-weight: bold; 71 | } 72 | img { 73 | max-width: 90%; 74 | height: auto; 75 | display: block; 76 | margin: 1em auto; 77 | border-radius: 10px; 78 | } 79 | `; 80 | 81 | const HTML_SPECIFIC_STYLES = ` 82 | body { 83 | max-width: 800px; 84 | margin: 0 auto; 85 | padding: 20px; 86 | } 87 | `; 88 | 89 | const HTML_TEMPLATE = ` 90 | 91 | 92 | 93 | 94 | 95 | 99 | 100 | 101 | {{content}} 102 | 103 | 104 | `; 105 | 106 | export interface DocumentGeneratorParams { 107 | metadata?: ToolMetadata>; 108 | } 109 | 110 | export class DocumentGenerator implements BaseTool { 111 | metadata: ToolMetadata>; 112 | 113 | constructor(params: DocumentGeneratorParams) { 114 | this.metadata = params.metadata ?? DEFAULT_METADATA; 115 | } 116 | 117 | private static async generateHtmlContent( 118 | originalContent: string, 119 | ): Promise { 120 | return await marked(originalContent); 121 | } 122 | 123 | private static generateHtmlDocument(htmlContent: string): string { 124 | return HTML_TEMPLATE.replace("{{content}}", htmlContent); 125 | } 126 | 127 | async call(input: DocumentParameter): Promise { 128 | const { originalContent, fileName } = input; 129 | 130 | const htmlContent = 131 | await DocumentGenerator.generateHtmlContent(originalContent); 132 | const fileContent = DocumentGenerator.generateHtmlDocument(htmlContent); 133 | 134 | const filePath = path.join(OUTPUT_DIR, `${fileName}.html`); 135 | 136 | return `URL: ${await saveDocument(filePath, fileContent)}`; 137 | } 138 | } 139 | 140 | export function getTools(): BaseTool[] { 141 | return [new DocumentGenerator({})]; 142 | } 143 | -------------------------------------------------------------------------------- /app/api/chat/engine/tools/duckduckgo.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchemaType } from "ajv"; 2 | import { search } from "duck-duck-scrape"; 3 | import { BaseTool, ToolMetadata } from "llamaindex"; 4 | 5 | export type DuckDuckGoParameter = { 6 | query: string; 7 | region?: string; 8 | maxResults?: number; 9 | }; 10 | 11 | export type DuckDuckGoToolParams = { 12 | metadata?: ToolMetadata>; 13 | }; 14 | 15 | const DEFAULT_SEARCH_METADATA: ToolMetadata< 16 | JSONSchemaType 17 | > = { 18 | name: "duckduckgo_search", 19 | description: 20 | "Use this function to search for information (only text) in the internet using DuckDuckGo.", 21 | parameters: { 22 | type: "object", 23 | properties: { 24 | query: { 25 | type: "string", 26 | description: "The query to search in DuckDuckGo.", 27 | }, 28 | region: { 29 | type: "string", 30 | description: 31 | "Optional, The region to be used for the search in [country-language] convention, ex us-en, uk-en, ru-ru, etc...", 32 | nullable: true, 33 | }, 34 | maxResults: { 35 | type: "number", 36 | description: 37 | "Optional, The maximum number of results to be returned. Default is 10.", 38 | nullable: true, 39 | }, 40 | }, 41 | required: ["query"], 42 | }, 43 | }; 44 | 45 | type DuckDuckGoSearchResult = { 46 | title: string; 47 | description: string; 48 | url: string; 49 | }; 50 | 51 | export class DuckDuckGoSearchTool implements BaseTool { 52 | metadata: ToolMetadata>; 53 | 54 | constructor(params: DuckDuckGoToolParams) { 55 | this.metadata = params.metadata ?? DEFAULT_SEARCH_METADATA; 56 | } 57 | 58 | async call(input: DuckDuckGoParameter) { 59 | const { query, region, maxResults = 10 } = input; 60 | const options = region ? { region } : {}; 61 | // Temporarily sleep to reduce overloading the DuckDuckGo 62 | await new Promise((resolve) => setTimeout(resolve, 1000)); 63 | 64 | const searchResults = await search(query, options); 65 | 66 | return searchResults.results.slice(0, maxResults).map((result) => { 67 | return { 68 | title: result.title, 69 | description: result.description, 70 | url: result.url, 71 | } as DuckDuckGoSearchResult; 72 | }); 73 | } 74 | } 75 | 76 | export function getTools() { 77 | return [new DuckDuckGoSearchTool({})]; 78 | } 79 | -------------------------------------------------------------------------------- /app/api/chat/engine/tools/form-filling.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchemaType } from "ajv"; 2 | import fs from "fs"; 3 | import { BaseTool, Settings, ToolMetadata } from "llamaindex"; 4 | import Papa from "papaparse"; 5 | import path from "path"; 6 | import { saveDocument } from "../../llamaindex/documents/helper"; 7 | 8 | type ExtractMissingCellsParameter = { 9 | filePath: string; 10 | }; 11 | 12 | export type MissingCell = { 13 | rowIndex: number; 14 | columnIndex: number; 15 | question: string; 16 | }; 17 | 18 | const CSV_EXTRACTION_PROMPT = `You are a data analyst. You are given a table with missing cells. 19 | Your task is to identify the missing cells and the questions needed to fill them. 20 | IMPORTANT: Column indices should be 0-based 21 | 22 | # Instructions: 23 | - Understand the entire content of the table and the topics of the table. 24 | - Identify the missing cells and the meaning of the data in the cells. 25 | - For each missing cell, provide the row index and the correct column index (remember: first data column is 1). 26 | - For each missing cell, provide the question needed to fill the cell (it's important to provide the question that is relevant to the topic of the table). 27 | - Since the cell's value should be concise, the question should request a numerical answer or a specific value. 28 | - Finally, only return the answer in JSON format with the following schema: 29 | { 30 | "missing_cells": [ 31 | { 32 | "rowIndex": number, 33 | "columnIndex": number, 34 | "question": string 35 | } 36 | ] 37 | } 38 | - If there are no missing cells, return an empty array. 39 | - The answer is only the JSON object, nothing else and don't wrap it inside markdown code block. 40 | 41 | # Example: 42 | # | | Name | Age | City | 43 | # |----|------|-----|------| 44 | # | 0 | John | | Paris| 45 | # | 1 | Mary | | | 46 | # | 2 | | 30 | | 47 | # 48 | # Your thoughts: 49 | # - The table is about people's names, ages, and cities. 50 | # - Row: 1, Column: 2 (Age column), Question: "How old is Mary? Please provide only the numerical answer." 51 | # - Row: 1, Column: 3 (City column), Question: "In which city does Mary live? Please provide only the city name." 52 | # Your answer: 53 | # { 54 | # "missing_cells": [ 55 | # { 56 | # "rowIndex": 1, 57 | # "columnIndex": 2, 58 | # "question": "How old is Mary? Please provide only the numerical answer." 59 | # }, 60 | # { 61 | # "rowIndex": 1, 62 | # "columnIndex": 3, 63 | # "question": "In which city does Mary live? Please provide only the city name." 64 | # } 65 | # ] 66 | # } 67 | 68 | 69 | # Here is your task: 70 | 71 | - Table content: 72 | {table_content} 73 | 74 | - Your answer: 75 | `; 76 | 77 | const DEFAULT_METADATA: ToolMetadata< 78 | JSONSchemaType 79 | > = { 80 | name: "extract_missing_cells", 81 | description: `Use this tool to extract missing cells in a CSV file and generate questions to fill them. This tool only works with local file path.`, 82 | parameters: { 83 | type: "object", 84 | properties: { 85 | filePath: { 86 | type: "string", 87 | description: "The local file path to the CSV file.", 88 | }, 89 | }, 90 | required: ["filePath"], 91 | }, 92 | }; 93 | 94 | export interface ExtractMissingCellsParams { 95 | metadata?: ToolMetadata>; 96 | } 97 | 98 | export class ExtractMissingCellsTool 99 | implements BaseTool 100 | { 101 | metadata: ToolMetadata>; 102 | defaultExtractionPrompt: string; 103 | 104 | constructor(params: ExtractMissingCellsParams) { 105 | this.metadata = params.metadata ?? DEFAULT_METADATA; 106 | this.defaultExtractionPrompt = CSV_EXTRACTION_PROMPT; 107 | } 108 | 109 | private readCsvFile(filePath: string): Promise { 110 | return new Promise((resolve, reject) => { 111 | fs.readFile(filePath, "utf8", (err, data) => { 112 | if (err) { 113 | reject(err); 114 | return; 115 | } 116 | 117 | const parsedData = Papa.parse(data, { 118 | skipEmptyLines: false, 119 | }); 120 | 121 | if (parsedData.errors.length) { 122 | reject(parsedData.errors); 123 | return; 124 | } 125 | 126 | // Ensure all rows have the same number of columns as the header 127 | const maxColumns = parsedData.data[0].length; 128 | const paddedRows = parsedData.data.map((row) => { 129 | return [...row, ...Array(maxColumns - row.length).fill("")]; 130 | }); 131 | 132 | resolve(paddedRows); 133 | }); 134 | }); 135 | } 136 | 137 | private formatToMarkdownTable(data: string[][]): string { 138 | if (data.length === 0) return ""; 139 | 140 | const maxColumns = data[0].length; 141 | 142 | const headerRow = `| ${data[0].join(" | ")} |`; 143 | const separatorRow = `| ${Array(maxColumns).fill("---").join(" | ")} |`; 144 | 145 | const dataRows = data.slice(1).map((row) => { 146 | return `| ${row.join(" | ")} |`; 147 | }); 148 | 149 | return [headerRow, separatorRow, ...dataRows].join("\n"); 150 | } 151 | 152 | async call(input: ExtractMissingCellsParameter): Promise { 153 | const { filePath } = input; 154 | let tableContent: string[][]; 155 | try { 156 | tableContent = await this.readCsvFile(filePath); 157 | } catch (error) { 158 | throw new Error( 159 | `Failed to read CSV file. Make sure that you are reading a local file path (not a sandbox path).`, 160 | ); 161 | } 162 | 163 | const prompt = this.defaultExtractionPrompt.replace( 164 | "{table_content}", 165 | this.formatToMarkdownTable(tableContent), 166 | ); 167 | 168 | const llm = Settings.llm; 169 | const response = await llm.complete({ 170 | prompt, 171 | }); 172 | const rawAnswer = response.text; 173 | const parsedResponse = JSON.parse(rawAnswer) as { 174 | missing_cells: MissingCell[]; 175 | }; 176 | if (!parsedResponse.missing_cells) { 177 | throw new Error( 178 | "The answer is not in the correct format. There should be a missing_cells array.", 179 | ); 180 | } 181 | const answer = parsedResponse.missing_cells; 182 | 183 | return answer; 184 | } 185 | } 186 | 187 | type FillMissingCellsParameter = { 188 | filePath: string; 189 | cells: { 190 | rowIndex: number; 191 | columnIndex: number; 192 | answer: string; 193 | }[]; 194 | }; 195 | 196 | const FILL_CELLS_METADATA: ToolMetadata< 197 | JSONSchemaType 198 | > = { 199 | name: "fill_missing_cells", 200 | description: `Use this tool to fill missing cells in a CSV file with provided answers. This tool only works with local file path.`, 201 | parameters: { 202 | type: "object", 203 | properties: { 204 | filePath: { 205 | type: "string", 206 | description: "The local file path to the CSV file.", 207 | }, 208 | cells: { 209 | type: "array", 210 | items: { 211 | type: "object", 212 | properties: { 213 | rowIndex: { type: "number" }, 214 | columnIndex: { type: "number" }, 215 | answer: { type: "string" }, 216 | }, 217 | required: ["rowIndex", "columnIndex", "answer"], 218 | }, 219 | description: "Array of cells to fill with their answers", 220 | }, 221 | }, 222 | required: ["filePath", "cells"], 223 | }, 224 | }; 225 | 226 | export interface FillMissingCellsParams { 227 | metadata?: ToolMetadata>; 228 | } 229 | 230 | export class FillMissingCellsTool 231 | implements BaseTool 232 | { 233 | metadata: ToolMetadata>; 234 | 235 | constructor(params: FillMissingCellsParams = {}) { 236 | this.metadata = params.metadata ?? FILL_CELLS_METADATA; 237 | } 238 | 239 | async call(input: FillMissingCellsParameter): Promise { 240 | const { filePath, cells } = input; 241 | 242 | // Read the CSV file 243 | const fileContent = await new Promise((resolve, reject) => { 244 | fs.readFile(filePath, "utf8", (err, data) => { 245 | if (err) { 246 | reject(err); 247 | } else { 248 | resolve(data); 249 | } 250 | }); 251 | }); 252 | 253 | // Parse CSV with PapaParse 254 | const parseResult = Papa.parse(fileContent, { 255 | header: false, // Ensure the header is not treated as a separate object 256 | skipEmptyLines: false, // Ensure empty lines are not skipped 257 | }); 258 | 259 | if (parseResult.errors.length) { 260 | throw new Error( 261 | "Failed to parse CSV file: " + parseResult.errors[0].message, 262 | ); 263 | } 264 | 265 | const rows = parseResult.data; 266 | 267 | // Fill the cells with answers 268 | for (const cell of cells) { 269 | // Adjust rowIndex to start from 1 for data rows 270 | const adjustedRowIndex = cell.rowIndex + 1; 271 | if ( 272 | adjustedRowIndex < rows.length && 273 | cell.columnIndex < rows[adjustedRowIndex].length 274 | ) { 275 | rows[adjustedRowIndex][cell.columnIndex] = cell.answer; 276 | } 277 | } 278 | 279 | // Convert back to CSV format 280 | const updatedContent = Papa.unparse(rows, { 281 | delimiter: parseResult.meta.delimiter, 282 | }); 283 | 284 | // Use the helper function to write the file 285 | const parsedPath = path.parse(filePath); 286 | const newFileName = `${parsedPath.name}-filled${parsedPath.ext}`; 287 | const newFilePath = path.join("output/tools", newFileName); 288 | 289 | const newFileUrl = await saveDocument(newFilePath, updatedContent); 290 | 291 | return ( 292 | "Successfully filled missing cells in the CSV file. File URL to show to the user: " + 293 | newFileUrl 294 | ); 295 | } 296 | } 297 | -------------------------------------------------------------------------------- /app/api/chat/engine/tools/img-gen.ts: -------------------------------------------------------------------------------- 1 | import type { JSONSchemaType } from "ajv"; 2 | import { FormData } from "formdata-node"; 3 | import fs from "fs"; 4 | import got from "got"; 5 | import { BaseTool, ToolMetadata } from "llamaindex"; 6 | import path from "node:path"; 7 | import { Readable } from "stream"; 8 | 9 | export type ImgGeneratorParameter = { 10 | prompt: string; 11 | }; 12 | 13 | export type ImgGeneratorToolParams = { 14 | metadata?: ToolMetadata>; 15 | }; 16 | 17 | export type ImgGeneratorToolOutput = { 18 | isSuccess: boolean; 19 | imageUrl?: string; 20 | errorMessage?: string; 21 | }; 22 | 23 | const DEFAULT_META_DATA: ToolMetadata> = { 24 | name: "image_generator", 25 | description: `Use this function to generate an image based on the prompt.`, 26 | parameters: { 27 | type: "object", 28 | properties: { 29 | prompt: { 30 | type: "string", 31 | description: "The prompt to generate the image", 32 | }, 33 | }, 34 | required: ["prompt"], 35 | }, 36 | }; 37 | 38 | export class ImgGeneratorTool implements BaseTool { 39 | readonly IMG_OUTPUT_FORMAT = "webp"; 40 | readonly IMG_OUTPUT_DIR = "output/tools"; 41 | readonly IMG_GEN_API = 42 | "https://api.stability.ai/v2beta/stable-image/generate/core"; 43 | 44 | metadata: ToolMetadata>; 45 | 46 | constructor(params?: ImgGeneratorToolParams) { 47 | this.checkRequiredEnvVars(); 48 | this.metadata = params?.metadata || DEFAULT_META_DATA; 49 | } 50 | 51 | async call(input: ImgGeneratorParameter): Promise { 52 | return await this.generateImage(input.prompt); 53 | } 54 | 55 | private generateImage = async ( 56 | prompt: string, 57 | ): Promise => { 58 | try { 59 | const buffer = await this.promptToImgBuffer(prompt); 60 | const imageUrl = this.saveImage(buffer); 61 | return { isSuccess: true, imageUrl }; 62 | } catch (error) { 63 | console.error(error); 64 | return { 65 | isSuccess: false, 66 | errorMessage: "Failed to generate image. Please try again.", 67 | }; 68 | } 69 | }; 70 | 71 | private promptToImgBuffer = async (prompt: string) => { 72 | const form = new FormData(); 73 | form.append("prompt", prompt); 74 | form.append("output_format", this.IMG_OUTPUT_FORMAT); 75 | const buffer = await got 76 | .post(this.IMG_GEN_API, { 77 | // Not sure why it shows an type error when passing form to body 78 | // Although I follow document: https://github.com/sindresorhus/got/blob/main/documentation/2-options.md#body 79 | // Tt still works fine, so I make casting to unknown to avoid the typescript warning 80 | // Found a similar issue: https://github.com/sindresorhus/got/discussions/1877 81 | body: form as unknown as Buffer | Readable | string, 82 | headers: { 83 | Authorization: `Bearer ${process.env.STABILITY_API_KEY}`, 84 | Accept: "image/*", 85 | }, 86 | }) 87 | .buffer(); 88 | return buffer; 89 | }; 90 | 91 | private saveImage = (buffer: Buffer) => { 92 | const filename = `${crypto.randomUUID()}.${this.IMG_OUTPUT_FORMAT}`; 93 | const outputPath = path.join(this.IMG_OUTPUT_DIR, filename); 94 | fs.writeFileSync(outputPath, buffer); 95 | const url = `${process.env.FILESERVER_URL_PREFIX}/${this.IMG_OUTPUT_DIR}/${filename}`; 96 | console.log(`Saved image to ${outputPath}.\nURL: ${url}`); 97 | return url; 98 | }; 99 | 100 | private checkRequiredEnvVars = () => { 101 | if (!process.env.STABILITY_API_KEY) { 102 | throw new Error( 103 | "STABILITY_API_KEY key is required to run image generator. Get it here: https://platform.stability.ai/account/keys", 104 | ); 105 | } 106 | if (!process.env.FILESERVER_URL_PREFIX) { 107 | throw new Error( 108 | "FILESERVER_URL_PREFIX is required to display file output after generation", 109 | ); 110 | } 111 | }; 112 | } 113 | -------------------------------------------------------------------------------- /app/api/chat/engine/tools/index.ts: -------------------------------------------------------------------------------- 1 | import { BaseToolWithCall } from "llamaindex"; 2 | import fs from "node:fs/promises"; 3 | import path from "node:path"; 4 | import { CodeGeneratorTool, CodeGeneratorToolParams } from "./code-generator"; 5 | import { 6 | DocumentGenerator, 7 | DocumentGeneratorParams, 8 | } from "./document-generator"; 9 | import { DuckDuckGoSearchTool, DuckDuckGoToolParams } from "./duckduckgo"; 10 | import { 11 | ExtractMissingCellsParams, 12 | ExtractMissingCellsTool, 13 | FillMissingCellsParams, 14 | FillMissingCellsTool, 15 | } from "./form-filling"; 16 | import { ImgGeneratorTool, ImgGeneratorToolParams } from "./img-gen"; 17 | import { InterpreterTool, InterpreterToolParams } from "./interpreter"; 18 | import { OpenAPIActionTool } from "./openapi-action"; 19 | import { WeatherTool, WeatherToolParams } from "./weather"; 20 | import { WikipediaTool, WikipediaToolParams } from "./wikipedia"; 21 | 22 | type ToolCreator = (config: unknown) => Promise; 23 | 24 | export async function createTools(toolConfig: { 25 | local: Record; 26 | llamahub: any; 27 | }): Promise { 28 | // add local tools from the 'tools' folder (if configured) 29 | const tools = await createLocalTools(toolConfig.local); 30 | return tools; 31 | } 32 | 33 | const toolFactory: Record = { 34 | "wikipedia.WikipediaToolSpec": async (config: unknown) => { 35 | return [new WikipediaTool(config as WikipediaToolParams)]; 36 | }, 37 | weather: async (config: unknown) => { 38 | return [new WeatherTool(config as WeatherToolParams)]; 39 | }, 40 | interpreter: async (config: unknown) => { 41 | return [new InterpreterTool(config as InterpreterToolParams)]; 42 | }, 43 | "openapi_action.OpenAPIActionToolSpec": async (config: unknown) => { 44 | const { openapi_uri, domain_headers } = config as { 45 | openapi_uri: string; 46 | domain_headers: Record>; 47 | }; 48 | const openAPIActionTool = new OpenAPIActionTool( 49 | openapi_uri, 50 | domain_headers, 51 | ); 52 | return await openAPIActionTool.toToolFunctions(); 53 | }, 54 | duckduckgo: async (config: unknown) => { 55 | return [new DuckDuckGoSearchTool(config as DuckDuckGoToolParams)]; 56 | }, 57 | img_gen: async (config: unknown) => { 58 | return [new ImgGeneratorTool(config as ImgGeneratorToolParams)]; 59 | }, 60 | artifact: async (config: unknown) => { 61 | return [new CodeGeneratorTool(config as CodeGeneratorToolParams)]; 62 | }, 63 | document_generator: async (config: unknown) => { 64 | return [new DocumentGenerator(config as DocumentGeneratorParams)]; 65 | }, 66 | form_filling: async (config: unknown) => { 67 | return [ 68 | new ExtractMissingCellsTool(config as ExtractMissingCellsParams), 69 | new FillMissingCellsTool(config as FillMissingCellsParams), 70 | ]; 71 | }, 72 | }; 73 | 74 | async function createLocalTools( 75 | localConfig: Record, 76 | ): Promise { 77 | const tools: BaseToolWithCall[] = []; 78 | 79 | for (const [key, toolConfig] of Object.entries(localConfig)) { 80 | if (key in toolFactory) { 81 | const newTools = await toolFactory[key](toolConfig); 82 | tools.push(...newTools); 83 | } 84 | } 85 | 86 | return tools; 87 | } 88 | 89 | export async function getConfiguredTools( 90 | configPath?: string, 91 | ): Promise { 92 | const configFile = path.join(configPath ?? "config", "tools.json"); 93 | const toolConfig = JSON.parse(await fs.readFile(configFile, "utf8")); 94 | const tools = await createTools(toolConfig); 95 | return tools; 96 | } 97 | 98 | export async function getTool( 99 | toolName: string, 100 | ): Promise { 101 | const tools = await getConfiguredTools(); 102 | return tools.find((tool) => tool.metadata.name === toolName); 103 | } 104 | -------------------------------------------------------------------------------- /app/api/chat/engine/tools/interpreter.ts: -------------------------------------------------------------------------------- 1 | import { Logs, Result, Sandbox } from "@e2b/code-interpreter"; 2 | import type { JSONSchemaType } from "ajv"; 3 | import fs from "fs"; 4 | import { BaseTool, ToolMetadata } from "llamaindex"; 5 | import crypto from "node:crypto"; 6 | import path from "node:path"; 7 | 8 | export type InterpreterParameter = { 9 | code: string; 10 | sandboxFiles?: string[]; 11 | retryCount?: number; 12 | }; 13 | 14 | export type InterpreterToolParams = { 15 | metadata?: ToolMetadata>; 16 | apiKey?: string; 17 | fileServerURLPrefix?: string; 18 | }; 19 | 20 | export type InterpreterToolOutput = { 21 | isError: boolean; 22 | logs: Logs; 23 | text?: string; 24 | extraResult: InterpreterExtraResult[]; 25 | retryCount?: number; 26 | }; 27 | 28 | type InterpreterExtraType = 29 | | "html" 30 | | "markdown" 31 | | "svg" 32 | | "png" 33 | | "jpeg" 34 | | "pdf" 35 | | "latex" 36 | | "json" 37 | | "javascript"; 38 | 39 | export type InterpreterExtraResult = { 40 | type: InterpreterExtraType; 41 | content?: string; 42 | filename?: string; 43 | url?: string; 44 | }; 45 | 46 | const DEFAULT_META_DATA: ToolMetadata> = { 47 | name: "interpreter", 48 | description: `Execute python code in a Jupyter notebook cell and return any result, stdout, stderr, display_data, and error. 49 | If the code needs to use a file, ALWAYS pass the file path in the sandbox_files argument. 50 | You have a maximum of 3 retries to get the code to run successfully. 51 | `, 52 | parameters: { 53 | type: "object", 54 | properties: { 55 | code: { 56 | type: "string", 57 | description: "The python code to execute in a single cell.", 58 | }, 59 | sandboxFiles: { 60 | type: "array", 61 | description: 62 | "List of local file paths to be used by the code. The tool will throw an error if a file is not found.", 63 | items: { 64 | type: "string", 65 | }, 66 | nullable: true, 67 | }, 68 | retryCount: { 69 | type: "number", 70 | description: "The number of times the tool has been retried", 71 | default: 0, 72 | nullable: true, 73 | }, 74 | }, 75 | required: ["code"], 76 | }, 77 | }; 78 | 79 | export class InterpreterTool implements BaseTool { 80 | private readonly outputDir = "output/tools"; 81 | private readonly uploadedFilesDir = "output/uploaded"; 82 | private apiKey?: string; 83 | private fileServerURLPrefix?: string; 84 | metadata: ToolMetadata>; 85 | codeInterpreter?: Sandbox; 86 | 87 | constructor(params?: InterpreterToolParams) { 88 | this.metadata = params?.metadata || DEFAULT_META_DATA; 89 | this.apiKey = params?.apiKey || process.env.E2B_API_KEY; 90 | this.fileServerURLPrefix = 91 | params?.fileServerURLPrefix || process.env.FILESERVER_URL_PREFIX; 92 | 93 | if (!this.apiKey) { 94 | throw new Error( 95 | "E2B_API_KEY key is required to run code interpreter. Get it here: https://e2b.dev/docs/getting-started/api-key", 96 | ); 97 | } 98 | if (!this.fileServerURLPrefix) { 99 | throw new Error( 100 | "FILESERVER_URL_PREFIX is required to display file output from sandbox", 101 | ); 102 | } 103 | } 104 | 105 | public async initInterpreter(input: InterpreterParameter) { 106 | if (!this.codeInterpreter) { 107 | this.codeInterpreter = await Sandbox.create({ 108 | apiKey: this.apiKey, 109 | }); 110 | // upload files to sandbox when it's initialized 111 | if (input.sandboxFiles) { 112 | console.log(`Uploading ${input.sandboxFiles.length} files to sandbox`); 113 | try { 114 | for (const filePath of input.sandboxFiles) { 115 | const fileName = path.basename(filePath); 116 | const localFilePath = path.join(this.uploadedFilesDir, fileName); 117 | const content = fs.readFileSync(localFilePath); 118 | 119 | const arrayBuffer = new Uint8Array(content).buffer; 120 | await this.codeInterpreter?.files.write(filePath, arrayBuffer); 121 | } 122 | } catch (error) { 123 | console.error("Got error when uploading files to sandbox", error); 124 | } 125 | } 126 | } 127 | 128 | return this.codeInterpreter; 129 | } 130 | 131 | public async codeInterpret( 132 | input: InterpreterParameter, 133 | ): Promise { 134 | console.log( 135 | `Sandbox files: ${input.sandboxFiles}. Retry count: ${input.retryCount}`, 136 | ); 137 | 138 | if (input.retryCount && input.retryCount >= 3) { 139 | return { 140 | isError: true, 141 | logs: { 142 | stdout: [], 143 | stderr: [], 144 | }, 145 | text: "Max retries reached", 146 | extraResult: [], 147 | }; 148 | } 149 | 150 | console.log( 151 | `\n${"=".repeat(50)}\n> Running following AI-generated code:\n${input.code}\n${"=".repeat(50)}`, 152 | ); 153 | const interpreter = await this.initInterpreter(input); 154 | const exec = await interpreter.runCode(input.code); 155 | if (exec.error) console.error("[Code Interpreter error]", exec.error); 156 | const extraResult = await this.getExtraResult(exec.results[0]); 157 | const result: InterpreterToolOutput = { 158 | isError: !!exec.error, 159 | logs: exec.logs, 160 | text: exec.text, 161 | extraResult, 162 | retryCount: input.retryCount ? input.retryCount + 1 : 1, 163 | }; 164 | return result; 165 | } 166 | 167 | async call(input: InterpreterParameter): Promise { 168 | const result = await this.codeInterpret(input); 169 | return result; 170 | } 171 | 172 | async close() { 173 | await this.codeInterpreter?.kill(); 174 | } 175 | 176 | private async getExtraResult( 177 | res?: Result, 178 | ): Promise { 179 | if (!res) return []; 180 | const output: InterpreterExtraResult[] = []; 181 | 182 | try { 183 | const formats = res.formats(); // formats available for the result. Eg: ['png', ...] 184 | const results = formats.map((f) => res[f as keyof Result]); // get base64 data for each format 185 | 186 | // save base64 data to file and return the url 187 | for (let i = 0; i < formats.length; i++) { 188 | const ext = formats[i]; 189 | const data = results[i]; 190 | switch (ext) { 191 | case "png": 192 | case "jpeg": 193 | case "svg": 194 | case "pdf": 195 | const { filename } = this.saveToDisk(data, ext); 196 | output.push({ 197 | type: ext as InterpreterExtraType, 198 | filename, 199 | url: this.getFileUrl(filename), 200 | }); 201 | break; 202 | default: 203 | output.push({ 204 | type: ext as InterpreterExtraType, 205 | content: data, 206 | }); 207 | break; 208 | } 209 | } 210 | } catch (error) { 211 | console.error("Error when parsing e2b response", error); 212 | } 213 | 214 | return output; 215 | } 216 | 217 | // Consider saving to cloud storage instead but it may cost more for you 218 | // See: https://e2b.dev/docs/sandbox/api/filesystem#write-to-file 219 | private saveToDisk( 220 | base64Data: string, 221 | ext: string, 222 | ): { 223 | outputPath: string; 224 | filename: string; 225 | } { 226 | const filename = `${crypto.randomUUID()}.${ext}`; // generate a unique filename 227 | const buffer = Buffer.from(base64Data, "base64"); 228 | const outputPath = this.getOutputPath(filename); 229 | fs.writeFileSync(outputPath, buffer); 230 | console.log(`Saved file to ${outputPath}`); 231 | return { 232 | outputPath, 233 | filename, 234 | }; 235 | } 236 | 237 | private getOutputPath(filename: string): string { 238 | // if outputDir doesn't exist, create it 239 | if (!fs.existsSync(this.outputDir)) { 240 | fs.mkdirSync(this.outputDir, { recursive: true }); 241 | } 242 | return path.join(this.outputDir, filename); 243 | } 244 | 245 | private getFileUrl(filename: string): string { 246 | return `${this.fileServerURLPrefix}/${this.outputDir}/${filename}`; 247 | } 248 | } 249 | -------------------------------------------------------------------------------- /app/api/chat/engine/tools/openapi-action.ts: -------------------------------------------------------------------------------- 1 | import SwaggerParser from "@apidevtools/swagger-parser"; 2 | import { JSONSchemaType } from "ajv"; 3 | import got from "got"; 4 | import { FunctionTool, JSONValue, ToolMetadata } from "llamaindex"; 5 | 6 | interface DomainHeaders { 7 | [key: string]: { [header: string]: string }; 8 | } 9 | 10 | type Input = { 11 | url: string; 12 | params: object; 13 | }; 14 | 15 | type APIInfo = { 16 | description: string; 17 | title: string; 18 | }; 19 | 20 | export class OpenAPIActionTool { 21 | // cache the loaded specs by URL 22 | private static specs: Record = {}; 23 | 24 | private readonly INVALID_URL_PROMPT = 25 | "This url did not include a hostname or scheme. Please determine the complete URL and try again."; 26 | 27 | private createLoadSpecMetaData = (info: APIInfo) => { 28 | return { 29 | name: "load_openapi_spec", 30 | description: `Use this to retrieve the OpenAPI spec for the API named ${info.title} with the following description: ${info.description}. Call it before making any requests to the API.`, 31 | }; 32 | }; 33 | 34 | private readonly createMethodCallMetaData = ( 35 | method: "POST" | "PATCH" | "GET", 36 | info: APIInfo, 37 | ) => { 38 | return { 39 | name: `${method.toLowerCase()}_request`, 40 | description: `Use this to call the ${method} method on the API named ${info.title}`, 41 | parameters: { 42 | type: "object", 43 | properties: { 44 | url: { 45 | type: "string", 46 | description: `The url to make the ${method} request against`, 47 | }, 48 | params: { 49 | type: "object", 50 | description: 51 | method === "GET" 52 | ? "the URL parameters to provide with the get request" 53 | : `the key-value pairs to provide with the ${method} request`, 54 | }, 55 | }, 56 | required: ["url"], 57 | }, 58 | } as ToolMetadata>; 59 | }; 60 | 61 | constructor( 62 | public openapi_uri: string, 63 | public domainHeaders: DomainHeaders = {}, 64 | ) {} 65 | 66 | async loadOpenapiSpec(url: string): Promise { 67 | const api = await SwaggerParser.validate(url); 68 | return { 69 | servers: "servers" in api ? api.servers : "", 70 | info: { description: api.info.description, title: api.info.title }, 71 | endpoints: api.paths, 72 | }; 73 | } 74 | 75 | async getRequest(input: Input): Promise { 76 | if (!this.validUrl(input.url)) { 77 | return this.INVALID_URL_PROMPT; 78 | } 79 | try { 80 | const data = await got 81 | .get(input.url, { 82 | headers: this.getHeadersForUrl(input.url), 83 | searchParams: input.params as URLSearchParams, 84 | }) 85 | .json(); 86 | return data as JSONValue; 87 | } catch (error) { 88 | return error as JSONValue; 89 | } 90 | } 91 | 92 | async postRequest(input: Input): Promise { 93 | if (!this.validUrl(input.url)) { 94 | return this.INVALID_URL_PROMPT; 95 | } 96 | try { 97 | const res = await got.post(input.url, { 98 | headers: this.getHeadersForUrl(input.url), 99 | json: input.params, 100 | }); 101 | return res.body as JSONValue; 102 | } catch (error) { 103 | return error as JSONValue; 104 | } 105 | } 106 | 107 | async patchRequest(input: Input): Promise { 108 | if (!this.validUrl(input.url)) { 109 | return this.INVALID_URL_PROMPT; 110 | } 111 | try { 112 | const res = await got.patch(input.url, { 113 | headers: this.getHeadersForUrl(input.url), 114 | json: input.params, 115 | }); 116 | return res.body as JSONValue; 117 | } catch (error) { 118 | return error as JSONValue; 119 | } 120 | } 121 | 122 | public async toToolFunctions() { 123 | if (!OpenAPIActionTool.specs[this.openapi_uri]) { 124 | console.log(`Loading spec for URL: ${this.openapi_uri}`); 125 | const spec = await this.loadOpenapiSpec(this.openapi_uri); 126 | OpenAPIActionTool.specs[this.openapi_uri] = spec; 127 | } 128 | const spec = OpenAPIActionTool.specs[this.openapi_uri]; 129 | // TODO: read endpoints with parameters from spec and create one tool for each endpoint 130 | // For now, we just create a tool for each HTTP method which does not work well for passing parameters 131 | return [ 132 | FunctionTool.from(() => { 133 | return spec; 134 | }, this.createLoadSpecMetaData(spec.info)), 135 | FunctionTool.from( 136 | this.getRequest.bind(this), 137 | this.createMethodCallMetaData("GET", spec.info), 138 | ), 139 | FunctionTool.from( 140 | this.postRequest.bind(this), 141 | this.createMethodCallMetaData("POST", spec.info), 142 | ), 143 | FunctionTool.from( 144 | this.patchRequest.bind(this), 145 | this.createMethodCallMetaData("PATCH", spec.info), 146 | ), 147 | ]; 148 | } 149 | 150 | private validUrl(url: string): boolean { 151 | const parsed = new URL(url); 152 | return !!parsed.protocol && !!parsed.hostname; 153 | } 154 | 155 | private getDomain(url: string): string { 156 | const parsed = new URL(url); 157 | return parsed.hostname; 158 | } 159 | 160 | private getHeadersForUrl(url: string): { [header: string]: string } { 161 | const domain = this.getDomain(url); 162 | return this.domainHeaders[domain] || {}; 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /app/api/chat/engine/tools/query-engine.ts: -------------------------------------------------------------------------------- 1 | import { 2 | BaseQueryEngine, 3 | CloudRetrieveParams, 4 | LlamaCloudIndex, 5 | MetadataFilters, 6 | QueryEngineTool, 7 | VectorStoreIndex, 8 | } from "llamaindex"; 9 | import { generateFilters } from "../queryFilter"; 10 | 11 | interface QueryEngineParams { 12 | documentIds?: string[]; 13 | topK?: number; 14 | } 15 | 16 | export function createQueryEngineTool( 17 | index: VectorStoreIndex | LlamaCloudIndex, 18 | params?: QueryEngineParams, 19 | name?: string, 20 | description?: string, 21 | ): QueryEngineTool { 22 | return new QueryEngineTool({ 23 | queryEngine: createQueryEngine(index, params), 24 | metadata: { 25 | name: name || "query_engine", 26 | description: 27 | description || 28 | `Use this tool to retrieve information about the text corpus from an index.`, 29 | }, 30 | }); 31 | } 32 | 33 | function createQueryEngine( 34 | index: VectorStoreIndex | LlamaCloudIndex, 35 | params?: QueryEngineParams, 36 | ): BaseQueryEngine { 37 | const baseQueryParams = { 38 | similarityTopK: 39 | params?.topK ?? 40 | (process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined), 41 | }; 42 | 43 | if (index instanceof LlamaCloudIndex) { 44 | return index.asQueryEngine({ 45 | ...baseQueryParams, 46 | retrieval_mode: "auto_routed", 47 | preFilters: generateFilters( 48 | params?.documentIds || [], 49 | ) as CloudRetrieveParams["filters"], 50 | }); 51 | } 52 | 53 | return index.asQueryEngine({ 54 | ...baseQueryParams, 55 | preFilters: generateFilters(params?.documentIds || []) as MetadataFilters, 56 | }); 57 | } 58 | -------------------------------------------------------------------------------- /app/api/chat/engine/tools/weather.ts: -------------------------------------------------------------------------------- 1 | import type { JSONSchemaType } from "ajv"; 2 | import { BaseTool, ToolMetadata } from "llamaindex"; 3 | 4 | interface GeoLocation { 5 | id: string; 6 | name: string; 7 | latitude: number; 8 | longitude: number; 9 | } 10 | 11 | export type WeatherParameter = { 12 | location: string; 13 | }; 14 | 15 | export type WeatherToolParams = { 16 | metadata?: ToolMetadata>; 17 | }; 18 | 19 | const DEFAULT_META_DATA: ToolMetadata> = { 20 | name: "get_weather_information", 21 | description: ` 22 | Use this function to get the weather of any given location. 23 | Note that the weather code should follow WMO Weather interpretation codes (WW): 24 | 0: Clear sky 25 | 1, 2, 3: Mainly clear, partly cloudy, and overcast 26 | 45, 48: Fog and depositing rime fog 27 | 51, 53, 55: Drizzle: Light, moderate, and dense intensity 28 | 56, 57: Freezing Drizzle: Light and dense intensity 29 | 61, 63, 65: Rain: Slight, moderate and heavy intensity 30 | 66, 67: Freezing Rain: Light and heavy intensity 31 | 71, 73, 75: Snow fall: Slight, moderate, and heavy intensity 32 | 77: Snow grains 33 | 80, 81, 82: Rain showers: Slight, moderate, and violent 34 | 85, 86: Snow showers slight and heavy 35 | 95: Thunderstorm: Slight or moderate 36 | 96, 99: Thunderstorm with slight and heavy hail 37 | `, 38 | parameters: { 39 | type: "object", 40 | properties: { 41 | location: { 42 | type: "string", 43 | description: "The location to get the weather information", 44 | }, 45 | }, 46 | required: ["location"], 47 | }, 48 | }; 49 | 50 | export class WeatherTool implements BaseTool { 51 | metadata: ToolMetadata>; 52 | 53 | private getGeoLocation = async (location: string): Promise => { 54 | const apiUrl = `https://geocoding-api.open-meteo.com/v1/search?name=${location}&count=10&language=en&format=json`; 55 | const response = await fetch(apiUrl); 56 | const data = await response.json(); 57 | const { id, name, latitude, longitude } = data.results[0]; 58 | return { id, name, latitude, longitude }; 59 | }; 60 | 61 | private getWeatherByLocation = async (location: string) => { 62 | console.log( 63 | "Calling open-meteo api to get weather information of location:", 64 | location, 65 | ); 66 | const { latitude, longitude } = await this.getGeoLocation(location); 67 | const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone; 68 | const apiUrl = `https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}¤t=temperature_2m,weather_code&hourly=temperature_2m,weather_code&daily=weather_code&timezone=${timezone}`; 69 | const response = await fetch(apiUrl); 70 | const data = await response.json(); 71 | return data; 72 | }; 73 | 74 | constructor(params?: WeatherToolParams) { 75 | this.metadata = params?.metadata || DEFAULT_META_DATA; 76 | } 77 | 78 | async call(input: WeatherParameter) { 79 | return await this.getWeatherByLocation(input.location); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /app/api/chat/engine/tools/wikipedia.ts: -------------------------------------------------------------------------------- 1 | import type { JSONSchemaType } from "ajv"; 2 | import type { BaseTool, ToolMetadata } from "llamaindex"; 3 | import { default as wiki } from "wikipedia"; 4 | 5 | type WikipediaParameter = { 6 | query: string; 7 | lang?: string; 8 | }; 9 | 10 | export type WikipediaToolParams = { 11 | metadata?: ToolMetadata>; 12 | }; 13 | 14 | const DEFAULT_META_DATA: ToolMetadata> = { 15 | name: "wikipedia_tool", 16 | description: "A tool that uses a query engine to search Wikipedia.", 17 | parameters: { 18 | type: "object", 19 | properties: { 20 | query: { 21 | type: "string", 22 | description: "The query to search for", 23 | }, 24 | lang: { 25 | type: "string", 26 | description: "The language to search in", 27 | nullable: true, 28 | }, 29 | }, 30 | required: ["query"], 31 | }, 32 | }; 33 | 34 | export class WikipediaTool implements BaseTool { 35 | private readonly DEFAULT_LANG = "en"; 36 | metadata: ToolMetadata>; 37 | 38 | constructor(params?: WikipediaToolParams) { 39 | this.metadata = params?.metadata || DEFAULT_META_DATA; 40 | } 41 | 42 | async loadData( 43 | page: string, 44 | lang: string = this.DEFAULT_LANG, 45 | ): Promise { 46 | wiki.setLang(lang); 47 | const pageResult = await wiki.page(page, { autoSuggest: false }); 48 | const content = await pageResult.content(); 49 | return content; 50 | } 51 | 52 | async call({ 53 | query, 54 | lang = this.DEFAULT_LANG, 55 | }: WikipediaParameter): Promise { 56 | const searchResult = await wiki.search(query); 57 | if (searchResult.results.length === 0) return "No search results."; 58 | return await this.loadData(searchResult.results[0].title, lang); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /app/api/chat/llamaindex/documents/helper.ts: -------------------------------------------------------------------------------- 1 | import { Document } from "llamaindex"; 2 | import crypto from "node:crypto"; 3 | import fs from "node:fs"; 4 | import path from "node:path"; 5 | import { getExtractors } from "../../engine/loader"; 6 | import { DocumentFile } from "../streaming/annotations"; 7 | 8 | const MIME_TYPE_TO_EXT: Record = { 9 | "application/pdf": "pdf", 10 | "text/plain": "txt", 11 | "text/csv": "csv", 12 | "application/vnd.openxmlformats-officedocument.wordprocessingml.document": 13 | "docx", 14 | }; 15 | 16 | export const UPLOADED_FOLDER = "output/uploaded"; 17 | 18 | export async function storeAndParseFile( 19 | name: string, 20 | fileBuffer: Buffer, 21 | mimeType: string, 22 | ): Promise { 23 | const file = await storeFile(name, fileBuffer, mimeType); 24 | const documents: Document[] = await parseFile(fileBuffer, name, mimeType); 25 | // Update document IDs in the file metadata 26 | file.refs = documents.map((document) => document.id_ as string); 27 | return file; 28 | } 29 | 30 | export async function storeFile( 31 | name: string, 32 | fileBuffer: Buffer, 33 | mimeType: string, 34 | ) { 35 | const fileExt = MIME_TYPE_TO_EXT[mimeType]; 36 | if (!fileExt) throw new Error(`Unsupported document type: ${mimeType}`); 37 | 38 | const fileId = crypto.randomUUID(); 39 | const newFilename = `${sanitizeFileName(name)}_${fileId}.${fileExt}`; 40 | const filepath = path.join(UPLOADED_FOLDER, newFilename); 41 | const fileUrl = await saveDocument(filepath, fileBuffer); 42 | return { 43 | id: fileId, 44 | name: newFilename, 45 | size: fileBuffer.length, 46 | type: fileExt, 47 | url: fileUrl, 48 | refs: [] as string[], 49 | } as DocumentFile; 50 | } 51 | 52 | export async function parseFile( 53 | fileBuffer: Buffer, 54 | filename: string, 55 | mimeType: string, 56 | ) { 57 | const documents = await loadDocuments(fileBuffer, mimeType); 58 | for (const document of documents) { 59 | document.metadata = { 60 | ...document.metadata, 61 | file_name: filename, 62 | private: "true", // to separate private uploads from public documents 63 | }; 64 | } 65 | return documents; 66 | } 67 | 68 | async function loadDocuments(fileBuffer: Buffer, mimeType: string) { 69 | const extractors = getExtractors(); 70 | const reader = extractors[MIME_TYPE_TO_EXT[mimeType]]; 71 | 72 | if (!reader) { 73 | throw new Error(`Unsupported document type: ${mimeType}`); 74 | } 75 | console.log(`Processing uploaded document of type: ${mimeType}`); 76 | return await reader.loadDataAsContent(fileBuffer); 77 | } 78 | 79 | // Save document to file server and return the file url 80 | export async function saveDocument(filepath: string, content: string | Buffer) { 81 | if (path.isAbsolute(filepath)) { 82 | throw new Error("Absolute file paths are not allowed."); 83 | } 84 | if (!process.env.FILESERVER_URL_PREFIX) { 85 | throw new Error("FILESERVER_URL_PREFIX environment variable is not set."); 86 | } 87 | 88 | const dirPath = path.dirname(filepath); 89 | await fs.promises.mkdir(dirPath, { recursive: true }); 90 | 91 | if (typeof content === "string") { 92 | await fs.promises.writeFile(filepath, content, "utf-8"); 93 | } else { 94 | await fs.promises.writeFile(filepath, content); 95 | } 96 | 97 | const fileurl = `${process.env.FILESERVER_URL_PREFIX}/${filepath}`; 98 | console.log(`Saved document to ${filepath}. Reachable at URL: ${fileurl}`); 99 | return fileurl; 100 | } 101 | 102 | function sanitizeFileName(fileName: string) { 103 | // Remove file extension and sanitize 104 | return fileName.split(".")[0].replace(/[^a-zA-Z0-9_-]/g, "_"); 105 | } 106 | -------------------------------------------------------------------------------- /app/api/chat/llamaindex/documents/pipeline.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Document, 3 | IngestionPipeline, 4 | Settings, 5 | SimpleNodeParser, 6 | storageContextFromDefaults, 7 | VectorStoreIndex, 8 | } from "llamaindex"; 9 | 10 | export async function runPipeline( 11 | currentIndex: VectorStoreIndex | null, 12 | documents: Document[], 13 | ) { 14 | // Use ingestion pipeline to process the documents into nodes and add them to the vector store 15 | const pipeline = new IngestionPipeline({ 16 | transformations: [ 17 | new SimpleNodeParser({ 18 | chunkSize: Settings.chunkSize, 19 | chunkOverlap: Settings.chunkOverlap, 20 | }), 21 | Settings.embedModel, 22 | ], 23 | }); 24 | const nodes = await pipeline.run({ documents }); 25 | if (currentIndex) { 26 | await currentIndex.insertNodes(nodes); 27 | currentIndex.storageContext.docStore.persist(); 28 | console.log("Added nodes to the vector store."); 29 | return documents.map((document) => document.id_); 30 | } else { 31 | // Initialize a new index with the documents 32 | console.log( 33 | "Got empty index, created new index with the uploaded documents", 34 | ); 35 | const persistDir = process.env.STORAGE_CACHE_DIR; 36 | if (!persistDir) { 37 | throw new Error("STORAGE_CACHE_DIR environment variable is required!"); 38 | } 39 | const storageContext = await storageContextFromDefaults({ 40 | persistDir, 41 | }); 42 | const newIndex = await VectorStoreIndex.fromDocuments(documents, { 43 | storageContext, 44 | }); 45 | await newIndex.storageContext.docStore.persist(); 46 | return documents.map((document) => document.id_); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /app/api/chat/llamaindex/documents/upload.ts: -------------------------------------------------------------------------------- 1 | import { Document, LLamaCloudFileService, VectorStoreIndex } from "llamaindex"; 2 | import { LlamaCloudIndex } from "llamaindex/cloud/LlamaCloudIndex"; 3 | import { DocumentFile } from "../streaming/annotations"; 4 | import { parseFile, storeFile } from "./helper"; 5 | import { runPipeline } from "./pipeline"; 6 | 7 | export async function uploadDocument( 8 | index: VectorStoreIndex | LlamaCloudIndex | null, 9 | name: string, 10 | raw: string, 11 | ): Promise { 12 | const [header, content] = raw.split(","); 13 | const mimeType = header.replace("data:", "").replace(";base64", ""); 14 | const fileBuffer = Buffer.from(content, "base64"); 15 | 16 | // Store file 17 | const fileMetadata = await storeFile(name, fileBuffer, mimeType); 18 | 19 | // Do not index csv files 20 | if (mimeType === "text/csv") { 21 | return fileMetadata; 22 | } 23 | let documentIds: string[] = []; 24 | if (index instanceof LlamaCloudIndex) { 25 | // trigger LlamaCloudIndex API to upload the file and run the pipeline 26 | const projectId = await index.getProjectId(); 27 | const pipelineId = await index.getPipelineId(); 28 | try { 29 | documentIds = [ 30 | await LLamaCloudFileService.addFileToPipeline( 31 | projectId, 32 | pipelineId, 33 | new File([fileBuffer], name, { type: mimeType }), 34 | { private: "true" }, 35 | ), 36 | ]; 37 | } catch (error) { 38 | if ( 39 | error instanceof ReferenceError && 40 | error.message.includes("File is not defined") 41 | ) { 42 | throw new Error( 43 | "File class is not supported in the current Node.js version. Please use Node.js 20 or higher.", 44 | ); 45 | } 46 | throw error; 47 | } 48 | } else { 49 | // run the pipeline for other vector store indexes 50 | const documents: Document[] = await parseFile( 51 | fileBuffer, 52 | fileMetadata.name, 53 | mimeType, 54 | ); 55 | documentIds = await runPipeline(index, documents); 56 | } 57 | 58 | // Update file metadata with document IDs 59 | fileMetadata.refs = documentIds; 60 | return fileMetadata; 61 | } 62 | -------------------------------------------------------------------------------- /app/api/chat/llamaindex/streaming/annotations.ts: -------------------------------------------------------------------------------- 1 | import { JSONValue, Message } from "ai"; 2 | import { 3 | ChatMessage, 4 | MessageContent, 5 | MessageContentDetail, 6 | MessageType, 7 | } from "llamaindex"; 8 | import { UPLOADED_FOLDER } from "../documents/helper"; 9 | 10 | export type DocumentFileType = "csv" | "pdf" | "txt" | "docx"; 11 | 12 | export type DocumentFile = { 13 | id: string; 14 | name: string; 15 | size: number; 16 | type: string; 17 | url: string; 18 | refs?: string[]; 19 | }; 20 | 21 | type Annotation = { 22 | type: string; 23 | data: object; 24 | }; 25 | 26 | export function isValidMessages(messages: Message[]): boolean { 27 | const lastMessage = 28 | messages && messages.length > 0 ? messages[messages.length - 1] : null; 29 | return lastMessage !== null && lastMessage.role === "user"; 30 | } 31 | 32 | export function retrieveDocumentIds(messages: Message[]): string[] { 33 | // retrieve document Ids from the annotations of all messages (if any) 34 | const documentFiles = retrieveDocumentFiles(messages); 35 | return documentFiles.map((file) => file.refs || []).flat(); 36 | } 37 | 38 | export function retrieveDocumentFiles(messages: Message[]): DocumentFile[] { 39 | const annotations = getAllAnnotations(messages); 40 | if (annotations.length === 0) return []; 41 | 42 | const files: DocumentFile[] = []; 43 | for (const { type, data } of annotations) { 44 | if ( 45 | type === "document_file" && 46 | "files" in data && 47 | Array.isArray(data.files) 48 | ) { 49 | files.push(...data.files); 50 | } 51 | } 52 | return files; 53 | } 54 | 55 | export function retrieveMessageContent(messages: Message[]): MessageContent { 56 | const userMessage = messages[messages.length - 1]; 57 | return [ 58 | { 59 | type: "text", 60 | text: userMessage.content, 61 | }, 62 | ...retrieveLatestArtifact(messages), 63 | ...convertAnnotations(messages), 64 | ]; 65 | } 66 | 67 | export function convertToChatHistory(messages: Message[]): ChatMessage[] { 68 | if (!messages || !Array.isArray(messages)) { 69 | return []; 70 | } 71 | const agentHistory = retrieveAgentHistoryMessage(messages); 72 | if (agentHistory) { 73 | const previousMessages = messages.slice(0, -1); 74 | return [...previousMessages, agentHistory].map((msg) => ({ 75 | role: msg.role as MessageType, 76 | content: msg.content, 77 | })); 78 | } 79 | return messages.map((msg) => ({ 80 | role: msg.role as MessageType, 81 | content: msg.content, 82 | })); 83 | } 84 | 85 | function retrieveAgentHistoryMessage( 86 | messages: Message[], 87 | maxAgentMessages = 10, 88 | ): ChatMessage | null { 89 | const agentAnnotations = getAnnotations<{ agent: string; text: string }>( 90 | messages, 91 | { role: "assistant", type: "agent" }, 92 | ).slice(-maxAgentMessages); 93 | 94 | if (agentAnnotations.length > 0) { 95 | const messageContent = 96 | "Here is the previous conversation of agents:\n" + 97 | agentAnnotations.map((annotation) => annotation.data.text).join("\n"); 98 | return { 99 | role: "assistant", 100 | content: messageContent, 101 | }; 102 | } 103 | return null; 104 | } 105 | 106 | function getFileContent(file: DocumentFile): string { 107 | let defaultContent = `=====File: ${file.name}=====\n`; 108 | // Include file URL if it's available 109 | const urlPrefix = process.env.FILESERVER_URL_PREFIX; 110 | let urlContent = ""; 111 | if (urlPrefix) { 112 | if (file.url) { 113 | urlContent = `File URL: ${file.url}\n`; 114 | } else { 115 | urlContent = `File URL (instruction: do not update this file URL yourself): ${urlPrefix}/output/uploaded/${file.name}\n`; 116 | } 117 | } else { 118 | console.warn( 119 | "Warning: FILESERVER_URL_PREFIX not set in environment variables. Can't use file server", 120 | ); 121 | } 122 | defaultContent += urlContent; 123 | 124 | // Include document IDs if it's available 125 | if (file.refs) { 126 | defaultContent += `Document IDs: ${file.refs}\n`; 127 | } 128 | // Include sandbox file paths 129 | const sandboxFilePath = `/tmp/${file.name}`; 130 | defaultContent += `Sandbox file path (instruction: only use sandbox path for artifact or code interpreter tool): ${sandboxFilePath}\n`; 131 | 132 | // Include local file path 133 | const localFilePath = `${UPLOADED_FOLDER}/${file.name}`; 134 | defaultContent += `Local file path (instruction: use for local tool that requires a local path): ${localFilePath}\n`; 135 | 136 | return defaultContent; 137 | } 138 | 139 | function getAllAnnotations(messages: Message[]): Annotation[] { 140 | return messages.flatMap((message) => 141 | (message.annotations ?? []).map((annotation) => 142 | getValidAnnotation(annotation), 143 | ), 144 | ); 145 | } 146 | 147 | // get latest artifact from annotations to append to the user message 148 | function retrieveLatestArtifact(messages: Message[]): MessageContentDetail[] { 149 | const annotations = getAllAnnotations(messages); 150 | if (annotations.length === 0) return []; 151 | 152 | for (const { type, data } of annotations.reverse()) { 153 | if ( 154 | type === "tools" && 155 | "toolCall" in data && 156 | "toolOutput" in data && 157 | typeof data.toolCall === "object" && 158 | typeof data.toolOutput === "object" && 159 | data.toolCall !== null && 160 | data.toolOutput !== null && 161 | "name" in data.toolCall && 162 | data.toolCall.name === "artifact" 163 | ) { 164 | const toolOutput = data.toolOutput as { output?: { code?: string } }; 165 | if (toolOutput.output?.code) { 166 | return [ 167 | { 168 | type: "text", 169 | text: `The existing code is:\n\`\`\`\n${toolOutput.output.code}\n\`\`\``, 170 | }, 171 | ]; 172 | } 173 | } 174 | } 175 | return []; 176 | } 177 | 178 | function convertAnnotations(messages: Message[]): MessageContentDetail[] { 179 | // get all annotations from user messages 180 | const annotations: Annotation[] = messages 181 | .filter((message) => message.role === "user" && message.annotations) 182 | .flatMap((message) => message.annotations?.map(getValidAnnotation) || []); 183 | if (annotations.length === 0) return []; 184 | 185 | const content: MessageContentDetail[] = []; 186 | annotations.forEach(({ type, data }) => { 187 | // convert image 188 | if (type === "image" && "url" in data && typeof data.url === "string") { 189 | content.push({ 190 | type: "image_url", 191 | image_url: { 192 | url: data.url, 193 | }, 194 | }); 195 | } 196 | // convert the content of files to a text message 197 | if ( 198 | type === "document_file" && 199 | "files" in data && 200 | Array.isArray(data.files) 201 | ) { 202 | const fileContent = data.files.map(getFileContent).join("\n"); 203 | content.push({ 204 | type: "text", 205 | text: fileContent, 206 | }); 207 | } 208 | }); 209 | 210 | return content; 211 | } 212 | 213 | function getValidAnnotation(annotation: JSONValue): Annotation { 214 | if ( 215 | !( 216 | annotation && 217 | typeof annotation === "object" && 218 | "type" in annotation && 219 | typeof annotation.type === "string" && 220 | "data" in annotation && 221 | annotation.data && 222 | typeof annotation.data === "object" 223 | ) 224 | ) { 225 | throw new Error("Client sent invalid annotation. Missing data and type"); 226 | } 227 | return { type: annotation.type, data: annotation.data }; 228 | } 229 | 230 | // validate and get all annotations of a specific type or role from the frontend messages 231 | export function getAnnotations< 232 | T extends Annotation["data"] = Annotation["data"], 233 | >( 234 | messages: Message[], 235 | options?: { 236 | role?: Message["role"]; // message role 237 | type?: Annotation["type"]; // annotation type 238 | }, 239 | ): { 240 | type: string; 241 | data: T; 242 | }[] { 243 | const messagesByRole = options?.role 244 | ? messages.filter((msg) => msg.role === options?.role) 245 | : messages; 246 | const annotations = getAllAnnotations(messagesByRole); 247 | const annotationsByType = options?.type 248 | ? annotations.filter((a) => a.type === options.type) 249 | : annotations; 250 | return annotationsByType as { type: string; data: T }[]; 251 | } 252 | -------------------------------------------------------------------------------- /app/api/chat/llamaindex/streaming/events.ts: -------------------------------------------------------------------------------- 1 | import { StreamData } from "ai"; 2 | import { 3 | CallbackManager, 4 | LLamaCloudFileService, 5 | Metadata, 6 | MetadataMode, 7 | NodeWithScore, 8 | ToolCall, 9 | ToolOutput, 10 | } from "llamaindex"; 11 | import path from "node:path"; 12 | import { DATA_DIR } from "../../engine/loader"; 13 | import { downloadFile } from "./file"; 14 | 15 | const LLAMA_CLOUD_DOWNLOAD_FOLDER = "output/llamacloud"; 16 | 17 | export function appendSourceData( 18 | data: StreamData, 19 | sourceNodes?: NodeWithScore[], 20 | ) { 21 | if (!sourceNodes?.length) return; 22 | try { 23 | const nodes = sourceNodes.map((node) => ({ 24 | metadata: node.node.metadata, 25 | id: node.node.id_, 26 | score: node.score ?? null, 27 | url: getNodeUrl(node.node.metadata), 28 | text: node.node.getContent(MetadataMode.NONE), 29 | })); 30 | data.appendMessageAnnotation({ 31 | type: "sources", 32 | data: { 33 | nodes, 34 | }, 35 | }); 36 | } catch (error) { 37 | console.error("Error appending source data:", error); 38 | } 39 | } 40 | 41 | export function appendEventData(data: StreamData, title?: string) { 42 | if (!title) return; 43 | data.appendMessageAnnotation({ 44 | type: "events", 45 | data: { 46 | title, 47 | }, 48 | }); 49 | } 50 | 51 | export function appendToolData( 52 | data: StreamData, 53 | toolCall: ToolCall, 54 | toolOutput: ToolOutput, 55 | ) { 56 | data.appendMessageAnnotation({ 57 | type: "tools", 58 | data: { 59 | toolCall: { 60 | id: toolCall.id, 61 | name: toolCall.name, 62 | input: toolCall.input, 63 | }, 64 | toolOutput: { 65 | output: toolOutput.output, 66 | isError: toolOutput.isError, 67 | }, 68 | }, 69 | }); 70 | } 71 | 72 | export function createCallbackManager(stream: StreamData) { 73 | const callbackManager = new CallbackManager(); 74 | 75 | callbackManager.on("retrieve-end", (data) => { 76 | const { nodes, query } = data.detail; 77 | appendSourceData(stream, nodes); 78 | appendEventData(stream, `Retrieving context for query: '${query.query}'`); 79 | appendEventData( 80 | stream, 81 | `Retrieved ${nodes.length} sources to use as context for the query`, 82 | ); 83 | downloadFilesFromNodes(nodes); // don't await to avoid blocking chat streaming 84 | }); 85 | 86 | callbackManager.on("llm-tool-call", (event) => { 87 | const { name, input } = event.detail.toolCall; 88 | const inputString = Object.entries(input) 89 | .map(([key, value]) => `${key}: ${value}`) 90 | .join(", "); 91 | appendEventData( 92 | stream, 93 | `Using tool: '${name}' with inputs: '${inputString}'`, 94 | ); 95 | }); 96 | 97 | callbackManager.on("llm-tool-result", (event) => { 98 | const { toolCall, toolResult } = event.detail; 99 | appendToolData(stream, toolCall, toolResult); 100 | }); 101 | 102 | return callbackManager; 103 | } 104 | 105 | function getNodeUrl(metadata: Metadata) { 106 | if (!process.env.FILESERVER_URL_PREFIX) { 107 | console.warn( 108 | "FILESERVER_URL_PREFIX is not set. File URLs will not be generated.", 109 | ); 110 | } 111 | const fileName = metadata["file_name"]; 112 | if (fileName && process.env.FILESERVER_URL_PREFIX) { 113 | // file_name exists and file server is configured 114 | const pipelineId = metadata["pipeline_id"]; 115 | if (pipelineId) { 116 | const name = toDownloadedName(pipelineId, fileName); 117 | return `${process.env.FILESERVER_URL_PREFIX}/${LLAMA_CLOUD_DOWNLOAD_FOLDER}/${name}`; 118 | } 119 | const isPrivate = metadata["private"] === "true"; 120 | if (isPrivate) { 121 | return `${process.env.FILESERVER_URL_PREFIX}/output/uploaded/${fileName}`; 122 | } 123 | const filePath = metadata["file_path"]; 124 | const dataDir = path.resolve(DATA_DIR); 125 | 126 | if (filePath && dataDir) { 127 | const relativePath = path.relative(dataDir, filePath); 128 | return `${process.env.FILESERVER_URL_PREFIX}/data/${relativePath}`; 129 | } 130 | } 131 | // fallback to URL in metadata (e.g. for websites) 132 | return metadata["URL"]; 133 | } 134 | 135 | async function downloadFilesFromNodes(nodes: NodeWithScore[]) { 136 | try { 137 | const files = nodesToLlamaCloudFiles(nodes); 138 | for (const { pipelineId, fileName, downloadedName } of files) { 139 | const downloadUrl = await LLamaCloudFileService.getFileUrl( 140 | pipelineId, 141 | fileName, 142 | ); 143 | if (downloadUrl) { 144 | await downloadFile( 145 | downloadUrl, 146 | downloadedName, 147 | LLAMA_CLOUD_DOWNLOAD_FOLDER, 148 | ); 149 | } 150 | } 151 | } catch (error) { 152 | console.error("Error downloading files from nodes:", error); 153 | } 154 | } 155 | 156 | function nodesToLlamaCloudFiles(nodes: NodeWithScore[]) { 157 | const files: Array<{ 158 | pipelineId: string; 159 | fileName: string; 160 | downloadedName: string; 161 | }> = []; 162 | for (const node of nodes) { 163 | const pipelineId = node.node.metadata["pipeline_id"]; 164 | const fileName = node.node.metadata["file_name"]; 165 | if (!pipelineId || !fileName) continue; 166 | const isDuplicate = files.some( 167 | (f) => f.pipelineId === pipelineId && f.fileName === fileName, 168 | ); 169 | if (!isDuplicate) { 170 | files.push({ 171 | pipelineId, 172 | fileName, 173 | downloadedName: toDownloadedName(pipelineId, fileName), 174 | }); 175 | } 176 | } 177 | return files; 178 | } 179 | 180 | function toDownloadedName(pipelineId: string, fileName: string) { 181 | return `${pipelineId}$${fileName}`; 182 | } 183 | -------------------------------------------------------------------------------- /app/api/chat/llamaindex/streaming/file.ts: -------------------------------------------------------------------------------- 1 | import fs from "node:fs"; 2 | import https from "node:https"; 3 | import path from "node:path"; 4 | 5 | export async function downloadFile( 6 | urlToDownload: string, 7 | filename: string, 8 | folder = "output/uploaded", 9 | ) { 10 | try { 11 | const downloadedPath = path.join(folder, filename); 12 | 13 | // Check if file already exists 14 | if (fs.existsSync(downloadedPath)) return; 15 | 16 | const file = fs.createWriteStream(downloadedPath); 17 | https 18 | .get(urlToDownload, (response) => { 19 | response.pipe(file); 20 | file.on("finish", () => { 21 | file.close(() => { 22 | console.log("File downloaded successfully"); 23 | }); 24 | }); 25 | }) 26 | .on("error", (err) => { 27 | fs.unlink(downloadedPath, () => { 28 | console.error("Error downloading file:", err); 29 | throw err; 30 | }); 31 | }); 32 | } catch (error) { 33 | throw new Error(`Error downloading file: ${error}`); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /app/api/chat/llamaindex/streaming/suggestion.ts: -------------------------------------------------------------------------------- 1 | import { ChatMessage, Settings } from "llamaindex"; 2 | 3 | export async function generateNextQuestions(conversation: ChatMessage[]) { 4 | const llm = Settings.llm; 5 | const NEXT_QUESTION_PROMPT = process.env.NEXT_QUESTION_PROMPT; 6 | if (!NEXT_QUESTION_PROMPT) { 7 | return []; 8 | } 9 | 10 | // Format conversation 11 | const conversationText = conversation 12 | .map((message) => `${message.role}: ${message.content}`) 13 | .join("\n"); 14 | const message = NEXT_QUESTION_PROMPT.replace( 15 | "{conversation}", 16 | conversationText, 17 | ); 18 | 19 | try { 20 | const response = await llm.complete({ prompt: message }); 21 | const questions = extractQuestions(response.text); 22 | return questions; 23 | } catch (error) { 24 | console.error("Error when generating the next questions: ", error); 25 | return []; 26 | } 27 | } 28 | 29 | // TODO: instead of parsing the LLM's result we can use structured predict, once LITS supports it 30 | function extractQuestions(text: string): string[] { 31 | // Extract the text inside the triple backticks 32 | // @ts-ignore 33 | const contentMatch = text.match(/```(.*?)```/s); 34 | const content = contentMatch ? contentMatch[1] : ""; 35 | 36 | // Split the content by newlines to get each question 37 | const questions = content 38 | .split("\n") 39 | .map((question) => question.trim()) 40 | .filter((question) => question !== ""); 41 | 42 | return questions; 43 | } 44 | -------------------------------------------------------------------------------- /app/api/chat/route.ts: -------------------------------------------------------------------------------- 1 | import { initObservability } from "@/app/observability"; 2 | import { LlamaIndexAdapter, Message, StreamData } from "ai"; 3 | import { ChatMessage, Settings } from "llamaindex"; 4 | import { NextRequest, NextResponse } from "next/server"; 5 | import { createChatEngine } from "./engine/chat"; 6 | import { initSettings } from "./engine/settings"; 7 | import { 8 | isValidMessages, 9 | retrieveDocumentIds, 10 | retrieveMessageContent, 11 | } from "./llamaindex/streaming/annotations"; 12 | import { createCallbackManager } from "./llamaindex/streaming/events"; 13 | import { generateNextQuestions } from "./llamaindex/streaming/suggestion"; 14 | 15 | initObservability(); 16 | initSettings(); 17 | 18 | export const runtime = "nodejs"; 19 | export const dynamic = "force-dynamic"; 20 | 21 | export async function POST(request: NextRequest) { 22 | // Init Vercel AI StreamData and timeout 23 | const vercelStreamData = new StreamData(); 24 | 25 | try { 26 | const body = await request.json(); 27 | const { messages, data }: { messages: Message[]; data?: any } = body; 28 | if (!isValidMessages(messages)) { 29 | return NextResponse.json( 30 | { 31 | error: 32 | "messages are required in the request body and the last message must be from the user", 33 | }, 34 | { status: 400 }, 35 | ); 36 | } 37 | 38 | // retrieve document ids from the annotations of all messages (if any) 39 | const ids = retrieveDocumentIds(messages); 40 | // create chat engine with index using the document ids 41 | const chatEngine = await createChatEngine(ids, data); 42 | 43 | // retrieve user message content from Vercel/AI format 44 | const userMessageContent = retrieveMessageContent(messages); 45 | 46 | // Setup callbacks 47 | const callbackManager = createCallbackManager(vercelStreamData); 48 | const chatHistory: ChatMessage[] = messages.slice(0, -1) as ChatMessage[]; 49 | 50 | // Calling LlamaIndex's ChatEngine to get a streamed response 51 | const response = await Settings.withCallbackManager(callbackManager, () => { 52 | return chatEngine.chat({ 53 | message: userMessageContent, 54 | chatHistory, 55 | stream: true, 56 | }); 57 | }); 58 | 59 | const onCompletion = (content: string) => { 60 | chatHistory.push({ role: "assistant", content: content }); 61 | generateNextQuestions(chatHistory) 62 | .then((questions: string[]) => { 63 | if (questions.length > 0) { 64 | vercelStreamData.appendMessageAnnotation({ 65 | type: "suggested_questions", 66 | data: questions, 67 | }); 68 | } 69 | }) 70 | .finally(() => { 71 | vercelStreamData.close(); 72 | }); 73 | }; 74 | 75 | return LlamaIndexAdapter.toDataStreamResponse(response, { 76 | data: vercelStreamData, 77 | callbacks: { onCompletion }, 78 | }); 79 | } catch (error) { 80 | console.error("[LlamaIndex]", error); 81 | return NextResponse.json( 82 | { 83 | detail: (error as Error).message, 84 | }, 85 | { 86 | status: 500, 87 | }, 88 | ); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /app/api/chat/upload/route.ts: -------------------------------------------------------------------------------- 1 | import { NextRequest, NextResponse } from "next/server"; 2 | import { getDataSource } from "../engine"; 3 | import { initSettings } from "../engine/settings"; 4 | import { uploadDocument } from "../llamaindex/documents/upload"; 5 | 6 | initSettings(); 7 | 8 | export const runtime = "nodejs"; 9 | export const dynamic = "force-dynamic"; 10 | 11 | export async function POST(request: NextRequest) { 12 | try { 13 | const { 14 | name, 15 | base64, 16 | params, 17 | }: { 18 | name: string; 19 | base64: string; 20 | params?: any; 21 | } = await request.json(); 22 | if (!base64 || !name) { 23 | return NextResponse.json( 24 | { error: "base64 and name is required in the request body" }, 25 | { status: 400 }, 26 | ); 27 | } 28 | const index = await getDataSource(params); 29 | const documentFile = await uploadDocument(index, name, base64); 30 | return NextResponse.json(documentFile); 31 | } catch (error) { 32 | console.error("[Upload API]", error); 33 | return NextResponse.json( 34 | { error: (error as Error).message }, 35 | { status: 500 }, 36 | ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /app/api/files/[...slug]/route.ts: -------------------------------------------------------------------------------- 1 | import { readFile } from "fs/promises"; 2 | import { NextRequest, NextResponse } from "next/server"; 3 | import path from "path"; 4 | import { DATA_DIR } from "../../chat/engine/loader"; 5 | 6 | /** 7 | * This API is to get file data from allowed folders 8 | * It receives path slug and response file data like serve static file 9 | */ 10 | export async function GET( 11 | _request: NextRequest, 12 | { params }: { params: Promise<{ slug: string[] }> }, 13 | ) { 14 | const slug = (await params).slug; 15 | 16 | if (!slug) { 17 | return NextResponse.json({ detail: "Missing file slug" }, { status: 400 }); 18 | } 19 | 20 | if (slug.includes("..") || path.isAbsolute(path.join(...slug))) { 21 | return NextResponse.json({ detail: "Invalid file path" }, { status: 400 }); 22 | } 23 | 24 | const [folder, ...pathTofile] = slug; // data, file.pdf 25 | const allowedFolders = ["data", "output"]; 26 | 27 | if (!allowedFolders.includes(folder)) { 28 | return NextResponse.json({ detail: "No permission" }, { status: 400 }); 29 | } 30 | 31 | try { 32 | const filePath = path.join( 33 | process.cwd(), 34 | folder === "data" ? DATA_DIR : folder, 35 | path.join(...pathTofile), 36 | ); 37 | const blob = await readFile(filePath); 38 | 39 | return new NextResponse(blob, { 40 | status: 200, 41 | statusText: "OK", 42 | headers: { 43 | "Content-Length": blob.byteLength.toString(), 44 | }, 45 | }); 46 | } catch (error) { 47 | console.error(error); 48 | return NextResponse.json({ detail: "File not found" }, { status: 404 }); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /app/api/sandbox/route.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 FoundryLabs, Inc. 3 | * Portions of this file are copied from the e2b project (https://github.com/e2b-dev/ai-artifacts) 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | import { ExecutionError, Result, Sandbox } from "@e2b/code-interpreter"; 17 | import fs from "node:fs/promises"; 18 | import path from "node:path"; 19 | import { saveDocument } from "../chat/llamaindex/documents/helper"; 20 | 21 | type CodeArtifact = { 22 | commentary: string; 23 | template: string; 24 | title: string; 25 | description: string; 26 | additional_dependencies: string[]; 27 | has_additional_dependencies: boolean; 28 | install_dependencies_command: string; 29 | port: number | null; 30 | file_path: string; 31 | code: string; 32 | files?: string[]; 33 | }; 34 | 35 | const sandboxTimeout = 10 * 60 * 1000; // 10 minute in ms 36 | 37 | export const maxDuration = 60; 38 | 39 | const OUTPUT_DIR = path.join("output", "tools"); 40 | 41 | export type ExecutionResult = { 42 | template: string; 43 | stdout: string[]; 44 | stderr: string[]; 45 | runtimeError?: ExecutionError; 46 | outputUrls: Array<{ url: string; filename: string }>; 47 | url: string; 48 | }; 49 | 50 | // see https://github.com/e2b-dev/fragments/tree/main/sandbox-templates 51 | const SUPPORTED_TEMPLATES = [ 52 | "nextjs-developer", 53 | "vue-developer", 54 | "streamlit-developer", 55 | "gradio-developer", 56 | ]; 57 | 58 | export async function POST(req: Request) { 59 | const { artifact }: { artifact: CodeArtifact } = await req.json(); 60 | 61 | let sbx: Sandbox; 62 | const sandboxOpts = { 63 | metadata: { template: artifact.template, userID: "default" }, 64 | timeoutMs: sandboxTimeout, 65 | }; 66 | if (SUPPORTED_TEMPLATES.includes(artifact.template)) { 67 | sbx = await Sandbox.create(artifact.template, sandboxOpts); 68 | } else { 69 | sbx = await Sandbox.create(sandboxOpts); 70 | } 71 | console.log("Created sandbox", sbx.sandboxId); 72 | 73 | // Install packages 74 | if (artifact.has_additional_dependencies) { 75 | await sbx.commands.run(artifact.install_dependencies_command); 76 | console.log( 77 | `Installed dependencies: ${artifact.additional_dependencies.join(", ")} in sandbox ${sbx.sandboxId}`, 78 | ); 79 | } 80 | 81 | // Copy files 82 | if (artifact.files) { 83 | artifact.files.forEach(async (sandboxFilePath) => { 84 | const fileName = path.basename(sandboxFilePath); 85 | const localFilePath = path.join("output", "uploaded", fileName); 86 | const fileContent = await fs.readFile(localFilePath); 87 | 88 | const arrayBuffer = new Uint8Array(fileContent).buffer; 89 | await sbx.files.write(sandboxFilePath, arrayBuffer); 90 | console.log(`Copied file to ${sandboxFilePath} in ${sbx.sandboxId}`); 91 | }); 92 | } 93 | 94 | // Copy code to fs 95 | if (artifact.code && Array.isArray(artifact.code)) { 96 | artifact.code.forEach(async (file) => { 97 | await sbx.files.write(file.file_path, file.file_content); 98 | console.log(`Copied file to ${file.file_path} in ${sbx.sandboxId}`); 99 | }); 100 | } else { 101 | await sbx.files.write(artifact.file_path, artifact.code); 102 | console.log(`Copied file to ${artifact.file_path} in ${sbx.sandboxId}`); 103 | } 104 | 105 | // Execute code or return a URL to the running sandbox 106 | if (artifact.template === "code-interpreter-multilang") { 107 | const result = await sbx.runCode(artifact.code || ""); 108 | await sbx.kill(); 109 | const outputUrls = await downloadCellResults(result.results); 110 | return new Response( 111 | JSON.stringify({ 112 | template: artifact.template, 113 | stdout: result.logs.stdout, 114 | stderr: result.logs.stderr, 115 | runtimeError: result.error, 116 | outputUrls: outputUrls, 117 | }), 118 | ); 119 | } else { 120 | return new Response( 121 | JSON.stringify({ 122 | template: artifact.template, 123 | url: `https://${sbx?.getHost(artifact.port || 80)}`, 124 | }), 125 | ); 126 | } 127 | } 128 | 129 | async function downloadCellResults( 130 | cellResults?: Result[], 131 | ): Promise> { 132 | if (!cellResults) return []; 133 | 134 | const results = await Promise.all( 135 | cellResults.map(async (res) => { 136 | const formats = res.formats(); // available formats in the result 137 | const formatResults = await Promise.all( 138 | formats 139 | .filter((ext) => ["png", "svg", "jpeg", "pdf"].includes(ext)) 140 | .map(async (ext) => { 141 | const filename = `${crypto.randomUUID()}.${ext}`; 142 | const base64 = res[ext as keyof Result]; 143 | const buffer = Buffer.from(base64, "base64"); 144 | const fileurl = await saveDocument( 145 | path.join(OUTPUT_DIR, filename), 146 | buffer, 147 | ); 148 | return { url: fileurl, filename }; 149 | }), 150 | ); 151 | return formatResults; 152 | }), 153 | ); 154 | return results.flat(); 155 | } 156 | -------------------------------------------------------------------------------- /app/components/chat-section.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { ChatSection as ChatSectionUI } from "@llamaindex/chat-ui"; 4 | import "@llamaindex/chat-ui/styles/markdown.css"; 5 | import "@llamaindex/chat-ui/styles/pdf.css"; 6 | import { useChat } from "ai/react"; 7 | import CustomChatInput from "./ui/chat/chat-input"; 8 | import CustomChatMessages from "./ui/chat/chat-messages"; 9 | import { useClientConfig } from "./ui/chat/hooks/use-config"; 10 | 11 | export default function ChatSection() { 12 | const { backend } = useClientConfig(); 13 | const handler = useChat({ 14 | api: `${backend}/api/chat`, 15 | onError: (error: unknown) => { 16 | if (!(error instanceof Error)) throw error; 17 | let errorMessage: string; 18 | try { 19 | errorMessage = JSON.parse(error.message).detail; 20 | } catch (e) { 21 | errorMessage = error.message; 22 | } 23 | alert(errorMessage); 24 | }, 25 | }); 26 | return ( 27 | 28 | 29 | 30 | 31 | ); 32 | } 33 | -------------------------------------------------------------------------------- /app/components/header.tsx: -------------------------------------------------------------------------------- 1 | import Image from "next/image"; 2 | 3 | export default function Header() { 4 | return ( 5 |
6 |

7 | Get started by editing  8 | app/page.tsx 9 |

10 | 26 |
27 | ); 28 | } 29 | -------------------------------------------------------------------------------- /app/components/ui/accordion.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import * as AccordionPrimitive from "@radix-ui/react-accordion"; 4 | import { ChevronDown } from "lucide-react"; 5 | import * as React from "react"; 6 | import { cn } from "./lib/utils"; 7 | 8 | const Accordion = AccordionPrimitive.Root; 9 | 10 | const AccordionItem = React.forwardRef< 11 | React.ElementRef, 12 | React.ComponentPropsWithoutRef 13 | >(({ className, ...props }, ref) => ( 14 | 19 | )); 20 | AccordionItem.displayName = "AccordionItem"; 21 | 22 | const AccordionTrigger = React.forwardRef< 23 | React.ElementRef, 24 | React.ComponentPropsWithoutRef 25 | >(({ className, children, ...props }, ref) => ( 26 | 27 | svg]:rotate-180", 31 | className, 32 | )} 33 | {...props} 34 | > 35 | {children} 36 | 37 | 38 | 39 | )); 40 | AccordionTrigger.displayName = AccordionPrimitive.Trigger.displayName; 41 | 42 | const AccordionContent = React.forwardRef< 43 | React.ElementRef, 44 | React.ComponentPropsWithoutRef 45 | >(({ className, children, ...props }, ref) => ( 46 | 51 |
{children}
52 | 53 | )); 54 | AccordionContent.displayName = AccordionPrimitive.Content.displayName; 55 | 56 | export { Accordion, AccordionContent, AccordionItem, AccordionTrigger }; 57 | -------------------------------------------------------------------------------- /app/components/ui/button.tsx: -------------------------------------------------------------------------------- 1 | import { Slot } from "@radix-ui/react-slot"; 2 | import { cva, type VariantProps } from "class-variance-authority"; 3 | import * as React from "react"; 4 | 5 | import { cn } from "./lib/utils"; 6 | 7 | const buttonVariants = cva( 8 | "inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50", 9 | { 10 | variants: { 11 | variant: { 12 | default: "bg-primary text-primary-foreground hover:bg-primary/90", 13 | destructive: 14 | "bg-destructive text-destructive-foreground hover:bg-destructive/90", 15 | outline: 16 | "border border-input bg-background hover:bg-accent hover:text-accent-foreground", 17 | secondary: 18 | "bg-secondary text-secondary-foreground hover:bg-secondary/80", 19 | ghost: "hover:bg-accent hover:text-accent-foreground", 20 | link: "text-primary underline-offset-4 hover:underline", 21 | }, 22 | size: { 23 | default: "h-10 px-4 py-2", 24 | sm: "h-9 rounded-md px-3", 25 | lg: "h-11 rounded-md px-8", 26 | icon: "h-10 w-10", 27 | }, 28 | }, 29 | defaultVariants: { 30 | variant: "default", 31 | size: "default", 32 | }, 33 | }, 34 | ); 35 | 36 | export interface ButtonProps 37 | extends React.ButtonHTMLAttributes, 38 | VariantProps { 39 | asChild?: boolean; 40 | } 41 | 42 | const Button = React.forwardRef( 43 | ({ className, variant, size, asChild = false, ...props }, ref) => { 44 | const Comp = asChild ? Slot : "button"; 45 | return ( 46 | 51 | ); 52 | }, 53 | ); 54 | Button.displayName = "Button"; 55 | 56 | export { Button, buttonVariants }; 57 | -------------------------------------------------------------------------------- /app/components/ui/card.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import { cn } from "./lib/utils"; 3 | 4 | const Card = React.forwardRef< 5 | HTMLDivElement, 6 | React.HTMLAttributes 7 | >(({ className, ...props }, ref) => ( 8 |
16 | )); 17 | Card.displayName = "Card"; 18 | 19 | const CardHeader = React.forwardRef< 20 | HTMLDivElement, 21 | React.HTMLAttributes 22 | >(({ className, ...props }, ref) => ( 23 |
28 | )); 29 | CardHeader.displayName = "CardHeader"; 30 | 31 | const CardTitle = React.forwardRef< 32 | HTMLDivElement, 33 | React.HTMLAttributes 34 | >(({ className, ...props }, ref) => ( 35 |
40 | )); 41 | CardTitle.displayName = "CardTitle"; 42 | 43 | const CardDescription = React.forwardRef< 44 | HTMLDivElement, 45 | React.HTMLAttributes 46 | >(({ className, ...props }, ref) => ( 47 |
52 | )); 53 | CardDescription.displayName = "CardDescription"; 54 | 55 | const CardContent = React.forwardRef< 56 | HTMLDivElement, 57 | React.HTMLAttributes 58 | >(({ className, ...props }, ref) => ( 59 |
60 | )); 61 | CardContent.displayName = "CardContent"; 62 | 63 | const CardFooter = React.forwardRef< 64 | HTMLDivElement, 65 | React.HTMLAttributes 66 | >(({ className, ...props }, ref) => ( 67 |
72 | )); 73 | CardFooter.displayName = "CardFooter"; 74 | 75 | export { 76 | Card, 77 | CardContent, 78 | CardDescription, 79 | CardFooter, 80 | CardHeader, 81 | CardTitle, 82 | }; 83 | -------------------------------------------------------------------------------- /app/components/ui/chat/chat-avatar.tsx: -------------------------------------------------------------------------------- 1 | import { useChatMessage } from "@llamaindex/chat-ui"; 2 | import { User2 } from "lucide-react"; 3 | import Image from "next/image"; 4 | 5 | export function ChatMessageAvatar() { 6 | const { message } = useChatMessage(); 7 | if (message.role === "user") { 8 | return ( 9 |
10 | 11 |
12 | ); 13 | } 14 | 15 | return ( 16 |
17 | Llama Logo 25 |
26 | ); 27 | } 28 | -------------------------------------------------------------------------------- /app/components/ui/chat/chat-input.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { ChatInput, useChatUI, useFile } from "@llamaindex/chat-ui"; 4 | import { DocumentInfo, ImagePreview } from "@llamaindex/chat-ui/widgets"; 5 | import { LlamaCloudSelector } from "./custom/llama-cloud-selector"; 6 | import { useClientConfig } from "./hooks/use-config"; 7 | 8 | export default function CustomChatInput() { 9 | const { requestData, isLoading, input } = useChatUI(); 10 | const { backend } = useClientConfig(); 11 | const { 12 | imageUrl, 13 | setImageUrl, 14 | uploadFile, 15 | files, 16 | removeDoc, 17 | reset, 18 | getAnnotations, 19 | } = useFile({ uploadAPI: `${backend}/api/chat/upload` }); 20 | 21 | /** 22 | * Handles file uploads. Overwrite to hook into the file upload behavior. 23 | * @param file The file to upload 24 | */ 25 | const handleUploadFile = async (file: File) => { 26 | // There's already an image uploaded, only allow one image at a time 27 | if (imageUrl) { 28 | alert("You can only upload one image at a time."); 29 | return; 30 | } 31 | 32 | try { 33 | // Upload the file and send with it the current request data 34 | await uploadFile(file, requestData); 35 | } catch (error: any) { 36 | // Show error message if upload fails 37 | alert(error.message); 38 | } 39 | }; 40 | 41 | // Get references to the upload files in message annotations format, see https://github.com/run-llama/chat-ui/blob/main/packages/chat-ui/src/hook/use-file.tsx#L56 42 | const annotations = getAnnotations(); 43 | 44 | return ( 45 | 50 |
51 | {/* Image preview section */} 52 | {imageUrl && ( 53 | setImageUrl(null)} /> 54 | )} 55 | {/* Document previews section */} 56 | {files.length > 0 && ( 57 |
58 | {files.map((file) => ( 59 | removeDoc(file)} 64 | /> 65 | ))} 66 |
67 | )} 68 |
69 | 70 | 71 | 72 | 73 | 78 | 79 | 80 | ); 81 | } 82 | -------------------------------------------------------------------------------- /app/components/ui/chat/chat-message-content.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | ChatMessage, 3 | ContentPosition, 4 | getSourceAnnotationData, 5 | useChatMessage, 6 | useChatUI, 7 | } from "@llamaindex/chat-ui"; 8 | import { DeepResearchCard } from "./custom/deep-research-card"; 9 | import { Markdown } from "./custom/markdown"; 10 | import { ToolAnnotations } from "./tools/chat-tools"; 11 | 12 | export function ChatMessageContent() { 13 | const { isLoading, append } = useChatUI(); 14 | const { message } = useChatMessage(); 15 | const customContent = [ 16 | { 17 | // override the default markdown component 18 | position: ContentPosition.MARKDOWN, 19 | component: ( 20 | 24 | ), 25 | }, 26 | // add the deep research card 27 | { 28 | position: ContentPosition.CHAT_EVENTS, 29 | component: , 30 | }, 31 | { 32 | // add the tool annotations after events 33 | position: ContentPosition.AFTER_EVENTS, 34 | component: , 35 | }, 36 | ]; 37 | return ( 38 | 43 | ); 44 | } 45 | -------------------------------------------------------------------------------- /app/components/ui/chat/chat-messages.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { ChatMessage, ChatMessages, useChatUI } from "@llamaindex/chat-ui"; 4 | import { ChatMessageAvatar } from "./chat-avatar"; 5 | import { ChatMessageContent } from "./chat-message-content"; 6 | import { ChatStarter } from "./chat-starter"; 7 | 8 | export default function CustomChatMessages() { 9 | const { messages } = useChatUI(); 10 | return ( 11 | 12 | 13 | {messages.map((message, index) => ( 14 | 19 | 20 | 21 | 22 | 23 | ))} 24 | 25 | 26 | 27 | 28 | 29 | ); 30 | } 31 | -------------------------------------------------------------------------------- /app/components/ui/chat/chat-starter.tsx: -------------------------------------------------------------------------------- 1 | import { useChatUI } from "@llamaindex/chat-ui"; 2 | import { StarterQuestions } from "@llamaindex/chat-ui/widgets"; 3 | import { useEffect, useState } from "react"; 4 | import { useClientConfig } from "./hooks/use-config"; 5 | 6 | export function ChatStarter() { 7 | const { append } = useChatUI(); 8 | const { backend } = useClientConfig(); 9 | const [starterQuestions, setStarterQuestions] = useState(); 10 | 11 | useEffect(() => { 12 | if (!starterQuestions) { 13 | fetch(`${backend}/api/chat/config`) 14 | .then((response) => response.json()) 15 | .then((data) => { 16 | if (data?.starterQuestions) { 17 | setStarterQuestions(data.starterQuestions); 18 | } 19 | }) 20 | .catch((error) => console.error("Error fetching config", error)); 21 | } 22 | }, [starterQuestions, backend]); 23 | 24 | if (!starterQuestions?.length) return null; 25 | return ; 26 | } 27 | -------------------------------------------------------------------------------- /app/components/ui/chat/custom/deep-research-card.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { Message } from "@llamaindex/chat-ui"; 4 | import { 5 | AlertCircle, 6 | CheckCircle2, 7 | CircleDashed, 8 | Clock, 9 | NotebookPen, 10 | Search, 11 | } from "lucide-react"; 12 | import { useMemo } from "react"; 13 | import { 14 | Accordion, 15 | AccordionContent, 16 | AccordionItem, 17 | AccordionTrigger, 18 | } from "../../accordion"; 19 | import { Card, CardContent, CardHeader, CardTitle } from "../../card"; 20 | import { cn } from "../../lib/utils"; 21 | import { Markdown } from "./markdown"; 22 | 23 | // Streaming event types 24 | type EventState = "pending" | "inprogress" | "done" | "error"; 25 | 26 | type DeepResearchEvent = { 27 | type: "deep_research_event"; 28 | data: { 29 | event: "retrieve" | "analyze" | "answer"; 30 | state: EventState; 31 | id?: string; 32 | question?: string; 33 | answer?: string | null; 34 | }; 35 | }; 36 | 37 | // UI state types 38 | type QuestionState = { 39 | id: string; 40 | question: string; 41 | answer: string | null; 42 | state: EventState; 43 | isOpen: boolean; 44 | }; 45 | 46 | type DeepResearchCardState = { 47 | retrieve: { 48 | state: EventState | null; 49 | }; 50 | analyze: { 51 | state: EventState | null; 52 | questions: QuestionState[]; 53 | }; 54 | }; 55 | 56 | interface DeepResearchCardProps { 57 | message: Message; 58 | className?: string; 59 | } 60 | 61 | const stateIcon: Record = { 62 | pending: , 63 | inprogress: , 64 | done: , 65 | error: , 66 | }; 67 | 68 | // Transform the state based on the event without mutations 69 | const transformState = ( 70 | state: DeepResearchCardState, 71 | event: DeepResearchEvent, 72 | ): DeepResearchCardState => { 73 | switch (event.data.event) { 74 | case "answer": { 75 | const { id, question, answer } = event.data; 76 | if (!id || !question) return state; 77 | 78 | const updatedQuestions = state.analyze.questions.map((q) => { 79 | if (q.id !== id) return q; 80 | return { 81 | ...q, 82 | state: event.data.state, 83 | answer: answer ?? q.answer, 84 | }; 85 | }); 86 | 87 | const newQuestion = !state.analyze.questions.some((q) => q.id === id) 88 | ? [ 89 | { 90 | id, 91 | question, 92 | answer: answer ?? null, 93 | state: event.data.state, 94 | isOpen: false, 95 | }, 96 | ] 97 | : []; 98 | 99 | return { 100 | ...state, 101 | analyze: { 102 | ...state.analyze, 103 | questions: [...updatedQuestions, ...newQuestion], 104 | }, 105 | }; 106 | } 107 | 108 | case "retrieve": 109 | case "analyze": 110 | return { 111 | ...state, 112 | [event.data.event]: { 113 | ...state[event.data.event], 114 | state: event.data.state, 115 | }, 116 | }; 117 | 118 | default: 119 | return state; 120 | } 121 | }; 122 | 123 | // Convert deep research events to state 124 | const deepResearchEventsToState = ( 125 | events: DeepResearchEvent[] | undefined, 126 | ): DeepResearchCardState => { 127 | if (!events?.length) { 128 | return { 129 | retrieve: { state: null }, 130 | analyze: { state: null, questions: [] }, 131 | }; 132 | } 133 | 134 | const initialState: DeepResearchCardState = { 135 | retrieve: { state: null }, 136 | analyze: { state: null, questions: [] }, 137 | }; 138 | 139 | return events.reduce( 140 | (acc: DeepResearchCardState, event: DeepResearchEvent) => 141 | transformState(acc, event), 142 | initialState, 143 | ); 144 | }; 145 | 146 | export function DeepResearchCard({ 147 | message, 148 | className, 149 | }: DeepResearchCardProps) { 150 | const deepResearchEvents = message.annotations as 151 | | DeepResearchEvent[] 152 | | undefined; 153 | const hasDeepResearchEvents = deepResearchEvents?.some( 154 | (event) => event.type === "deep_research_event", 155 | ); 156 | 157 | const state = useMemo( 158 | () => deepResearchEventsToState(deepResearchEvents), 159 | [deepResearchEvents], 160 | ); 161 | 162 | if (!hasDeepResearchEvents) { 163 | return null; 164 | } 165 | 166 | return ( 167 | 168 | 169 | {state.retrieve.state !== null && ( 170 | 171 | 172 | {state.retrieve.state === "inprogress" 173 | ? "Searching..." 174 | : "Search completed"} 175 | 176 | )} 177 | {state.analyze.state !== null && ( 178 | 179 | 180 | {state.analyze.state === "inprogress" ? "Analyzing..." : "Analysis"} 181 | 182 | )} 183 | 184 | 185 | 186 | {state.analyze.questions.length > 0 && ( 187 | 188 | {state.analyze.questions.map((question: QuestionState) => ( 189 | 194 | 195 |
196 |
197 | {stateIcon[question.state]} 198 |
199 | 200 | {question.question} 201 | 202 |
203 | 204 | {question.answer && ( 205 | 206 | 207 | 208 | )} 209 | 210 | ))} 211 | 212 | )} 213 | 214 | 215 | ); 216 | } 217 | -------------------------------------------------------------------------------- /app/components/ui/chat/custom/llama-cloud-selector.tsx: -------------------------------------------------------------------------------- 1 | import { useChatUI } from "@llamaindex/chat-ui"; 2 | import { Loader2 } from "lucide-react"; 3 | import { useCallback, useEffect, useState } from "react"; 4 | import { 5 | Select, 6 | SelectContent, 7 | SelectGroup, 8 | SelectItem, 9 | SelectLabel, 10 | SelectTrigger, 11 | SelectValue, 12 | } from "../../select"; 13 | import { useClientConfig } from "../hooks/use-config"; 14 | 15 | type LLamaCloudPipeline = { 16 | id: string; 17 | name: string; 18 | }; 19 | 20 | type LLamaCloudProject = { 21 | id: string; 22 | organization_id: string; 23 | name: string; 24 | is_default: boolean; 25 | pipelines: Array; 26 | }; 27 | 28 | type PipelineConfig = { 29 | project: string; // project name 30 | pipeline: string; // pipeline name 31 | }; 32 | 33 | type LlamaCloudConfig = { 34 | projects?: LLamaCloudProject[]; 35 | pipeline?: PipelineConfig; 36 | }; 37 | 38 | export interface LlamaCloudSelectorProps { 39 | onSelect?: (pipeline: PipelineConfig | undefined) => void; 40 | defaultPipeline?: PipelineConfig; 41 | shouldCheckValid?: boolean; 42 | } 43 | 44 | export function LlamaCloudSelector({ 45 | onSelect, 46 | defaultPipeline, 47 | shouldCheckValid = false, 48 | }: LlamaCloudSelectorProps) { 49 | const { backend } = useClientConfig(); 50 | const { setRequestData } = useChatUI(); 51 | const [config, setConfig] = useState(); 52 | 53 | const updateRequestParams = useCallback( 54 | (pipeline?: PipelineConfig) => { 55 | if (setRequestData) { 56 | setRequestData({ 57 | llamaCloudPipeline: pipeline, 58 | }); 59 | } else { 60 | onSelect?.(pipeline); 61 | } 62 | }, 63 | [onSelect, setRequestData], 64 | ); 65 | 66 | useEffect(() => { 67 | if (process.env.NEXT_PUBLIC_USE_LLAMACLOUD === "true" && !config) { 68 | fetch(`${backend}/api/chat/config/llamacloud`) 69 | .then((response) => { 70 | if (!response.ok) { 71 | return response.json().then((errorData) => { 72 | window.alert( 73 | `Error: ${JSON.stringify(errorData) || "Unknown error occurred"}`, 74 | ); 75 | }); 76 | } 77 | return response.json(); 78 | }) 79 | .then((data) => { 80 | const pipeline = defaultPipeline ?? data.pipeline; // defaultPipeline will override pipeline in .env 81 | setConfig({ ...data, pipeline }); 82 | updateRequestParams(pipeline); 83 | }) 84 | .catch((error) => console.error("Error fetching config", error)); 85 | } 86 | }, [backend, config, defaultPipeline, updateRequestParams]); 87 | 88 | const setPipeline = (pipelineConfig?: PipelineConfig) => { 89 | setConfig((prevConfig: any) => ({ 90 | ...prevConfig, 91 | pipeline: pipelineConfig, 92 | })); 93 | updateRequestParams(pipelineConfig); 94 | }; 95 | 96 | const handlePipelineSelect = async (value: string) => { 97 | setPipeline(JSON.parse(value) as PipelineConfig); 98 | }; 99 | 100 | if (process.env.NEXT_PUBLIC_USE_LLAMACLOUD !== "true") { 101 | return null; 102 | } 103 | 104 | if (!config) { 105 | return ( 106 |
107 | 108 |
109 | ); 110 | } 111 | 112 | if (shouldCheckValid && !isValid(config.projects, config.pipeline)) { 113 | return ( 114 |

115 | Invalid LlamaCloud configuration. Check console logs. 116 |

117 | ); 118 | } 119 | const { projects, pipeline } = config; 120 | 121 | return ( 122 | 155 | ); 156 | } 157 | 158 | function isValid( 159 | projects: LLamaCloudProject[] | undefined, 160 | pipeline: PipelineConfig | undefined, 161 | logErrors: boolean = true, 162 | ): boolean { 163 | if (!projects?.length) return false; 164 | if (!pipeline) return false; 165 | const matchedProject = projects.find( 166 | (project: LLamaCloudProject) => project.name === pipeline.project, 167 | ); 168 | if (!matchedProject) { 169 | if (logErrors) { 170 | console.error( 171 | `LlamaCloud project ${pipeline.project} not found. Check LLAMA_CLOUD_PROJECT_NAME variable`, 172 | ); 173 | } 174 | return false; 175 | } 176 | const pipelineExists = matchedProject.pipelines.some( 177 | (p) => p.name === pipeline.pipeline, 178 | ); 179 | if (!pipelineExists) { 180 | if (logErrors) { 181 | console.error( 182 | `LlamaCloud pipeline ${pipeline.pipeline} not found. Check LLAMA_CLOUD_INDEX_NAME variable`, 183 | ); 184 | } 185 | return false; 186 | } 187 | return true; 188 | } 189 | -------------------------------------------------------------------------------- /app/components/ui/chat/custom/markdown.tsx: -------------------------------------------------------------------------------- 1 | import { SourceData } from "@llamaindex/chat-ui"; 2 | import { Markdown as MarkdownUI } from "@llamaindex/chat-ui/widgets"; 3 | import { useClientConfig } from "../hooks/use-config"; 4 | 5 | const preprocessMedia = (content: string) => { 6 | // Remove `sandbox:` from the beginning of the URL before rendering markdown 7 | // OpenAI models sometimes prepend `sandbox:` to relative URLs - this fixes it 8 | return content.replace(/(sandbox|attachment|snt):/g, ""); 9 | }; 10 | 11 | export function Markdown({ 12 | content, 13 | sources, 14 | }: { 15 | content: string; 16 | sources?: SourceData; 17 | }) { 18 | const { backend } = useClientConfig(); 19 | const processedContent = preprocessMedia(content); 20 | return ( 21 | 26 | ); 27 | } 28 | -------------------------------------------------------------------------------- /app/components/ui/chat/hooks/use-config.ts: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | export interface ChatConfig { 4 | backend?: string; 5 | } 6 | 7 | function getBackendOrigin(): string { 8 | const chatAPI = process.env.NEXT_PUBLIC_CHAT_API; 9 | if (chatAPI) { 10 | return new URL(chatAPI).origin; 11 | } else { 12 | if (typeof window !== "undefined") { 13 | // Use BASE_URL from window.ENV 14 | return (window as any).ENV?.BASE_URL || ""; 15 | } 16 | return ""; 17 | } 18 | } 19 | 20 | export function useClientConfig(): ChatConfig { 21 | return { 22 | backend: getBackendOrigin(), 23 | }; 24 | } 25 | -------------------------------------------------------------------------------- /app/components/ui/chat/hooks/use-copy-to-clipboard.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import * as React from "react"; 4 | 5 | export interface useCopyToClipboardProps { 6 | timeout?: number; 7 | } 8 | 9 | export function useCopyToClipboard({ 10 | timeout = 2000, 11 | }: useCopyToClipboardProps) { 12 | const [isCopied, setIsCopied] = React.useState(false); 13 | 14 | const copyToClipboard = (value: string) => { 15 | if (typeof window === "undefined" || !navigator.clipboard?.writeText) { 16 | return; 17 | } 18 | 19 | if (!value) { 20 | return; 21 | } 22 | 23 | navigator.clipboard.writeText(value).then(() => { 24 | setIsCopied(true); 25 | 26 | setTimeout(() => { 27 | setIsCopied(false); 28 | }, timeout); 29 | }); 30 | }; 31 | 32 | return { isCopied, copyToClipboard }; 33 | } 34 | -------------------------------------------------------------------------------- /app/components/ui/chat/tools/chat-tools.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Message, 3 | MessageAnnotation, 4 | getAnnotationData, 5 | useChatUI, 6 | } from "@llamaindex/chat-ui"; 7 | import { JSONValue } from "ai"; 8 | import { useMemo } from "react"; 9 | import { Artifact, CodeArtifact } from "./artifact"; 10 | import { WeatherCard, WeatherData } from "./weather-card"; 11 | 12 | export function ToolAnnotations({ message }: { message: Message }) { 13 | // TODO: This is a bit of a hack to get the artifact version. better to generate the version in the tool call and 14 | // store it in CodeArtifact 15 | const { messages } = useChatUI(); 16 | const artifactVersion = useMemo( 17 | () => getArtifactVersion(messages, message), 18 | [messages, message], 19 | ); 20 | // Get the tool data from the message annotations 21 | const annotations = message.annotations as MessageAnnotation[] | undefined; 22 | const toolData = annotations 23 | ? (getAnnotationData(annotations, "tools") as unknown as ToolData[]) 24 | : null; 25 | return toolData?.[0] ? ( 26 | 27 | ) : null; 28 | } 29 | 30 | // TODO: Used to render outputs of tools. If needed, add more renderers here. 31 | function ChatTools({ 32 | data, 33 | artifactVersion, 34 | }: { 35 | data: ToolData; 36 | artifactVersion: number | undefined; 37 | }) { 38 | if (!data) return null; 39 | const { toolCall, toolOutput } = data; 40 | 41 | if (toolOutput.isError) { 42 | return ( 43 |
44 | There was an error when calling the tool {toolCall.name} with input:{" "} 45 |
46 | {JSON.stringify(toolCall.input)} 47 |
48 | ); 49 | } 50 | 51 | switch (toolCall.name) { 52 | case "get_weather_information": 53 | const weatherData = toolOutput.output as unknown as WeatherData; 54 | return ; 55 | case "artifact": 56 | return ( 57 | 61 | ); 62 | default: 63 | return null; 64 | } 65 | } 66 | 67 | type ToolData = { 68 | toolCall: { 69 | id: string; 70 | name: string; 71 | input: { 72 | [key: string]: JSONValue; 73 | }; 74 | }; 75 | toolOutput: { 76 | output: JSONValue; 77 | isError: boolean; 78 | }; 79 | }; 80 | 81 | function getArtifactVersion( 82 | messages: Message[], 83 | message: Message, 84 | ): number | undefined { 85 | const messageId = "id" in message ? message.id : undefined; 86 | if (!messageId) return undefined; 87 | let versionIndex = 1; 88 | for (const m of messages) { 89 | const toolData = m.annotations 90 | ? (getAnnotationData(m.annotations, "tools") as unknown as ToolData[]) 91 | : null; 92 | 93 | if (toolData?.some((t) => t.toolCall.name === "artifact")) { 94 | if ("id" in m && m.id === messageId) { 95 | return versionIndex; 96 | } 97 | versionIndex++; 98 | } 99 | } 100 | return undefined; 101 | } 102 | -------------------------------------------------------------------------------- /app/components/ui/chat/tools/weather-card.tsx: -------------------------------------------------------------------------------- 1 | export interface WeatherData { 2 | latitude: number; 3 | longitude: number; 4 | generationtime_ms: number; 5 | utc_offset_seconds: number; 6 | timezone: string; 7 | timezone_abbreviation: string; 8 | elevation: number; 9 | current_units: { 10 | time: string; 11 | interval: string; 12 | temperature_2m: string; 13 | weather_code: string; 14 | }; 15 | current: { 16 | time: string; 17 | interval: number; 18 | temperature_2m: number; 19 | weather_code: number; 20 | }; 21 | hourly_units: { 22 | time: string; 23 | temperature_2m: string; 24 | weather_code: string; 25 | }; 26 | hourly: { 27 | time: string[]; 28 | temperature_2m: number[]; 29 | weather_code: number[]; 30 | }; 31 | daily_units: { 32 | time: string; 33 | weather_code: string; 34 | }; 35 | daily: { 36 | time: string[]; 37 | weather_code: number[]; 38 | }; 39 | } 40 | 41 | // Follow WMO Weather interpretation codes (WW) 42 | const weatherCodeDisplayMap: Record< 43 | string, 44 | { 45 | icon: React.ReactNode; 46 | status: string; 47 | } 48 | > = { 49 | "0": { 50 | icon: ☀️, 51 | status: "Clear sky", 52 | }, 53 | "1": { 54 | icon: 🌤️, 55 | status: "Mainly clear", 56 | }, 57 | "2": { 58 | icon: ☁️, 59 | status: "Partly cloudy", 60 | }, 61 | "3": { 62 | icon: ☁️, 63 | status: "Overcast", 64 | }, 65 | "45": { 66 | icon: 🌫️, 67 | status: "Fog", 68 | }, 69 | "48": { 70 | icon: 🌫️, 71 | status: "Depositing rime fog", 72 | }, 73 | "51": { 74 | icon: 🌧️, 75 | status: "Drizzle", 76 | }, 77 | "53": { 78 | icon: 🌧️, 79 | status: "Drizzle", 80 | }, 81 | "55": { 82 | icon: 🌧️, 83 | status: "Drizzle", 84 | }, 85 | "56": { 86 | icon: 🌧️, 87 | status: "Freezing Drizzle", 88 | }, 89 | "57": { 90 | icon: 🌧️, 91 | status: "Freezing Drizzle", 92 | }, 93 | "61": { 94 | icon: 🌧️, 95 | status: "Rain", 96 | }, 97 | "63": { 98 | icon: 🌧️, 99 | status: "Rain", 100 | }, 101 | "65": { 102 | icon: 🌧️, 103 | status: "Rain", 104 | }, 105 | "66": { 106 | icon: 🌧️, 107 | status: "Freezing Rain", 108 | }, 109 | "67": { 110 | icon: 🌧️, 111 | status: "Freezing Rain", 112 | }, 113 | "71": { 114 | icon: ❄️, 115 | status: "Snow fall", 116 | }, 117 | "73": { 118 | icon: ❄️, 119 | status: "Snow fall", 120 | }, 121 | "75": { 122 | icon: ❄️, 123 | status: "Snow fall", 124 | }, 125 | "77": { 126 | icon: ❄️, 127 | status: "Snow grains", 128 | }, 129 | "80": { 130 | icon: 🌧️, 131 | status: "Rain showers", 132 | }, 133 | "81": { 134 | icon: 🌧️, 135 | status: "Rain showers", 136 | }, 137 | "82": { 138 | icon: 🌧️, 139 | status: "Rain showers", 140 | }, 141 | "85": { 142 | icon: ❄️, 143 | status: "Snow showers", 144 | }, 145 | "86": { 146 | icon: ❄️, 147 | status: "Snow showers", 148 | }, 149 | "95": { 150 | icon: ⛈️, 151 | status: "Thunderstorm", 152 | }, 153 | "96": { 154 | icon: ⛈️, 155 | status: "Thunderstorm", 156 | }, 157 | "99": { 158 | icon: ⛈️, 159 | status: "Thunderstorm", 160 | }, 161 | }; 162 | 163 | const displayDay = (time: string) => { 164 | return new Date(time).toLocaleDateString("en-US", { 165 | weekday: "long", 166 | }); 167 | }; 168 | 169 | export function WeatherCard({ data }: { data: WeatherData }) { 170 | const currentDayString = new Date(data.current.time).toLocaleDateString( 171 | "en-US", 172 | { 173 | weekday: "long", 174 | month: "long", 175 | day: "numeric", 176 | }, 177 | ); 178 | 179 | return ( 180 |
181 |
182 |
183 |
{currentDayString}
184 |
185 | 186 | {data.current.temperature_2m} {data.current_units.temperature_2m} 187 | 188 | {weatherCodeDisplayMap[data.current.weather_code].icon} 189 |
190 |
191 | 192 | {weatherCodeDisplayMap[data.current.weather_code].status} 193 | 194 |
195 |
196 | {data.daily.time.map((time, index) => { 197 | if (index === 0) return null; // skip the current day 198 | return ( 199 |
200 | {displayDay(time)} 201 |
202 | {weatherCodeDisplayMap[data.daily.weather_code[index]].icon} 203 |
204 | 205 | {weatherCodeDisplayMap[data.daily.weather_code[index]].status} 206 | 207 |
208 | ); 209 | })} 210 |
211 |
212 | ); 213 | } 214 | -------------------------------------------------------------------------------- /app/components/ui/collapsible.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import * as CollapsiblePrimitive from "@radix-ui/react-collapsible"; 4 | 5 | const Collapsible = CollapsiblePrimitive.Root; 6 | 7 | const CollapsibleTrigger = CollapsiblePrimitive.CollapsibleTrigger; 8 | 9 | const CollapsibleContent = CollapsiblePrimitive.CollapsibleContent; 10 | 11 | export { Collapsible, CollapsibleContent, CollapsibleTrigger }; 12 | -------------------------------------------------------------------------------- /app/components/ui/icons/docx.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /app/components/ui/icons/pdf.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 8 | 9 | 18 | 19 | -------------------------------------------------------------------------------- /app/components/ui/icons/sheet.svg: -------------------------------------------------------------------------------- 1 | 2 | 5 | Sheets-icon 6 | Created with Sketch. 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /app/components/ui/icons/txt.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 6 | 7 | 8 | 10 | 11 | 13 | 17 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /app/components/ui/input.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | 3 | import { cn } from "./lib/utils"; 4 | 5 | export interface InputProps 6 | extends React.InputHTMLAttributes {} 7 | 8 | const Input = React.forwardRef( 9 | ({ className, type, ...props }, ref) => { 10 | return ( 11 | 20 | ); 21 | }, 22 | ); 23 | Input.displayName = "Input"; 24 | 25 | export { Input }; 26 | -------------------------------------------------------------------------------- /app/components/ui/lib/utils.ts: -------------------------------------------------------------------------------- 1 | import { clsx, type ClassValue } from "clsx"; 2 | import { twMerge } from "tailwind-merge"; 3 | 4 | export function cn(...inputs: ClassValue[]) { 5 | return twMerge(clsx(inputs)); 6 | } 7 | -------------------------------------------------------------------------------- /app/components/ui/select.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import * as SelectPrimitive from "@radix-ui/react-select"; 4 | import { Check, ChevronDown, ChevronUp } from "lucide-react"; 5 | import * as React from "react"; 6 | import { cn } from "./lib/utils"; 7 | 8 | const Select = SelectPrimitive.Root; 9 | 10 | const SelectGroup = SelectPrimitive.Group; 11 | 12 | const SelectValue = SelectPrimitive.Value; 13 | 14 | const SelectTrigger = React.forwardRef< 15 | React.ElementRef, 16 | React.ComponentPropsWithoutRef 17 | >(({ className, children, ...props }, ref) => ( 18 | span]:line-clamp-1", 22 | className, 23 | )} 24 | {...props} 25 | > 26 | {children} 27 | 28 | 29 | 30 | 31 | )); 32 | SelectTrigger.displayName = SelectPrimitive.Trigger.displayName; 33 | 34 | const SelectScrollUpButton = React.forwardRef< 35 | React.ElementRef, 36 | React.ComponentPropsWithoutRef 37 | >(({ className, ...props }, ref) => ( 38 | 46 | 47 | 48 | )); 49 | SelectScrollUpButton.displayName = SelectPrimitive.ScrollUpButton.displayName; 50 | 51 | const SelectScrollDownButton = React.forwardRef< 52 | React.ElementRef, 53 | React.ComponentPropsWithoutRef 54 | >(({ className, ...props }, ref) => ( 55 | 63 | 64 | 65 | )); 66 | SelectScrollDownButton.displayName = 67 | SelectPrimitive.ScrollDownButton.displayName; 68 | 69 | const SelectContent = React.forwardRef< 70 | React.ElementRef, 71 | React.ComponentPropsWithoutRef 72 | >(({ className, children, position = "popper", ...props }, ref) => ( 73 | 74 | 85 | 86 | 93 | {children} 94 | 95 | 96 | 97 | 98 | )); 99 | SelectContent.displayName = SelectPrimitive.Content.displayName; 100 | 101 | const SelectLabel = React.forwardRef< 102 | React.ElementRef, 103 | React.ComponentPropsWithoutRef 104 | >(({ className, ...props }, ref) => ( 105 | 110 | )); 111 | SelectLabel.displayName = SelectPrimitive.Label.displayName; 112 | 113 | const SelectItem = React.forwardRef< 114 | React.ElementRef, 115 | React.ComponentPropsWithoutRef 116 | >(({ className, children, ...props }, ref) => ( 117 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | {children} 132 | 133 | )); 134 | SelectItem.displayName = SelectPrimitive.Item.displayName; 135 | 136 | const SelectSeparator = React.forwardRef< 137 | React.ElementRef, 138 | React.ComponentPropsWithoutRef 139 | >(({ className, ...props }, ref) => ( 140 | 145 | )); 146 | SelectSeparator.displayName = SelectPrimitive.Separator.displayName; 147 | 148 | export { 149 | Select, 150 | SelectContent, 151 | SelectGroup, 152 | SelectItem, 153 | SelectLabel, 154 | SelectScrollDownButton, 155 | SelectScrollUpButton, 156 | SelectSeparator, 157 | SelectTrigger, 158 | SelectValue, 159 | }; 160 | -------------------------------------------------------------------------------- /app/components/ui/tabs.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import * as TabsPrimitive from "@radix-ui/react-tabs"; 4 | import * as React from "react"; 5 | import { cn } from "./lib/utils"; 6 | 7 | const Tabs = TabsPrimitive.Root; 8 | 9 | const TabsList = React.forwardRef< 10 | React.ElementRef, 11 | React.ComponentPropsWithoutRef 12 | >(({ className, ...props }, ref) => ( 13 | 21 | )); 22 | TabsList.displayName = TabsPrimitive.List.displayName; 23 | 24 | const TabsTrigger = React.forwardRef< 25 | React.ElementRef, 26 | React.ComponentPropsWithoutRef 27 | >(({ className, ...props }, ref) => ( 28 | 36 | )); 37 | TabsTrigger.displayName = TabsPrimitive.Trigger.displayName; 38 | 39 | const TabsContent = React.forwardRef< 40 | React.ElementRef, 41 | React.ComponentPropsWithoutRef 42 | >(({ className, ...props }, ref) => ( 43 | 51 | )); 52 | TabsContent.displayName = TabsPrimitive.Content.displayName; 53 | 54 | export { Tabs, TabsContent, TabsList, TabsTrigger }; 55 | -------------------------------------------------------------------------------- /app/components/ui/textarea.tsx: -------------------------------------------------------------------------------- 1 | import * as React from "react"; 2 | import { cn } from "./lib/utils"; 3 | 4 | export interface TextareaProps 5 | extends React.TextareaHTMLAttributes {} 6 | 7 | const Textarea = React.forwardRef( 8 | ({ className, ...props }, ref) => { 9 | return ( 10 |