├── .devcontainer └── devcontainer.json ├── .env.sample ├── .github ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yaml └── workflows │ ├── azure-bicep-validate.yaml │ ├── azure-dev.yaml │ └── python-check.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── .vscode ├── launch.json └── settings.json ├── LICENSE.md ├── README.md ├── SECURITY.md ├── azure.yaml ├── docker-compose.yaml ├── docs ├── deploy_existing.md ├── local_ollama.md ├── screenshot_chatimage.png └── screenshot_chatspeech.png ├── infra ├── aca.bicep ├── core │ ├── ai │ │ └── cognitiveservices.bicep │ ├── host │ │ ├── container-app-upsert.bicep │ │ ├── container-app.bicep │ │ ├── container-apps-environment.bicep │ │ ├── container-apps.bicep │ │ └── container-registry.bicep │ ├── monitor │ │ └── loganalytics.bicep │ └── security │ │ ├── registry-access.bicep │ │ └── role.bicep ├── getkey.sh ├── main.bicep └── main.parameters.json ├── notebooks ├── azure_arch.png ├── chat_pdf_images.ipynb ├── chat_vision.ipynb ├── dented_car.jpg ├── dishwasher.png ├── menu.png ├── mystery_reptile.png ├── page_0.png ├── page_1.png ├── page_10.png ├── page_11.png ├── page_12.png ├── page_13.png ├── page_14.png ├── page_2.png ├── page_3.png ├── page_4.png ├── page_5.png ├── page_6.png ├── page_7.png ├── page_8.png ├── page_9.png ├── plants.pdf └── ur.jpg ├── pyproject.toml ├── readme_diagram.png ├── requirements-dev.txt ├── scripts ├── write_env.ps1 └── write_env.sh ├── src ├── .dockerignore ├── Dockerfile ├── __init__.py ├── gunicorn.conf.py ├── pyproject.toml ├── quartapp │ ├── __init__.py │ ├── chat.py │ ├── static │ │ ├── speech-input.js │ │ ├── speech-output.js │ │ └── styles.css │ └── templates │ │ └── index.html └── requirements.txt └── tests ├── __init__.py ├── conftest.py ├── mock_cred.py ├── snapshots └── test_app │ ├── test_chat_stream_text │ └── result.json │ └── test_chat_stream_text_history │ └── result.json └── test_app.py /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "openai-chat-app-quickstart", 3 | "image": "mcr.microsoft.com/devcontainers/python:3.11-bullseye", 4 | "forwardPorts": [50505], 5 | "features": { 6 | "ghcr.io/azure/azure-dev/azd:latest": {} 7 | }, 8 | "customizations": { 9 | "vscode": { 10 | "extensions": [ 11 | "ms-azuretools.azure-dev", 12 | "ms-azuretools.vscode-bicep", 13 | "ms-python.python", 14 | "ms-toolsai.jupyter", 15 | "GitHub.vscode-github-actions" 16 | ] 17 | } 18 | }, 19 | "postCreateCommand": "python3 -m pip install -r requirements-dev.txt && python3 -m pip install -e src", 20 | "remoteUser": "vscode", 21 | "hostRequirements": { 22 | "memory": "8gb" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /.env.sample: -------------------------------------------------------------------------------- 1 | # Can be "azure", "github", or "local" 2 | OPENAI_HOST="azure" 3 | # For Azure, the model name should actually be the deployment name 4 | OPENAI_MODEL="gpt-4o" 5 | 6 | # For Azure host: 7 | AZURE_OPENAI_API_VERSION="" 8 | AZURE_OPENAI_ENDPOINT="https://YOUR-ENDPOINT-HERE.openai.azure.com/" 9 | 10 | # For local models, like Ollama/llamafile: 11 | LOCAL_OPENAI_ENDPOINT="http://localhost:8080/v1" 12 | -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to [project-title] 2 | 3 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 4 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 5 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 6 | 7 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 8 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 9 | provided by the bot. You will only need to do this once across all repos using our CLA. 10 | 11 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 12 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 13 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 14 | 15 | - [Code of Conduct](#coc) 16 | - [Issues and Bugs](#issue) 17 | - [Feature Requests](#feature) 18 | - [Submission Guidelines](#submit) 19 | 20 | ## Code of Conduct 21 | Help us keep this project open and inclusive. Please read and follow our [Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 22 | 23 | ## Found an Issue? 24 | If you find a bug in the source code or a mistake in the documentation, you can help us by 25 | [submitting an issue](#submit-issue) to the GitHub Repository. Even better, you can 26 | [submit a Pull Request](#submit-pr) with a fix. 27 | 28 | ## Want a Feature? 29 | You can *request* a new feature by [submitting an issue](#submit-issue) to the GitHub 30 | Repository. If you would like to *implement* a new feature, please submit an issue with 31 | a proposal for your work first, to be sure that we can use it. 32 | 33 | * **Small Features** can be crafted and directly [submitted as a Pull Request](#submit-pr). 34 | 35 | ## Submission Guidelines 36 | 37 | ### Submitting an Issue 38 | Before you submit an issue, search the archive, maybe your question was already answered. 39 | 40 | If your issue appears to be a bug, and hasn't been reported, open a new issue. 41 | Help us to maximize the effort we can spend fixing issues and adding new 42 | features, by not reporting duplicate issues. Providing the following information will increase the 43 | chances of your issue being dealt with quickly: 44 | 45 | * **Overview of the Issue** - if an error is being thrown a non-minified stack trace helps 46 | * **Version** - what version is affected (e.g. 0.1.2) 47 | * **Motivation for or Use Case** - explain what are you trying to do and why the current behavior is a bug for you 48 | * **Browsers and Operating System** - is this a problem with all browsers? 49 | * **Reproduce the Error** - provide a live example or a unambiguous set of steps 50 | * **Related Issues** - has a similar issue been reported before? 51 | * **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be 52 | causing the problem (line of code or commit) 53 | 54 | You can file new issues by providing the above information at the corresponding repository's issues link: https://github.com/[organization-name]/[repository-name]/issues/new]. 55 | 56 | ### Submitting a Pull Request (PR) 57 | Before you submit your Pull Request (PR) consider the following guidelines: 58 | 59 | * Search the repository (https://github.com/[organization-name]/[repository-name]/pulls) for an open or closed PR 60 | that relates to your submission. You don't want to duplicate effort. 61 | 62 | * Make your changes in a new git fork: 63 | 64 | * Commit your changes using a descriptive commit message 65 | * Push your fork to GitHub: 66 | * In GitHub, create a pull request 67 | * If we suggest changes then: 68 | * Make the required updates. 69 | * Rebase your fork and force push to your GitHub repository (this will update your Pull Request): 70 | 71 | ```shell 72 | git rebase master -i 73 | git push -f 74 | ``` 75 | 76 | That's it! Thank you for your contribution! 77 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 4 | > Please provide us with the following information: 5 | > --------------------------------------------------------------- 6 | 7 | ### This issue is for a: (mark with an `x`) 8 | ``` 9 | - [ ] bug report -> please search issues before submitting 10 | - [ ] feature request 11 | - [ ] documentation issue or request 12 | - [ ] regression (a behavior that used to work and stopped in a new release) 13 | ``` 14 | 15 | ### Minimal steps to reproduce 16 | > 17 | 18 | ### Any log messages given by the failure 19 | > 20 | 21 | ### Expected/desired behavior 22 | > 23 | 24 | ### OS and Version? 25 | > Windows 7, 8 or 10. Linux (which distribution). macOS (Yosemite? El Capitan? Sierra?) 26 | 27 | ### Versions 28 | > 29 | 30 | ### Mention any other details that might be useful 31 | 32 | > --------------------------------------------------------------- 33 | > Thanks! We'll be in touch soon. 34 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Purpose 2 | 3 | * ... 4 | 5 | ## Does this introduce a breaking change? 6 | 7 | ``` 8 | [ ] Yes 9 | [ ] No 10 | ``` 11 | 12 | ## Pull Request Type 13 | What kind of change does this Pull Request introduce? 14 | 15 | 16 | ``` 17 | [ ] Bugfix 18 | [ ] Feature 19 | [ ] Code style update (formatting, local variables) 20 | [ ] Refactoring (no functional changes, no api changes) 21 | [ ] Documentation content changes 22 | [ ] Other... Please describe: 23 | ``` 24 | 25 | ## How to Test 26 | * Get the code 27 | 28 | ``` 29 | git clone [repo-address] 30 | cd [repo-name] 31 | git checkout [branch-name] 32 | npm install 33 | ``` 34 | 35 | * Test the code 36 | 37 | ``` 38 | ``` 39 | 40 | ## What to Check 41 | Verify that the following are valid 42 | * ... 43 | 44 | ## Other Information 45 | 46 | -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: github-actions 9 | directory: / 10 | schedule: 11 | interval: monthly 12 | groups: 13 | github-actions: 14 | patterns: 15 | - "*" 16 | - package-ecosystem: "pip-compile" # See documentation for possible values 17 | directory: "/" # Location of package manifests 18 | schedule: 19 | interval: "weekly" 20 | -------------------------------------------------------------------------------- /.github/workflows/azure-bicep-validate.yaml: -------------------------------------------------------------------------------- 1 | name: Validate bicep scripts 2 | on: 3 | workflow_dispatch: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - "**/*.bicep" 9 | pull_request: 10 | branches: 11 | - main 12 | paths: 13 | - "**/*.bicep" 14 | 15 | jobs: 16 | build: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v4 21 | 22 | - name: Build Bicep for linting 23 | uses: azure/CLI@v2 24 | with: 25 | inlineScript: | 26 | export DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 27 | az config set bicep.use_binary_from_path=false && az bicep build -f infra/main.bicep --stdout 28 | 29 | - name: Run Microsoft Security DevOps Analysis 30 | uses: microsoft/security-devops-action@preview 31 | id: msdo 32 | continue-on-error: true 33 | with: 34 | tools: templateanalyzer 35 | 36 | - name: Upload alerts to Security tab 37 | uses: github/codeql-action/upload-sarif@v3 38 | if: github.repository_owner == 'Azure-Samples' 39 | with: 40 | sarif_file: ${{ steps.msdo.outputs.sarifFile }} 41 | -------------------------------------------------------------------------------- /.github/workflows/azure-dev.yaml: -------------------------------------------------------------------------------- 1 | name: Deploy to Azure 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | # Run when commits are pushed to mainline branch (main or master) 7 | # Set this to the mainline branch you are using 8 | branches: 9 | - main 10 | 11 | # GitHub Actions workflow to deploy to Azure using azd 12 | # To configure required secrets for connecting to Azure, simply run `azd pipeline config` 13 | 14 | # Set up permissions for deploying with secretless Azure federated credentials 15 | # https://learn.microsoft.com/en-us/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication 16 | permissions: 17 | id-token: write 18 | contents: read 19 | 20 | jobs: 21 | build: 22 | runs-on: ubuntu-latest 23 | env: 24 | AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} 25 | AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} 26 | AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} 27 | AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} 28 | # Project-specific variables 29 | AZURE_OPENAI_LOCATION: ${{ vars.AZURE_OPENAI_LOCATION }} 30 | AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_MODEL }} 31 | AZURE_OPENAI_MODEL_VERSION: ${{ vars.AZURE_OPENAI_MODEL_VERSION }} 32 | AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_OPENAI_DEPLOYMENT }} 33 | AZURE_OPENAI_DEPLOYMENT_CAPACITY: ${{ vars.AZURE_OPENAI_DEPLOYMENT_CAPACITY }} 34 | AZURE_OPENAI_DEPLOYMENT_SKU_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_SKU_NAME }} 35 | AZURE_OPENAI_RESOURCE: ${{ vars.AZURE_OPENAI_RESOURCE }} 36 | AZURE_OPENAI_RESOURCE_GROUP: ${{ vars.AZURE_OPENAI_RESOURCE_GROUP }} 37 | AZURE_OPENAI_RESOURCE_GROUP_LOCATION: ${{ vars.AZURE_OPENAI_RESOURCE_GROUP_LOCATION }} 38 | AZURE_OPENAI_SKU_NAME: ${{ vars.AZURE_OPENAI_SKU_NAME }} 39 | AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }} 40 | CREATE_AZURE_OPENAI: ${{ vars.CREATE_AZURE_OPENAI }} 41 | AZURE_OPENAI_KEY_FOR_CHATVISION: ${{ vars.AZURE_OPENAI_KEY_FOR_CHATVISION }} 42 | AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} 43 | CREATE_ROLE_FOR_USER: ${{ vars.CREATE_ROLE_FOR_USER }} 44 | SERVICE_ACA_RESOURCE_EXISTS: ${{ vars.SERVICE_ACA_RESOURCE_EXISTS }} 45 | DISABLE_KEY_BASED_AUTH: ${{ vars.DISABLE_KEY_BASED_AUTH }} 46 | steps: 47 | - name: Checkout 48 | uses: actions/checkout@v4 49 | 50 | - name: Install azd 51 | uses: Azure/setup-azd@v2.0.0 52 | 53 | - name: Log in with Azure (Federated Credentials) 54 | if: ${{ env.AZURE_CLIENT_ID != '' }} 55 | run: | 56 | azd auth login ` 57 | --client-id "$Env:AZURE_CLIENT_ID" ` 58 | --federated-credential-provider "github" ` 59 | --tenant-id "$Env:AZURE_TENANT_ID" 60 | shell: pwsh 61 | 62 | - name: Log in with Azure (Client Credentials) 63 | if: ${{ env.AZURE_CREDENTIALS != '' }} 64 | run: | 65 | $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; 66 | Write-Host "::add-mask::$($info.clientSecret)" 67 | 68 | azd auth login ` 69 | --client-id "$($info.clientId)" ` 70 | --client-secret "$($info.clientSecret)" ` 71 | --tenant-id "$($info.tenantId)" 72 | shell: pwsh 73 | env: 74 | AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} 75 | 76 | - name: Provision Infrastructure 77 | run: | 78 | azd env set CREATE_ROLE_FOR_USER false --no-prompt 79 | azd provision --no-prompt 80 | env: 81 | AZURE_ENV_NAME: ${{ vars.AZURE_ENV_NAME }} 82 | AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} 83 | AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} 84 | 85 | - name: Deploy Application 86 | run: azd deploy --no-prompt 87 | env: 88 | AZURE_ENV_NAME: ${{ vars.AZURE_ENV_NAME }} 89 | AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} 90 | AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }} 91 | -------------------------------------------------------------------------------- /.github/workflows/python-check.yaml: -------------------------------------------------------------------------------- 1 | name: Python check 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | test_package: 11 | name: Test ${{ matrix.os }} Python ${{ matrix.python_version }} 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | os: ["ubuntu-20.04"] 17 | python_version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 18 | steps: 19 | - uses: actions/checkout@v4 20 | - name: Setup python 21 | uses: actions/setup-python@v5 22 | with: 23 | python-version: ${{ matrix.python_version }} 24 | architecture: x64 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | pip install -r requirements-dev.txt 29 | - name: Lint with ruff 30 | run: python3 -m ruff check . 31 | - name: Check formatting with ruff 32 | run: python3 -m ruff format --check . 33 | - name: Run tests with pytest 34 | run: python3 -m pytest 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .azure 3 | .pyc 4 | __pycache__/ 5 | .coverage 6 | .idea/ 7 | .venv/ 8 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.5.0 4 | hooks: 5 | - id: check-yaml 6 | - id: end-of-file-fixer 7 | - id: trailing-whitespace 8 | - repo: https://github.com/astral-sh/ruff-pre-commit 9 | rev: v0.1.13 10 | hooks: 11 | - id: ruff 12 | - id: ruff-format 13 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Python App Backend", 9 | "type": "debugpy", 10 | "request": "launch", 11 | "module": "quart", 12 | "env": { 13 | "QUART_APP": "src.quartapp", 14 | "QUART_ENV": "development", 15 | "QUART_DEBUG": "0" 16 | }, 17 | "args": [ 18 | "run", 19 | "--no-reload", 20 | "-p 50505" 21 | ], 22 | "console": "integratedTerminal", 23 | "justMyCode": false 24 | } 25 | ] 26 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.testing.pytestArgs": [ 3 | "tests" 4 | ], 5 | "python.testing.unittestEnabled": false, 6 | "python.testing.pytestEnabled": true, 7 | "notebook.output.wordWrap": true 8 | } 9 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 18 | # Chat + Vision using Azure OpenAI (Python) 19 | 20 | [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/Azure-Samples/openai-chat-vision-quickstart) 21 | [![Open in Dev Containers](https://img.shields.io/static/v1?style=for-the-badge&label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/azure-samples/openai-chat-vision-quickstart) 22 | 23 | This repository includes a Python app that uses Azure OpenAI to generate responses to user messages and uploaded images. 24 | 25 | The project includes all the infrastructure and configuration needed to provision Azure OpenAI resources and deploy the app to [Azure Container Apps](https://learn.microsoft.com/azure/container-apps/overview) using the [Azure Developer CLI](https://learn.microsoft.com/azure/developer/azure-developer-cli/overview). By default, the app will use managed identity to authenticate with Azure OpenAI, and it will deploy a GPT-4o model with the GlobalStandard SKU. 26 | 27 | We recommend first going through the [deploying steps](#deploying) before running this app locally, 28 | since the local app needs credentials for Azure OpenAI to work properly. 29 | 30 | * [Features](#features) 31 | * [Architecture diagram](#architecture-diagram) 32 | * [Getting started](#getting-started) 33 | * [GitHub Codespaces](#github-codespaces) 34 | * [VS Code Dev Containers](#vs-code-dev-containers) 35 | * [Local environment](#local-environment) 36 | * [Deploying](#deploying) 37 | * [Development server](#development-server) 38 | * [Costs](#costs) 39 | * [Security guidelines](#security-guidelines) 40 | * [Resources](#resources) 41 | 42 | ## Features 43 | 44 | * A Python [Quart](https://quart.palletsprojects.com/en/latest/) that uses the [openai](https://pypi.org/project/openai/) package to generate responses to user messages with uploaded image files. 45 | * A basic HTML/JS frontend that streams responses from the backend using [JSON Lines](http://jsonlines.org/) over a [ReadableStream](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream). 46 | * Speech input and output buttons that use the free built-in browser APIs. 47 | * [Bicep files](https://docs.microsoft.com/azure/azure-resource-manager/bicep/) for provisioning Azure resources, including Azure OpenAI, Azure Container Apps, Azure Container Registry, Azure Log Analytics, and RBAC roles. 48 | * Support for using [GitHub models](https://github.com/marketplace/models) during development. 49 | 50 | ![Screenshot of the chat app](docs/screenshot_chatspeech.png) 51 | 52 | ## Architecture diagram 53 | 54 | ![Architecture diagram: Azure Container Apps inside Container Apps Environment, connected to Container Registry with Container, connected to Managed Identity for Azure OpenAI](readme_diagram.png) 55 | 56 | ## Getting started 57 | 58 | You have a few options for getting started with this template. 59 | The quickest way to get started is GitHub Codespaces, since it will setup all the tools for you, but you can also [set it up locally](#local-environment). 60 | 61 | ### GitHub Codespaces 62 | 63 | You can run this template virtually by using GitHub Codespaces. The button will open a web-based VS Code instance in your browser: 64 | 65 | 1. Open the template (this may take several minutes): 66 | 67 | [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/Azure-Samples/openai-chat-vision-quickstart) 68 | 69 | 2. Open a terminal window 70 | 3. Continue with the [deploying steps](#deploying) 71 | 72 | ### VS Code Dev Containers 73 | 74 | A related option is VS Code Dev Containers, which will open the project in your local VS Code using the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers): 75 | 76 | 1. Start Docker Desktop (install it if not already installed) 77 | 2. Open the project: 78 | 79 | [![Open in Dev Containers](https://img.shields.io/static/v1?style=for-the-badge&label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/azure-samples/openai-chat-vision-quickstart) 80 | 81 | 3. In the VS Code window that opens, once the project files show up (this may take several minutes), open a terminal window. 82 | 4. Continue with the [deploying steps](#deploying) 83 | 84 | ### Local environment 85 | 86 | If you're not using one of the above options for opening the project, then you'll need to: 87 | 88 | 1. Make sure the following tools are installed: 89 | 90 | * [Azure Developer CLI (azd)](https://aka.ms/install-azd) 91 | * [Python 3.10+](https://www.python.org/downloads/) 92 | * [Docker Desktop](https://www.docker.com/products/docker-desktop/) 93 | * [Git](https://git-scm.com/downloads) 94 | 95 | 2. Download the project code: 96 | 97 | ```shell 98 | azd init -t openai-chat-vision-quickstart 99 | ``` 100 | 101 | 3. Open the project folder 102 | 4. Create a [Python virtual environment](https://docs.python.org/3/tutorial/venv.html#creating-virtual-environments) and activate it. 103 | 5. Install required Python packages: 104 | 105 | ```shell 106 | pip install -r requirements-dev.txt 107 | ``` 108 | 109 | 6. Install the app as an editable package: 110 | 111 | ```shell 112 | python -m pip install -e src 113 | ``` 114 | 115 | 7. Continue with the [deploying steps](#deploying). 116 | 117 | ## Deploying 118 | 119 | Once you've opened the project in [Codespaces](#github-codespaces), in [Dev Containers](#vs-code-dev-containers), or [locally](#local-environment), you can deploy it to Azure. 120 | 121 | ### Azure account setup 122 | 123 | 1. Sign up for a [free Azure account](https://azure.microsoft.com/free/) and create an Azure Subscription. 124 | 2. Check that you have the necessary permissions: 125 | * Your Azure account must have `Microsoft.Authorization/roleAssignments/write` permissions, such as [Role Based Access Control Administrator](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#role-based-access-control-administrator-preview), [User Access Administrator](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#user-access-administrator), or [Owner](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#owner). If you don't have subscription-level permissions, you must be granted [RBAC](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#role-based-access-control-administrator-preview) for an existing resource group and [deploy to that existing group](/docs/deploy_existing.md#resource-group). 126 | * Your Azure account also needs `Microsoft.Resources/deployments/write` permissions on the subscription level. 127 | 128 | ### Deploying with azd 129 | 130 | 1. Login to Azure: 131 | 132 | ```shell 133 | azd auth login 134 | ``` 135 | 136 | 2. Provision and deploy all the resources: 137 | 138 | ```shell 139 | azd up 140 | ``` 141 | 142 | It will prompt you to provide an `azd` environment name (like "chat-app"), select a subscription from your Azure account, and select a [location where OpenAI is available](https://azure.microsoft.com/explore/global-infrastructure/products-by-region/?products=cognitive-services®ions=all) (like "francecentral"). Then it will provision the resources in your account and deploy the latest code. If you get an error or timeout with deployment, changing the location can help, as there may be availability constraints for the OpenAI resource. 143 | 144 | 3. When `azd` has finished deploying, you'll see an endpoint URI in the command output. Visit that URI, and you should see the chat app! 🎉 145 | 4. When you've made any changes to the app code, you can just run: 146 | 147 | ```shell 148 | azd deploy 149 | ``` 150 | 151 | ### Continuous deployment with GitHub Actions 152 | 153 | This project includes a Github workflow for deploying the resources to Azure 154 | on every push to main. That workflow requires several Azure-related authentication secrets 155 | to be stored as Github action secrets. To set that up, run: 156 | 157 | ```shell 158 | azd pipeline config 159 | ``` 160 | 161 | ## Development server 162 | 163 | In order to run this app, you need to either have an Azure OpenAI account deployed (from the [deploying steps](#deploying)) or use a model from [GitHub models](https://github.com/marketplace/models). 164 | 165 | 1. If you already deployed the app using `azd up`, then a `.env` file was created with the necessary environment variables, and you can skip to step 3. 166 | 167 | 2. To use the app with GitHub models, either copy `.env.sample` into a `.env` file or start from the created `.env` file. 168 | Change `OPENAI_HOST` to "github" in the `.env` file. 169 | 170 | You'll need a `GITHUB_TOKEN` environment variable that stores a GitHub personal access token. 171 | If you're running this inside a GitHub Codespace, the token will be automatically available. 172 | If not, generate a new [personal access token](https://github.com/settings/tokens) and run this command to set the `GITHUB_TOKEN` environment variable: 173 | 174 | ```shell 175 | export GITHUB_TOKEN="" 176 | ``` 177 | 178 | 3. Start the development server: 179 | 180 | ```shell 181 | python -m quart --app src.quartapp run --port 50505 --reload 182 | ``` 183 | 184 | This will start the app on port 50505, and you can access it at `http://localhost:50505`. 185 | 186 | ## Guidance 187 | 188 | ### Costs 189 | 190 | Pricing varies per region and usage, so it isn't possible to predict exact costs for your usage. 191 | The majority of the Azure resources used in this infrastructure are on usage-based pricing tiers. 192 | However, Azure Container Registry has a fixed cost per registry per day. 193 | 194 | You can try the [Azure pricing calculator](https://azure.com/e/3987c81282c84410b491d28094030c9a) for the resources: 195 | 196 | * Azure OpenAI Service: S0 tier, GPT-4o model. Pricing is based on token count. [Pricing](https://azure.microsoft.com/pricing/details/cognitive-services/openai-service/) 197 | * Azure Container App: Consumption tier with 0.5 CPU, 1GiB memory/storage. Pricing is based on resource allocation, and each month allows for a certain amount of free usage. [Pricing](https://azure.microsoft.com/pricing/details/container-apps/) 198 | * Azure Container Registry: Basic tier. [Pricing](https://azure.microsoft.com/pricing/details/container-registry/) 199 | * Log analytics: Pay-as-you-go tier. Costs based on data ingested. [Pricing](https://azure.microsoft.com/pricing/details/monitor/) 200 | 201 | ⚠️ To avoid unnecessary costs, remember to take down your app if it's no longer in use, 202 | either by deleting the resource group in the Portal or running `azd down`. 203 | 204 | ### Security guidelines 205 | 206 | This template uses [Managed Identity](https://learn.microsoft.com/entra/identity/managed-identities-azure-resources/overview) for authenticating to the Azure OpenAI service. 207 | 208 | Additionally, we have added a [GitHub Action](https://github.com/microsoft/security-devops-action) that scans the infrastructure-as-code files and generates a report containing any detected issues. To ensure continued best practices in your own repository, we recommend that anyone creating solutions based on our templates ensure that the [Github secret scanning](https://docs.github.com/code-security/secret-scanning/about-secret-scanning) setting is enabled. 209 | 210 | You may want to consider additional security measures, such as: 211 | 212 | * Protecting the Azure Container Apps instance with a [firewall](https://learn.microsoft.com/azure/container-apps/waf-app-gateway) and/or [Virtual Network](https://learn.microsoft.com/azure/container-apps/networking?tabs=workload-profiles-env%2Cazure-cli). 213 | 214 | ### Resources 215 | 216 | About this app: 217 | * [Get started with multimodal vision chat apps using Azure OpenAI](https://learn.microsoft.com/azure/developer/ai/get-started-app-chat-vision?tabs=github-codespaces): The Microsoft Learn Quickstart article for this sample, walks through both deployment and the relevant code for working with images in chat. 218 | * [Video: Using vision models with Python](https://www.youtube.com/watch?v=toR644E--w8): A live stream recording that steps through the Python notebook and app code. 219 | * [Blog post: Add speech input/output to your app](https://blog.pamelafox.org/2024/12/add-browser-speech-inputoutput-to-your.html): Explains the speech buttons used in this app. 220 | 221 | Related samples and docs: 222 | * [OpenAI Chat Application Quickstart](https://github.com/Azure-Samples/openai-chat-app-quickstart): Similar to this project, but without the vision and image uploads. 223 | * [OpenAI Chat Application with Microsoft Entra Authentication - MSAL SDK](https://github.com/Azure-Samples/openai-chat-app-entra-auth-local): Similar to this project, but adds user authentication with Microsoft Entra using the Microsoft Graph SDK and built-in authentication feature of Azure Container Apps. 224 | * [OpenAI Chat Application with Microsoft Entra Authentication - Built-in Auth](https://github.com/Azure-Samples/openai-chat-app-entra-auth-local): Similar to this project, but adds user authentication with Microsoft Entra using the Microsoft Graph SDK and MSAL SDK. 225 | * [RAG chat with Azure AI Search + Python](https://github.com/Azure-Samples/azure-search-openai-demo/): A more advanced chat app that uses Azure AI Search to ground responses in domain knowledge. Includes user authentication with Microsoft Entra as well as data access controls. 226 | * [Develop Python apps that use Azure AI services](https://learn.microsoft.com/azure/developer/python/azure-ai-for-python-developers) 227 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /azure.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json 2 | 3 | name: openai-chat-vision-quickstart 4 | metadata: 5 | template: openai-chat-vision-quickstart@0.1.0-beta 6 | requiredVersions: 7 | azd: ">= 1.10.0" 8 | services: 9 | aca: 10 | project: ./src 11 | language: py 12 | host: containerapp 13 | docker: 14 | remoteBuild: true 15 | hooks: 16 | postprovision: 17 | windows: 18 | shell: pwsh 19 | run: ./scripts/write_env.ps1 20 | continueOnError: true 21 | posix: 22 | shell: sh 23 | run: ./scripts/write_env.sh 24 | continueOnError: true 25 | pipeline: 26 | variables: 27 | - AZURE_OPENAI_LOCATION 28 | - AZURE_OPENAI_MODEL 29 | - AZURE_OPENAI_MODEL_VERSION 30 | - AZURE_OPENAI_DEPLOYMENT 31 | - AZURE_OPENAI_DEPLOYMENT_CAPACITY 32 | - AZURE_OPENAI_DEPLOYMENT_SKU_NAME 33 | - AZURE_OPENAI_RESOURCE 34 | - AZURE_OPENAI_RESOURCE_GROUP 35 | - AZURE_OPENAI_RESOURCE_GROUP_LOCATION 36 | - AZURE_OPENAI_SKU_NAME 37 | - AZURE_OPENAI_API_VERSION 38 | - CREATE_AZURE_OPENAI 39 | - AZURE_OPENAI_KEY_FOR_CHATVISION 40 | - AZURE_OPENAI_ENDPOINT 41 | - CREATE_ROLE_FOR_USER 42 | - SERVICE_ACA_RESOURCE_EXISTS 43 | - DISABLE_KEY_BASED_AUTH 44 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | app: 3 | build: 4 | context: ./src 5 | env_file: 6 | - .env 7 | ports: 8 | - 50505:50505 9 | volumes: 10 | - ./src:/code 11 | -------------------------------------------------------------------------------- /docs/deploy_existing.md: -------------------------------------------------------------------------------- 1 | 2 | # Deploying with existing Azure resources 3 | 4 | If you already have existing Azure resources, or if you want to specify the exact name of new Azure Resource, you can do so by setting `azd` environment values. 5 | You should set these values before running `azd up`. Once you've set them, return to the [deployment steps](../README.md#deployment). 6 | 7 | * [Resource group](#resource-group) 8 | * [Azure OpenAI resource](#azure-openai-resource) 9 | 10 | ## Resource group 11 | 12 | 1. Run `azd env set AZURE_RESOURCE_GROUP {Name of existing resource group}` 13 | 1. Run `azd env set AZURE_LOCATION {Location of existing resource group}` 14 | 15 | ## Azure OpenAI resource 16 | 17 | If you already have an OpenAI resource and would like to re-use it, run `azd env set` to specify the values for the existing OpenAI resource. 18 | 19 | ```shell 20 | azd env set AZURE_OPENAI_RESOURCE {name of OpenAI resource} 21 | azd env set AZURE_OPENAI_RESOURCE_GROUP {name of resource group that it's inside} 22 | azd env set AZURE_OPENAI_RESOURCE_GROUP_LOCATION {location for that group} 23 | azd env set AZURE_OPENAI_SKU_NAME {name of the SKU, defaults to "S0"} 24 | ``` 25 | 26 | If you don't want to deploy a new Azure OpenAI resource and just want to use an existing one via its endpoint and key, you can set the following values: 27 | 28 | ```shell 29 | azd env set CREATE_AZURE_OPENAI false 30 | azd env set AZURE_OPENAI_DEPLOYMENT gpt-35-turbo 31 | azd env set AZURE_OPENAI_ENDPOINT https://YOUR-ENDPOINT-HERE 32 | azd env set AZURE_OPENAI_KEY_FOR_CHATVISION YOUR-KEY-HERE 33 | ``` 34 | 35 | ⚠️ We don't recommend using key-based access in production, but it may be useful for testing or development purposes. 36 | -------------------------------------------------------------------------------- /docs/local_ollama.md: -------------------------------------------------------------------------------- 1 | # Using a local LLM server 2 | 3 | You may want to save costs by developing against a local LLM server, such as 4 | [llamafile](https://github.com/Mozilla-Ocho/llamafile/). Note that a local LLM 5 | will generally be slower and not as sophisticated. 6 | 7 | Once you've got your local LLM running and serving an OpenAI-compatible endpoint, define `LOCAL_OPENAI_ENDPOINT` in your `.env` file. 8 | 9 | For example, to point at a local llamafile server running on its default port: 10 | 11 | ```shell 12 | LOCAL_OPENAI_ENDPOINT="http://localhost:8080/v1" 13 | ``` 14 | 15 | If you're running inside a dev container, use this local URL instead: 16 | 17 | ```shell 18 | LOCAL_OPENAI_ENDPOINT="http://host.docker.internal:8080/v1" 19 | ``` 20 | -------------------------------------------------------------------------------- /docs/screenshot_chatimage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/docs/screenshot_chatimage.png -------------------------------------------------------------------------------- /docs/screenshot_chatspeech.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/docs/screenshot_chatspeech.png -------------------------------------------------------------------------------- /infra/aca.bicep: -------------------------------------------------------------------------------- 1 | param name string 2 | param location string = resourceGroup().location 3 | param tags object = {} 4 | 5 | param identityName string 6 | param containerAppsEnvironmentName string 7 | param containerRegistryName string 8 | param serviceName string = 'aca' 9 | param exists bool 10 | param openAiDeploymentName string 11 | param openAiEndpoint string 12 | param openAiApiVersion string 13 | @secure() 14 | param openAiKey string = '' 15 | 16 | resource acaIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { 17 | name: identityName 18 | location: location 19 | } 20 | 21 | var env = [ 22 | { 23 | name: 'OPENAI_HOST' 24 | value: 'azure' 25 | } 26 | { 27 | name: 'OPENAI_MODEL' 28 | value: openAiDeploymentName 29 | } 30 | { 31 | name: 'AZURE_OPENAI_ENDPOINT' 32 | value: openAiEndpoint 33 | } 34 | { 35 | name: 'AZURE_OPENAI_API_VERSION' 36 | value: openAiApiVersion 37 | } 38 | { 39 | name: 'RUNNING_IN_PRODUCTION' 40 | value: 'true' 41 | } 42 | { 43 | // ManagedIdentityCredential will be passed this environment variable: 44 | name: 'AZURE_CLIENT_ID' 45 | value: acaIdentity.properties.clientId 46 | } 47 | ] 48 | 49 | var envWithSecret = !empty(openAiKey) ? union(env, [ 50 | { 51 | name: 'AZURE_OPENAI_KEY_FOR_CHATVISION' 52 | secretRef: 'azure-openai-key' 53 | } 54 | ]) : env 55 | 56 | var secrets = !empty(openAiKey) ? { 57 | 'azure-openai-key': openAiKey 58 | } : {} 59 | 60 | module app 'core/host/container-app-upsert.bicep' = { 61 | name: '${serviceName}-container-app-module' 62 | params: { 63 | name: name 64 | location: location 65 | tags: union(tags, { 'azd-service-name': serviceName }) 66 | identityName: acaIdentity.name 67 | exists: exists 68 | containerAppsEnvironmentName: containerAppsEnvironmentName 69 | containerRegistryName: containerRegistryName 70 | env: envWithSecret 71 | secrets: secrets 72 | targetPort: 50505 73 | } 74 | } 75 | 76 | output SERVICE_ACA_IDENTITY_PRINCIPAL_ID string = acaIdentity.properties.principalId 77 | output SERVICE_ACA_NAME string = app.outputs.name 78 | output SERVICE_ACA_URI string = app.outputs.uri 79 | output SERVICE_ACA_IMAGE_NAME string = app.outputs.imageName 80 | -------------------------------------------------------------------------------- /infra/core/ai/cognitiveservices.bicep: -------------------------------------------------------------------------------- 1 | metadata description = 'Creates an Azure Cognitive Services instance.' 2 | param name string 3 | param location string = resourceGroup().location 4 | param tags object = {} 5 | @description('The custom subdomain name used to access the API. Defaults to the value of the name parameter.') 6 | param customSubDomainName string = name 7 | param disableLocalAuth bool = false 8 | param deployments array = [] 9 | param kind string = 'OpenAI' 10 | 11 | @allowed([ 'Enabled', 'Disabled' ]) 12 | param publicNetworkAccess string = 'Enabled' 13 | param sku object = { 14 | name: 'S0' 15 | } 16 | 17 | param allowedIpRules array = [] 18 | param networkAcls object = empty(allowedIpRules) ? { 19 | defaultAction: 'Allow' 20 | } : { 21 | ipRules: allowedIpRules 22 | defaultAction: 'Deny' 23 | } 24 | 25 | resource account 'Microsoft.CognitiveServices/accounts@2023-05-01' = { 26 | name: name 27 | location: location 28 | tags: tags 29 | kind: kind 30 | properties: { 31 | customSubDomainName: customSubDomainName 32 | publicNetworkAccess: publicNetworkAccess 33 | networkAcls: networkAcls 34 | disableLocalAuth: disableLocalAuth 35 | } 36 | sku: sku 37 | } 38 | 39 | @batchSize(1) 40 | resource deployment 'Microsoft.CognitiveServices/accounts/deployments@2023-05-01' = [for deployment in deployments: { 41 | parent: account 42 | name: deployment.name 43 | properties: { 44 | model: deployment.model 45 | raiPolicyName: contains(deployment, 'raiPolicyName') ? deployment.raiPolicyName : null 46 | } 47 | sku: contains(deployment, 'sku') ? deployment.sku : { 48 | name: 'Standard' 49 | capacity: 20 50 | } 51 | }] 52 | 53 | output endpoint string = account.properties.endpoint 54 | output endpoints object = account.properties.endpoints 55 | output id string = account.id 56 | output name string = account.name 57 | -------------------------------------------------------------------------------- /infra/core/host/container-app-upsert.bicep: -------------------------------------------------------------------------------- 1 | metadata description = 'Creates or updates an existing Azure Container App.' 2 | param name string 3 | param location string = resourceGroup().location 4 | param tags object = {} 5 | 6 | @description('The environment name for the container apps') 7 | param containerAppsEnvironmentName string 8 | 9 | @description('The number of CPU cores allocated to a single container instance, e.g., 0.5') 10 | param containerCpuCoreCount string = '0.5' 11 | 12 | @description('The maximum number of replicas to run. Must be at least 1.') 13 | @minValue(1) 14 | param containerMaxReplicas int = 10 15 | 16 | @description('The amount of memory allocated to a single container instance, e.g., 1Gi') 17 | param containerMemory string = '1.0Gi' 18 | 19 | @description('The minimum number of replicas to run. Must be at least 1.') 20 | @minValue(1) 21 | param containerMinReplicas int = 1 22 | 23 | @description('The name of the container') 24 | param containerName string = 'main' 25 | 26 | @description('The name of the container registry') 27 | param containerRegistryName string = '' 28 | 29 | @description('Hostname suffix for container registry. Set when deploying to sovereign clouds') 30 | param containerRegistryHostSuffix string = 'azurecr.io' 31 | 32 | @allowed([ 'http', 'grpc' ]) 33 | @description('The protocol used by Dapr to connect to the app, e.g., HTTP or gRPC') 34 | param daprAppProtocol string = 'http' 35 | 36 | @description('Enable or disable Dapr for the container app') 37 | param daprEnabled bool = false 38 | 39 | @description('The Dapr app ID') 40 | param daprAppId string = containerName 41 | 42 | @description('Specifies if the resource already exists') 43 | param exists bool = false 44 | 45 | @description('Specifies if Ingress is enabled for the container app') 46 | param ingressEnabled bool = true 47 | 48 | @description('The type of identity for the resource') 49 | @allowed([ 'None', 'SystemAssigned', 'UserAssigned' ]) 50 | param identityType string = 'None' 51 | 52 | @description('The name of the user-assigned identity') 53 | param identityName string = '' 54 | 55 | @description('The name of the container image') 56 | param imageName string = '' 57 | 58 | @description('The secrets required for the container') 59 | @secure() 60 | param secrets object = {} 61 | 62 | @description('The environment variables for the container') 63 | param env array = [] 64 | 65 | @description('Specifies if the resource ingress is exposed externally') 66 | param external bool = true 67 | 68 | @description('The service binds associated with the container') 69 | param serviceBinds array = [] 70 | 71 | @description('The target port for the container') 72 | param targetPort int = 80 73 | 74 | resource existingApp 'Microsoft.App/containerApps@2023-05-02-preview' existing = if (exists) { 75 | name: name 76 | } 77 | 78 | module app 'container-app.bicep' = { 79 | name: '${deployment().name}-update' 80 | params: { 81 | name: name 82 | location: location 83 | tags: tags 84 | identityType: identityType 85 | identityName: identityName 86 | ingressEnabled: ingressEnabled 87 | containerName: containerName 88 | containerAppsEnvironmentName: containerAppsEnvironmentName 89 | containerRegistryName: containerRegistryName 90 | containerRegistryHostSuffix: containerRegistryHostSuffix 91 | containerCpuCoreCount: containerCpuCoreCount 92 | containerMemory: containerMemory 93 | containerMinReplicas: containerMinReplicas 94 | containerMaxReplicas: containerMaxReplicas 95 | daprEnabled: daprEnabled 96 | daprAppId: daprAppId 97 | daprAppProtocol: daprAppProtocol 98 | secrets: secrets 99 | external: external 100 | env: env 101 | imageName: !empty(imageName) ? imageName : exists ? existingApp.properties.template.containers[0].image : '' 102 | targetPort: targetPort 103 | serviceBinds: serviceBinds 104 | } 105 | } 106 | 107 | output defaultDomain string = app.outputs.defaultDomain 108 | output imageName string = app.outputs.imageName 109 | output name string = app.outputs.name 110 | output uri string = app.outputs.uri 111 | -------------------------------------------------------------------------------- /infra/core/host/container-app.bicep: -------------------------------------------------------------------------------- 1 | metadata description = 'Creates a container app in an Azure Container App environment.' 2 | param name string 3 | param location string = resourceGroup().location 4 | param tags object = {} 5 | 6 | @description('Allowed origins') 7 | param allowedOrigins array = [] 8 | 9 | @description('Name of the environment for container apps') 10 | param containerAppsEnvironmentName string 11 | 12 | @description('CPU cores allocated to a single container instance, e.g., 0.5') 13 | param containerCpuCoreCount string = '0.5' 14 | 15 | @description('The maximum number of replicas to run. Must be at least 1.') 16 | @minValue(1) 17 | param containerMaxReplicas int = 10 18 | 19 | @description('Memory allocated to a single container instance, e.g., 1Gi') 20 | param containerMemory string = '1.0Gi' 21 | 22 | @description('The minimum number of replicas to run. Must be at least 1.') 23 | param containerMinReplicas int = 1 24 | 25 | @description('The name of the container') 26 | param containerName string = 'main' 27 | 28 | @description('The name of the container registry') 29 | param containerRegistryName string = '' 30 | 31 | @description('Hostname suffix for container registry. Set when deploying to sovereign clouds') 32 | param containerRegistryHostSuffix string = 'azurecr.io' 33 | 34 | @description('The protocol used by Dapr to connect to the app, e.g., http or grpc') 35 | @allowed([ 'http', 'grpc' ]) 36 | param daprAppProtocol string = 'http' 37 | 38 | @description('The Dapr app ID') 39 | param daprAppId string = containerName 40 | 41 | @description('Enable Dapr') 42 | param daprEnabled bool = false 43 | 44 | @description('The environment variables for the container') 45 | param env array = [] 46 | 47 | @description('Specifies if the resource ingress is exposed externally') 48 | param external bool = true 49 | 50 | @description('The name of the user-assigned identity') 51 | param identityName string = '' 52 | 53 | @description('The type of identity for the resource') 54 | @allowed([ 'None', 'SystemAssigned', 'UserAssigned' ]) 55 | param identityType string = 'None' 56 | 57 | @description('The name of the container image') 58 | param imageName string = '' 59 | 60 | @description('Specifies if Ingress is enabled for the container app') 61 | param ingressEnabled bool = true 62 | 63 | param revisionMode string = 'Single' 64 | 65 | @description('The secrets required for the container') 66 | @secure() 67 | param secrets object = {} 68 | 69 | @description('The service binds associated with the container') 70 | param serviceBinds array = [] 71 | 72 | @description('The name of the container apps add-on to use. e.g. redis') 73 | param serviceType string = '' 74 | 75 | @description('The target port for the container') 76 | param targetPort int = 80 77 | 78 | resource userIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (!empty(identityName)) { 79 | name: identityName 80 | } 81 | 82 | // Private registry support requires both an ACR name and a User Assigned managed identity 83 | var usePrivateRegistry = !empty(identityName) && !empty(containerRegistryName) 84 | 85 | // Automatically set to `UserAssigned` when an `identityName` has been set 86 | var normalizedIdentityType = !empty(identityName) ? 'UserAssigned' : identityType 87 | 88 | module containerRegistryAccess '../security/registry-access.bicep' = if (usePrivateRegistry) { 89 | name: '${deployment().name}-registry-access' 90 | params: { 91 | containerRegistryName: containerRegistryName 92 | principalId: usePrivateRegistry ? userIdentity.properties.principalId : '' 93 | } 94 | } 95 | 96 | resource app 'Microsoft.App/containerApps@2023-05-02-preview' = { 97 | name: name 98 | location: location 99 | tags: tags 100 | // It is critical that the identity is granted ACR pull access before the app is created 101 | // otherwise the container app will throw a provision error 102 | // This also forces us to use an user assigned managed identity since there would no way to 103 | // provide the system assigned identity with the ACR pull access before the app is created 104 | dependsOn: usePrivateRegistry ? [ containerRegistryAccess ] : [] 105 | identity: { 106 | type: normalizedIdentityType 107 | userAssignedIdentities: !empty(identityName) && normalizedIdentityType == 'UserAssigned' ? { '${userIdentity.id}': {} } : null 108 | } 109 | properties: { 110 | managedEnvironmentId: containerAppsEnvironment.id 111 | configuration: { 112 | activeRevisionsMode: revisionMode 113 | ingress: ingressEnabled ? { 114 | external: external 115 | targetPort: targetPort 116 | transport: 'auto' 117 | corsPolicy: { 118 | allowedOrigins: union([ 'https://portal.azure.com', 'https://ms.portal.azure.com' ], allowedOrigins) 119 | } 120 | } : null 121 | dapr: daprEnabled ? { 122 | enabled: true 123 | appId: daprAppId 124 | appProtocol: daprAppProtocol 125 | appPort: ingressEnabled ? targetPort : 0 126 | } : { enabled: false } 127 | secrets: [for secret in items(secrets): { 128 | name: secret.key 129 | value: secret.value 130 | }] 131 | service: !empty(serviceType) ? { type: serviceType } : null 132 | registries: usePrivateRegistry ? [ 133 | { 134 | server: '${containerRegistryName}.${containerRegistryHostSuffix}' 135 | identity: userIdentity.id 136 | } 137 | ] : [] 138 | } 139 | template: { 140 | serviceBinds: !empty(serviceBinds) ? serviceBinds : null 141 | containers: [ 142 | { 143 | image: !empty(imageName) ? imageName : 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' 144 | name: containerName 145 | env: env 146 | resources: { 147 | cpu: json(containerCpuCoreCount) 148 | memory: containerMemory 149 | } 150 | } 151 | ] 152 | scale: { 153 | minReplicas: containerMinReplicas 154 | maxReplicas: containerMaxReplicas 155 | } 156 | } 157 | } 158 | } 159 | 160 | resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' existing = { 161 | name: containerAppsEnvironmentName 162 | } 163 | 164 | output defaultDomain string = containerAppsEnvironment.properties.defaultDomain 165 | output identityPrincipalId string = normalizedIdentityType == 'None' ? '' : (empty(identityName) ? app.identity.principalId : userIdentity.properties.principalId) 166 | output imageName string = imageName 167 | output name string = app.name 168 | output serviceBind object = !empty(serviceType) ? { serviceId: app.id, name: name } : {} 169 | output uri string = ingressEnabled ? 'https://${app.properties.configuration.ingress.fqdn}' : '' 170 | -------------------------------------------------------------------------------- /infra/core/host/container-apps-environment.bicep: -------------------------------------------------------------------------------- 1 | metadata description = 'Creates an Azure Container Apps environment.' 2 | param name string 3 | param location string = resourceGroup().location 4 | param tags object = {} 5 | 6 | @description('Name of the Application Insights resource') 7 | param applicationInsightsName string = '' 8 | 9 | @description('Specifies if Dapr is enabled') 10 | param daprEnabled bool = false 11 | 12 | @description('Name of the Log Analytics workspace') 13 | param logAnalyticsWorkspaceName string 14 | 15 | resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' = { 16 | name: name 17 | location: location 18 | tags: tags 19 | properties: { 20 | appLogsConfiguration: { 21 | destination: 'log-analytics' 22 | logAnalyticsConfiguration: { 23 | customerId: logAnalyticsWorkspace.properties.customerId 24 | sharedKey: logAnalyticsWorkspace.listKeys().primarySharedKey 25 | } 26 | } 27 | daprAIInstrumentationKey: daprEnabled && !empty(applicationInsightsName) ? applicationInsights.properties.InstrumentationKey : '' 28 | } 29 | } 30 | 31 | resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = { 32 | name: logAnalyticsWorkspaceName 33 | } 34 | 35 | resource applicationInsights 'Microsoft.Insights/components@2020-02-02' existing = if (daprEnabled && !empty(applicationInsightsName)) { 36 | name: applicationInsightsName 37 | } 38 | 39 | output defaultDomain string = containerAppsEnvironment.properties.defaultDomain 40 | output id string = containerAppsEnvironment.id 41 | output name string = containerAppsEnvironment.name 42 | -------------------------------------------------------------------------------- /infra/core/host/container-apps.bicep: -------------------------------------------------------------------------------- 1 | metadata description = 'Creates an Azure Container Registry and an Azure Container Apps environment.' 2 | param name string 3 | param location string = resourceGroup().location 4 | param tags object = {} 5 | 6 | param containerAppsEnvironmentName string 7 | param containerRegistryName string 8 | param containerRegistryResourceGroupName string = '' 9 | param containerRegistryAdminUserEnabled bool = false 10 | param logAnalyticsWorkspaceName string 11 | param applicationInsightsName string = '' 12 | param daprEnabled bool = false 13 | 14 | module containerAppsEnvironment 'container-apps-environment.bicep' = { 15 | name: '${name}-container-apps-environment' 16 | params: { 17 | name: containerAppsEnvironmentName 18 | location: location 19 | tags: tags 20 | logAnalyticsWorkspaceName: logAnalyticsWorkspaceName 21 | applicationInsightsName: applicationInsightsName 22 | daprEnabled: daprEnabled 23 | } 24 | } 25 | 26 | module containerRegistry 'container-registry.bicep' = { 27 | name: '${name}-container-registry' 28 | scope: !empty(containerRegistryResourceGroupName) ? resourceGroup(containerRegistryResourceGroupName) : resourceGroup() 29 | params: { 30 | name: containerRegistryName 31 | location: location 32 | adminUserEnabled: containerRegistryAdminUserEnabled 33 | tags: tags 34 | } 35 | } 36 | 37 | output defaultDomain string = containerAppsEnvironment.outputs.defaultDomain 38 | output environmentName string = containerAppsEnvironment.outputs.name 39 | output environmentId string = containerAppsEnvironment.outputs.id 40 | 41 | output registryLoginServer string = containerRegistry.outputs.loginServer 42 | output registryName string = containerRegistry.outputs.name 43 | -------------------------------------------------------------------------------- /infra/core/host/container-registry.bicep: -------------------------------------------------------------------------------- 1 | metadata description = 'Creates an Azure Container Registry.' 2 | param name string 3 | param location string = resourceGroup().location 4 | param tags object = {} 5 | 6 | @description('Indicates whether admin user is enabled') 7 | param adminUserEnabled bool = false 8 | 9 | @description('Indicates whether anonymous pull is enabled') 10 | param anonymousPullEnabled bool = false 11 | 12 | @description('Azure ad authentication as arm policy settings') 13 | param azureADAuthenticationAsArmPolicy object = { 14 | status: 'enabled' 15 | } 16 | 17 | @description('Indicates whether data endpoint is enabled') 18 | param dataEndpointEnabled bool = false 19 | 20 | @description('Encryption settings') 21 | param encryption object = { 22 | status: 'disabled' 23 | } 24 | 25 | @description('Export policy settings') 26 | param exportPolicy object = { 27 | status: 'enabled' 28 | } 29 | 30 | @description('Metadata search settings') 31 | param metadataSearch string = 'Disabled' 32 | 33 | @description('Options for bypassing network rules') 34 | param networkRuleBypassOptions string = 'AzureServices' 35 | 36 | @description('Public network access setting') 37 | param publicNetworkAccess string = 'Enabled' 38 | 39 | @description('Quarantine policy settings') 40 | param quarantinePolicy object = { 41 | status: 'disabled' 42 | } 43 | 44 | @description('Retention policy settings') 45 | param retentionPolicy object = { 46 | days: 7 47 | status: 'disabled' 48 | } 49 | 50 | @description('Scope maps setting') 51 | param scopeMaps array = [] 52 | 53 | @description('SKU settings') 54 | param sku object = { 55 | name: 'Basic' 56 | } 57 | 58 | @description('Soft delete policy settings') 59 | param softDeletePolicy object = { 60 | retentionDays: 7 61 | status: 'disabled' 62 | } 63 | 64 | @description('Trust policy settings') 65 | param trustPolicy object = { 66 | type: 'Notary' 67 | status: 'disabled' 68 | } 69 | 70 | @description('Zone redundancy setting') 71 | param zoneRedundancy string = 'Disabled' 72 | 73 | @description('The log analytics workspace ID used for logging and monitoring') 74 | param workspaceId string = '' 75 | 76 | // 2023-11-01-preview needed for metadataSearch 77 | resource containerRegistry 'Microsoft.ContainerRegistry/registries@2023-11-01-preview' = { 78 | name: name 79 | location: location 80 | tags: tags 81 | sku: sku 82 | properties: { 83 | adminUserEnabled: adminUserEnabled 84 | anonymousPullEnabled: anonymousPullEnabled 85 | dataEndpointEnabled: dataEndpointEnabled 86 | encryption: encryption 87 | metadataSearch: metadataSearch 88 | networkRuleBypassOptions: networkRuleBypassOptions 89 | policies:{ 90 | quarantinePolicy: quarantinePolicy 91 | trustPolicy: trustPolicy 92 | retentionPolicy: retentionPolicy 93 | exportPolicy: exportPolicy 94 | azureADAuthenticationAsArmPolicy: azureADAuthenticationAsArmPolicy 95 | softDeletePolicy: softDeletePolicy 96 | } 97 | publicNetworkAccess: publicNetworkAccess 98 | zoneRedundancy: zoneRedundancy 99 | } 100 | 101 | resource scopeMap 'scopeMaps' = [for scopeMap in scopeMaps: { 102 | name: scopeMap.name 103 | properties: scopeMap.properties 104 | }] 105 | } 106 | 107 | // TODO: Update diagnostics to be its own module 108 | // Blocking issue: https://github.com/Azure/bicep/issues/622 109 | // Unable to pass in a `resource` scope or unable to use string interpolation in resource types 110 | resource diagnostics 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = if (!empty(workspaceId)) { 111 | name: 'registry-diagnostics' 112 | scope: containerRegistry 113 | properties: { 114 | workspaceId: workspaceId 115 | logs: [ 116 | { 117 | category: 'ContainerRegistryRepositoryEvents' 118 | enabled: true 119 | } 120 | { 121 | category: 'ContainerRegistryLoginEvents' 122 | enabled: true 123 | } 124 | ] 125 | metrics: [ 126 | { 127 | category: 'AllMetrics' 128 | enabled: true 129 | timeGrain: 'PT1M' 130 | } 131 | ] 132 | } 133 | } 134 | 135 | output id string = containerRegistry.id 136 | output loginServer string = containerRegistry.properties.loginServer 137 | output name string = containerRegistry.name 138 | -------------------------------------------------------------------------------- /infra/core/monitor/loganalytics.bicep: -------------------------------------------------------------------------------- 1 | metadata description = 'Creates a Log Analytics workspace.' 2 | param name string 3 | param location string = resourceGroup().location 4 | param tags object = {} 5 | 6 | resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { 7 | name: name 8 | location: location 9 | tags: tags 10 | properties: any({ 11 | retentionInDays: 30 12 | features: { 13 | searchVersion: 1 14 | } 15 | sku: { 16 | name: 'PerGB2018' 17 | } 18 | }) 19 | } 20 | 21 | output id string = logAnalytics.id 22 | output name string = logAnalytics.name 23 | -------------------------------------------------------------------------------- /infra/core/security/registry-access.bicep: -------------------------------------------------------------------------------- 1 | metadata description = 'Assigns ACR Pull permissions to access an Azure Container Registry.' 2 | param containerRegistryName string 3 | param principalId string 4 | 5 | var acrPullRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') 6 | 7 | resource aksAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { 8 | scope: containerRegistry // Use when specifying a scope that is different than the deployment scope 9 | name: guid(subscription().id, resourceGroup().id, principalId, acrPullRole) 10 | properties: { 11 | roleDefinitionId: acrPullRole 12 | principalType: 'ServicePrincipal' 13 | principalId: principalId 14 | } 15 | } 16 | 17 | resource containerRegistry 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' existing = { 18 | name: containerRegistryName 19 | } 20 | -------------------------------------------------------------------------------- /infra/core/security/role.bicep: -------------------------------------------------------------------------------- 1 | metadata description = 'Creates a role assignment for a service principal.' 2 | param principalId string 3 | 4 | @allowed([ 5 | 'Device' 6 | 'ForeignGroup' 7 | 'Group' 8 | 'ServicePrincipal' 9 | 'User' 10 | ]) 11 | param principalType string = 'ServicePrincipal' 12 | param roleDefinitionId string 13 | 14 | resource role 'Microsoft.Authorization/roleAssignments@2022-04-01' = { 15 | name: guid(subscription().id, resourceGroup().id, principalId, roleDefinitionId) 16 | properties: { 17 | principalId: principalId 18 | principalType: principalType 19 | roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', roleDefinitionId) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /infra/getkey.sh: -------------------------------------------------------------------------------- 1 | if [ -z "$(az account show)" ]; then 2 | echo "You are not logged in. Please run 'az login' or 'az login --use-device-code' first." 3 | exit 1 4 | fi 5 | 6 | echo "Logged in, using this subscription:" 7 | az account show --query "{subscriptionId:id, name:name}" 8 | echo "If that is not the correct subscription, please run 'az account set --subscription \"\"'" 9 | 10 | echo "Getting environment variables from .env file..." 11 | openAiService=$(grep "AZURE_OPENAI_RESOURCE=" .env | cut -d '=' -f2 | tr -d '"') 12 | resourceGroupName=$(grep "AZURE_OPENAI_RESOURCE_GROUP=" .env | cut -d '=' -f2 | tr -d '"') 13 | 14 | echo "Getting OpenAI key from $openAiService in resourceGroup $resourceGroupName..." 15 | openAiKey=$(az cognitiveservices account keys list --name $openAiService --resource-group $resourceGroupName --query key1 --output tsv) 16 | 17 | echo "AZURE_OPENAI_KEY_FOR_CHATVISION=\"$openAiKey\"" >> .env 18 | 19 | echo "OpenAI key has been saved to .env file." 20 | -------------------------------------------------------------------------------- /infra/main.bicep: -------------------------------------------------------------------------------- 1 | targetScope = 'subscription' 2 | 3 | @minLength(1) 4 | @maxLength(64) 5 | @description('Name which is used to generate a short unique hash for each resource') 6 | param name string 7 | 8 | @minLength(1) 9 | @description('Primary location for all resources') 10 | param location string 11 | 12 | @description('Id of the user or app to assign application roles') 13 | param principalId string = '' 14 | 15 | @description('Flag to decide where to create OpenAI role for current user') 16 | param createRoleForUser bool = true 17 | 18 | @minLength(1) 19 | @description('Location for the OpenAI resource') 20 | // Look for desired models on the availability table: 21 | // https://learn.microsoft.com/azure/ai-services/openai/concepts/models#global-standard-model-availability 22 | @allowed([ 23 | 'australiaeast' 24 | 'brazilsouth' 25 | 'canadaeast' 26 | 'eastus' 27 | 'eastus2' 28 | 'francecentral' 29 | 'germanywestcentral' 30 | 'japaneast' 31 | 'koreacentral' 32 | 'northcentralus' 33 | 'norwayeast' 34 | 'polandcentral' 35 | 'spaincentral' 36 | 'southafricanorth' 37 | 'southcentralus' 38 | 'southindia' 39 | 'swedencentral' 40 | 'switzerlandnorth' 41 | 'uksouth' 42 | 'westeurope' 43 | 'westus' 44 | 'westus3' 45 | ]) 46 | @metadata({ 47 | azd: { 48 | type: 'location' 49 | } 50 | }) 51 | param openAILocation string 52 | 53 | // These parameters can be customized via azd env variables referenced in main.parameters.json: 54 | param openAiResourceName string = '' 55 | param openAiResourceGroupName string = '' 56 | param openAiApiVersion string = '' 57 | param disableKeyBasedAuth bool = true 58 | // These parameters can be customized, but are set to default values in main.parameters.json: 59 | param openAiSkuName string 60 | param openAiModelName string 61 | param openAiModelVersion string 62 | param openAiDeploymentName string 63 | param openAiDeploymentCapacity int 64 | param openAiDeploymentSkuName string 65 | 66 | @description('Flag to decide whether to create Azure OpenAI instance or not') 67 | param createAzureOpenAi bool = true 68 | 69 | @description('Azure OpenAI key to use for authentication. If not provided, managed identity will be used (and is preferred)') 70 | @secure() 71 | param openAiKey string = '' 72 | 73 | @description('Azure OpenAI endpoint to use. If provided, no Azure OpenAI instance will be created.') 74 | param openAiEndpoint string = '' 75 | 76 | param acaExists bool = false 77 | 78 | var resourceToken = toLower(uniqueString(subscription().id, name, location)) 79 | var tags = { 'azd-env-name': name } 80 | 81 | resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { 82 | name: '${name}-rg' 83 | location: location 84 | tags: tags 85 | } 86 | 87 | resource openAiResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = if (!empty(openAiResourceGroupName)) { 88 | name: !empty(openAiResourceGroupName) ? openAiResourceGroupName : resourceGroup.name 89 | } 90 | 91 | var prefix = '${name}-${resourceToken}' 92 | 93 | module openAi 'core/ai/cognitiveservices.bicep' = if (createAzureOpenAi) { 94 | name: 'openai' 95 | scope: openAiResourceGroup 96 | params: { 97 | name: !empty(openAiResourceName) ? openAiResourceName : '${resourceToken}-cog' 98 | location: openAILocation 99 | tags: tags 100 | disableLocalAuth: disableKeyBasedAuth 101 | sku: { 102 | name: openAiSkuName 103 | } 104 | deployments: [ 105 | { 106 | name: openAiDeploymentName 107 | model: { 108 | format: 'OpenAI' 109 | name: openAiModelName 110 | version: openAiModelVersion 111 | } 112 | sku: { 113 | name: openAiDeploymentSkuName 114 | capacity: openAiDeploymentCapacity 115 | } 116 | } 117 | ] 118 | } 119 | } 120 | 121 | module logAnalyticsWorkspace 'core/monitor/loganalytics.bicep' = { 122 | name: 'loganalytics' 123 | scope: resourceGroup 124 | params: { 125 | name: '${prefix}-loganalytics' 126 | location: location 127 | tags: tags 128 | } 129 | } 130 | 131 | // Container apps host (including container registry) 132 | module containerApps 'core/host/container-apps.bicep' = { 133 | name: 'container-apps' 134 | scope: resourceGroup 135 | params: { 136 | name: 'app' 137 | location: location 138 | tags: tags 139 | containerAppsEnvironmentName: '${prefix}-containerapps-env' 140 | containerRegistryName: '${replace(prefix, '-', '')}registry' 141 | logAnalyticsWorkspaceName: logAnalyticsWorkspace.outputs.name 142 | } 143 | } 144 | 145 | // Container app frontend 146 | module aca 'aca.bicep' = { 147 | name: 'aca' 148 | scope: resourceGroup 149 | params: { 150 | name: replace('${take(prefix,19)}-ca', '--', '-') 151 | location: location 152 | tags: tags 153 | identityName: '${prefix}-id-aca' 154 | containerAppsEnvironmentName: containerApps.outputs.environmentName 155 | containerRegistryName: containerApps.outputs.registryName 156 | openAiDeploymentName: openAiDeploymentName 157 | openAiEndpoint: createAzureOpenAi ? openAi.outputs.endpoint : openAiEndpoint 158 | openAiApiVersion: openAiApiVersion 159 | openAiKey: openAiKey 160 | exists: acaExists 161 | } 162 | } 163 | 164 | 165 | module openAiRoleUser 'core/security/role.bicep' = if (createRoleForUser && createAzureOpenAi) { 166 | scope: openAiResourceGroup 167 | name: 'openai-role-user' 168 | params: { 169 | principalId: principalId 170 | roleDefinitionId: '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' 171 | principalType: 'User' 172 | } 173 | } 174 | 175 | 176 | module openAiRoleBackend 'core/security/role.bicep' = if (createAzureOpenAi) { 177 | scope: openAiResourceGroup 178 | name: 'openai-role-backend' 179 | params: { 180 | principalId: aca.outputs.SERVICE_ACA_IDENTITY_PRINCIPAL_ID 181 | roleDefinitionId: '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' 182 | principalType: 'ServicePrincipal' 183 | } 184 | } 185 | 186 | output AZURE_LOCATION string = location 187 | output AZURE_RESOURCE_GROUP string = resourceGroup.name 188 | output AZURE_TENANT_ID string = tenant().tenantId 189 | 190 | output AZURE_OPENAI_RESOURCE_NAME string = openAi.outputs.name 191 | output AZURE_OPENAI_DEPLOYMENT string = openAiDeploymentName 192 | output AZURE_OPENAI_API_VERSION string = openAiApiVersion 193 | output AZURE_OPENAI_ENDPOINT string = createAzureOpenAi ? openAi.outputs.endpoint : openAiEndpoint 194 | 195 | output SERVICE_ACA_IDENTITY_PRINCIPAL_ID string = aca.outputs.SERVICE_ACA_IDENTITY_PRINCIPAL_ID 196 | output SERVICE_ACA_NAME string = aca.outputs.SERVICE_ACA_NAME 197 | output SERVICE_ACA_URI string = aca.outputs.SERVICE_ACA_URI 198 | output SERVICE_ACA_IMAGE_NAME string = aca.outputs.SERVICE_ACA_IMAGE_NAME 199 | 200 | output AZURE_CONTAINER_ENVIRONMENT_NAME string = containerApps.outputs.environmentName 201 | output AZURE_CONTAINER_REGISTRY_ENDPOINT string = containerApps.outputs.registryLoginServer 202 | output AZURE_CONTAINER_REGISTRY_NAME string = containerApps.outputs.registryName 203 | -------------------------------------------------------------------------------- /infra/main.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "name": { 6 | "value": "${AZURE_ENV_NAME}" 7 | }, 8 | "location": { 9 | "value": "${AZURE_LOCATION}" 10 | }, 11 | "principalId": { 12 | "value": "${AZURE_PRINCIPAL_ID}" 13 | }, 14 | "openAILocation": { 15 | "value": "${AZURE_OPENAI_LOCATION}" 16 | }, 17 | "openAiModelName": { 18 | "value": "${AZURE_OPENAI_MODEL=gpt-4o}" 19 | }, 20 | "openAiModelVersion": { 21 | "value": "${AZURE_OPENAI_MODEL_VERSION=2024-05-13}" 22 | }, 23 | "openAiDeploymentName": { 24 | "value": "${AZURE_OPENAI_DEPLOYMENT=gpt-4o}" 25 | }, 26 | "openAiDeploymentCapacity": { 27 | "value": "${AZURE_OPENAI_DEPLOYMENT_CAPACITY=30}" 28 | }, 29 | "openAiDeploymentSkuName": { 30 | "value": "${AZURE_OPENAI_DEPLOYMENT_SKU_NAME=GlobalStandard}" 31 | }, 32 | "openAiResourceName": { 33 | "value": "${AZURE_OPENAI_RESOURCE}" 34 | }, 35 | "openAiResourceGroupName": { 36 | "value": "${AZURE_OPENAI_RESOURCE_GROUP}" 37 | }, 38 | "openAiResourceGroupLocation": { 39 | "value": "${AZURE_OPENAI_RESOURCE_GROUP_LOCATION}" 40 | }, 41 | "openAiSkuName": { 42 | "value": "${AZURE_OPENAI_SKU_NAME=S0}" 43 | }, 44 | "openAiApiVersion": { 45 | "value": "${AZURE_OPENAI_API_VERSION}" 46 | }, 47 | "createAzureOpenAi": { 48 | "value": "${CREATE_AZURE_OPENAI=true}" 49 | }, 50 | "openAiKey": { 51 | "value": "${AZURE_OPENAI_KEY_FOR_CHATVISION}" 52 | }, 53 | "openAiEndpoint": { 54 | "value": "${AZURE_OPENAI_ENDPOINT}" 55 | }, 56 | "createRoleForUser": { 57 | "value": "${CREATE_ROLE_FOR_USER=true}" 58 | }, 59 | "acaExists": { 60 | "value": "${SERVICE_ACA_RESOURCE_EXISTS=false}" 61 | }, 62 | "disableKeyBasedAuth": { 63 | "value": "${DISABLE_KEY_BASED_AUTH=true}" 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /notebooks/azure_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/azure_arch.png -------------------------------------------------------------------------------- /notebooks/chat_pdf_images.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Chat with PDF page images\n", 8 | "\n", 9 | "**If you're looking or the web application, check the src/ folder.** \n", 10 | "\n", 11 | "This notebook demonstrates how to convert PDF pages to images and send them to a vision model for inference" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "## Authenticate to OpenAI\n", 19 | "\n", 20 | "The following code connects to OpenAI, either using an Azure OpenAI account, GitHub models, or local Ollama model. See the README for instruction on configuring the `.env` file." 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 8, 26 | "metadata": {}, 27 | "outputs": [ 28 | { 29 | "name": "stdout", 30 | "output_type": "stream", 31 | "text": [ 32 | "Using GitHub Models with GITHUB_TOKEN as key\n", 33 | "Using model gpt-4o\n" 34 | ] 35 | } 36 | ], 37 | "source": [ 38 | "import os\n", 39 | "\n", 40 | "import azure.identity\n", 41 | "import openai\n", 42 | "from dotenv import load_dotenv\n", 43 | "\n", 44 | "load_dotenv(\".env\", override=True)\n", 45 | "\n", 46 | "openai_host = os.getenv(\"OPENAI_HOST\", \"github\")\n", 47 | "model_name = os.getenv(\"OPENAI_MODEL\", \"gpt-4o\")\n", 48 | "\n", 49 | "if openai_host == \"local\":\n", 50 | " print(\"Using local OpenAI-compatible API with no key\")\n", 51 | " openai_client = openai.OpenAI(api_key=\"no-key-required\", base_url=os.environ[\"LOCAL_OPENAI_ENDPOINT\"])\n", 52 | "elif openai_host == \"github\":\n", 53 | " print(\"Using GitHub Models with GITHUB_TOKEN as key\")\n", 54 | " openai_client = openai.OpenAI(\n", 55 | " api_key=os.environ[\"GITHUB_TOKEN\"],\n", 56 | " base_url=\"https://models.inference.ai.azure.com\",\n", 57 | " )\n", 58 | "elif openai_host == \"azure\" and os.getenv(\"AZURE_OPENAI_KEY_FOR_CHATVISION\"):\n", 59 | " # Authenticate using an Azure OpenAI API key\n", 60 | " # This is generally discouraged, but is provided as a convenience\n", 61 | " print(\"Using Azure OpenAI with key\")\n", 62 | " openai_client = openai.AzureOpenAI(\n", 63 | " api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\") or \"2024-02-15-preview\",\n", 64 | " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", 65 | " api_key=os.environ[\"AZURE_OPENAI_KEY_FOR_CHATVISION\"],\n", 66 | " )\n", 67 | "elif openai_host == \"azure\" and os.getenv(\"AZURE_OPENAI_ENDPOINT\"):\n", 68 | " tenant_id = os.environ[\"AZURE_TENANT_ID\"]\n", 69 | " print(\"Using Azure OpenAI with Azure Developer CLI credential for tenant id\", tenant_id)\n", 70 | " default_credential = azure.identity.AzureDeveloperCliCredential(tenant_id=tenant_id)\n", 71 | " token_provider = azure.identity.get_bearer_token_provider(\n", 72 | " default_credential, \"https://cognitiveservices.azure.com/.default\"\n", 73 | " )\n", 74 | " openai_client = openai.AzureOpenAI(\n", 75 | " api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\") or \"2024-02-15-preview\",\n", 76 | " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", 77 | " azure_ad_token_provider=token_provider,\n", 78 | " )\n", 79 | "print(f\"Using model {model_name}\")" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "## Convert PDFs to images" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 9, 92 | "metadata": {}, 93 | "outputs": [ 94 | { 95 | "name": "stdout", 96 | "output_type": "stream", 97 | "text": [ 98 | "Defaulting to user installation because normal site-packages is not writeable\n", 99 | "Requirement already satisfied: Pillow in /home/vscode/.local/lib/python3.11/site-packages (11.1.0)\n", 100 | "Requirement already satisfied: PyMuPDF in /home/vscode/.local/lib/python3.11/site-packages (1.25.3)\n", 101 | "\n", 102 | "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.0.1\u001b[0m\n", 103 | "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", 104 | "Note: you may need to restart the kernel to use updated packages.\n" 105 | ] 106 | } 107 | ], 108 | "source": [ 109 | "%pip install Pillow PyMuPDF" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 10, 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "import pymupdf\n", 119 | "from PIL import Image\n", 120 | "\n", 121 | "filename = \"plants.pdf\"\n", 122 | "doc = pymupdf.open(filename)\n", 123 | "for i in range(doc.page_count):\n", 124 | " doc = pymupdf.open(filename)\n", 125 | " page = doc.load_page(i)\n", 126 | " pix = page.get_pixmap()\n", 127 | " original_img = Image.frombytes(\"RGB\", [pix.width, pix.height], pix.samples)\n", 128 | " original_img.save(f\"page_{i}.png\")" 129 | ] 130 | }, 131 | { 132 | "cell_type": "markdown", 133 | "metadata": {}, 134 | "source": [ 135 | "## Send images to vision model" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 11, 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [ 144 | "import base64\n", 145 | "\n", 146 | "\n", 147 | "def open_image_as_base64(filename):\n", 148 | " with open(filename, \"rb\") as image_file:\n", 149 | " image_data = image_file.read()\n", 150 | " image_base64 = base64.b64encode(image_data).decode(\"utf-8\")\n", 151 | " return f\"data:image/png;base64,{image_base64}\"" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": null, 157 | "metadata": {}, 158 | "outputs": [ 159 | { 160 | "name": "stdout", 161 | "output_type": "stream", 162 | "text": [ 163 | "The plants listed on these pages from The Watershed Nursery are categorized into Annuals, Bulbs, Grasses, Perennials, and other categories. Here is a detailed list of the plants:\n", 164 | "\n", 165 | "### Annuals\n", 166 | "1. Centromadia pungens (Common tarweed)\n", 167 | "2. Epilobium densiflorum (Dense Spike-primrose)\n", 168 | "3. Eschscholzia caespitosa (Tufted Poppy)\n", 169 | "4. Eschscholzia californica (California poppy)\n", 170 | "5. Eschscholzia californica 'Purple Gleam' (Purple Gleam Poppy)\n", 171 | "6. Eschscholzia californica var. maritima (Coastal California Poppy)\n", 172 | "7. Madia elegans (Tarweed)\n", 173 | "8. Mentzelia lindleyi (Lindley's Blazing Star)\n", 174 | "9. Symphyotrichum subulatum (Slim marsh aster)\n", 175 | "10. Trichostema lanceolatum (Vinegar weed)\n", 176 | "11. Trichostema lanceolatum (Vinegar weed)\n", 177 | "\n", 178 | "### Bulbs\n", 179 | "1. Brodiaea californica (California brodiaea)\n", 180 | "2. Chlorogalum pomeridianum (Soap plant)\n", 181 | "3. Epipactis gigantea (Stream orchid)\n", 182 | "4. Wyethia angustifolia (Narrowleaf mule ears)\n", 183 | "5. Wyethia angustifolia (Narrowleaf mule ears)\n", 184 | "6. Wyethia angustifolia (Narrowleaf mule ears)\n", 185 | "7. Wyethia mollis (Woolly Mule's Ear's)\n", 186 | "\n", 187 | "### Grasses\n", 188 | "1. Agrostis pallens (Thingrass)\n", 189 | "2. Anthoxanthum occidentale (Vanilla grass)\n", 190 | "3. Bouteloua gracilis (Blue grama)\n", 191 | "4. Bouteloua gracilis (Blue grama)\n", 192 | "\n", 193 | "### Perennials\n", 194 | "1. Achillea millefolium (Yarrow)\n", 195 | "2. Ambrosia chamissonis (Silvery beachweed)\n", 196 | "3. Ambrosia chamissonis (Silvery beachweed)\n", 197 | "4. Anemopsis californica (Yerba mansa)\n", 198 | "5. Angelica hendersonii (Coast Angelica)\n", 199 | "6. Apocynum cannabinum (Hemp Dogbane)\n", 200 | "7. Apocynum cannabinum (Hemp Dogbane)\n", 201 | "8. Aquilegia eximia (Serpentine columbine)\n", 202 | "9. Aristolochia californica (California Dutchman's)\n", 203 | "10. Armeria maritima (Sea thrift)\n", 204 | "11. Artemisia douglasiana (Mugwort)\n", 205 | "12. Asarum caudatum (Wild ginger)\n", 206 | "13. Asclepias californica (California Milkweed)\n", 207 | "14. Asclepias eriocarpa (Woolypod milkweed)\n", 208 | "15. Asclepias fascicularis (Narrow leaf milkweed)\n", 209 | "16. Asclepias fascicularis (Narrow leaf milkweed)\n", 210 | "17. Asclepias fascicularis (Narrow leaf milkweed)\n", 211 | "18. Asclepias fascicularis (Narrow leaf milkweed)\n", 212 | "19. Asclepias fascicularis (Narrow leaf milkweed)\n", 213 | "20. Asclepias fascicularis (Narrow leaf milkweed)\n", 214 | "21. Asclepias fascicularis (Narrow leaf milkweed)\n", 215 | "22. Asclepias speciosa (Showy milkweed)\n", 216 | "23. Atriplex leucophylla (Beach Salt Bush)\n", 217 | "24. Baccharis glutinosa (Marsh baccharis)\n", 218 | "25. Baccharis glutinosa (Marsh baccharis)\n", 219 | "26. Baccharis glutinosa (Marsh baccharis)\n", 220 | "27. Baccharis glutinosa (Marsh baccharis)\n", 221 | "28. Baccharis glutinosa (Marsh baccharis)\n", 222 | "29. Bidens laevis (Smooth Bur Marigold)\n", 223 | "30. Bolboschoenus maritimus (Cosmopolitan bulrush)\n", 224 | "31. Calystegia occidentalis (Western chaparral)\n", 225 | "32. Calystegia purpurata ssp. saxicola (Bodega Morning Glory)\n", 226 | "33. Camissoniopsis cheiranthifolia (Beach primrose)\n", 227 | "34. Campanula rotundifolia (Round Leaf Harebell)\n", 228 | "35. Carex athrostachya (Slenderbeak sedge)\n", 229 | "36. Carex barbarae (Santa Barbara sedge)\n", 230 | "37. Carex barbarae (Santa Barbara sedge)\n", 231 | "38. Carex barbarae (Santa Barbara sedge)\n", 232 | "39. Carex nudata (California black-flowering)\n", 233 | "40. Carex obnupta (Slough sedge)\n", 234 | "41. Carex panza (California meadow sedge)\n", 235 | "42. Carex praegracilis (Deer-bed Sedge)\n", 236 | "43. Chenopodium californicum (California Goosefoot)\n", 237 | "44. Clinopodium douglasii (Yerba buena)\n", 238 | "45. Datura wrightii (Sacred Datura)\n", 239 | "46. Drymocallis glandulosa (Sticky cinquefoil)\n", 240 | "47. Drymocallis glandulosa (Sticky cinquefoil)\n", 241 | "48. Drymocallis glandulosa (Sticky cinquefoil)\n", 242 | "49. Dudleya brittonii (Giant Chalk Dudleya)\n", 243 | "50. Dudleya farinosa (Bluff lettuce)\n", 244 | "51. Dudleya pulverulenta (Chalk dudleya)\n", 245 | "52. Eleocharis macrostachya (Pale spikerush)\n", 246 | "53. Epilobium canum (California fuschia)\n", 247 | "54. Epilobium canum (California fuschia)\n", 248 | "55. Eriogon glaucus (Seaside daisy)\n", 249 | "56. Eriogon grande var. rubescens (Red-flowering buckwheat)\n", 250 | "57. Eriogon latifolium (Coast buckwheat)\n", 251 | "58. Eriogon latifolium (Coast buckwheat)\n", 252 | "59. Eriogon nudum (Buckwheat)\n", 253 | "60. Eriogon nudum 'Ella Nelson's Yellow' (Ella Nelson's Yellow)\n", 254 | "61. Erysimum capitatum Yellow Form (Foothill wallflower)\n", 255 | "62. Eschscholzia californica 'Carmine King' (Carmine King Poppy)\n", 256 | "63. Eschscholzia californica 'Moonglow' (Moonglow Poppy)\n", 257 | "64. Fragaria chiloensis (Beach strawberry)\n", 258 | "65. Fragaria chiloensis (Beach strawberry)\n", 259 | "66. Gamocheta ustulata (Featherweed)\n", 260 | "67. Glycyrrhiza lepidota (Wild licorice)\n", 261 | "68. Grindelia hirsutula (Hairy gumplant)\n", 262 | "69. Helenium puberulum (Rosilla)\n", 263 | "70. Heliotropium curvassavicum var. (Wild heliotrope)\n", 264 | "71. Heliotropium curvassavicum var. (Wild heliotrope)\n", 265 | "72. Heliotropium curvassavicum var. (Wild heliotrope)\n", 266 | "73. Heliotropium curvassavicum var. (Wild heliotrope)\n", 267 | "74. Hesperoyucca whipplei (Chaparral Yucca)\n", 268 | "75. Heuchera maxima (Island Alum Root)\n", 269 | "76. Heuchera mirantha (Coral bells)\n", 270 | "77. Heuchera mirantha (Coral bells)\n", 271 | "78. Hosackia gracilis (Harlequin Lotus)\n", 272 | "79. Iris douglasiana (Douglas iris)\n", 273 | "80. Iris douglasiana (Douglas iris)\n", 274 | "81. Juncus effusus (Common rush)\n", 275 | "82. Juncus effusus (Common rush)\n", 276 | "83. Lilium pardalinum (Leopard lily)\n", 277 | "84. Lilium pardalinum (Leopard lily)\n", 278 | "85. Lilium pardalinum (Leopard lily)\n", 279 | "86. Marah fabacea (Wild cucumber)\n", 280 | "87. Monardella macrantha (Red Monardella)\n", 281 | "88. Oxalis oregana (Redwood sorrel)\n", 282 | "89. Penstemon centranthifolius (Scarlet bugler)\n", 283 | "90. Penstemon centranthifolius (Scarlet bugler)\n", 284 | "91. Penstemon palmeri (Balloon flower penstemon)\n", 285 | "92. Penstemon pseudospectabilis (Desert Penstemon)\n", 286 | "93. Penstemon spectabilis (Showy penstemon)\n", 287 | "94. Perideridia gairdneri (Gardner's yampah)\n", 288 | "95. Phacelia californica (California phacelia)\n", 289 | "96. Phacelia nemoralis (Shade phacelia)\n", 290 | "97. Phyla nodiflora (Lippia)\n", 291 | "98. Plantago maritima (Coastal plantain)\n", 292 | "99. Ranunculus californicus (California buttercup)\n", 293 | "100. Ranunculus californicus (California buttercup)\n", 294 | "101. Ranunculus californicus (California buttercup)\n", 295 | "102. Salicornia pacifica (Pickleweed)\n", 296 | "103. Scirpus microcarpus (Small-fruited bulrush)\n", 297 | "104. Scutellaria californica (California skullcap)\n", 298 | "105. Sisyrinchium bellum (Blue-eyed grass)\n", 299 | "106. Sisyrinchium bellum (Blue-eyed grass)\n", 300 | "107. Sisyrinchium bellum (Blue-eyed grass)\n", 301 | "108. Sisyrinchium californicum (Golden-eyed grass)\n", 302 | "109. Solanum umbelliferum (Bluewitch nightshade)\n", 303 | "110. Sparganium eurycarpum (Broadfruit bur-reed)\n", 304 | "111. Stachys ajugoides (Hedge nettle)\n", 305 | "112. Stachys albus (Whitestem hedge nettle)\n", 306 | "113. Symphyotrichum chilense (California aster)\n", 307 | "114. Symphyotrichum chilense (Common aster)\n", 308 | "115. Tanacetum bipinnatum (Dune tansy)\n", 309 | "116. Tanacetum bipinnatum (Dune tansy)\n", 310 | "117. Tellima grandiflora (Fringe cups)\n", 311 | "118. Urtica dioica (Stinging nettle)\n", 312 | "119. Verbena lasiostachys (Western vervain)\n", 313 | "120. Viola adunca (Blue violet)\n", 314 | "121. Viola pedunculata (Johnny-jump-up)\n", 315 | "122. Vitis californica (California grape)\n", 316 | "123. Wyethia angustifolia (Narrowleaf mule ears)\n", 317 | "\n", 318 | "### Shrubs\n", 319 | "1. Adenostoma fasciculatum (Chamise)\n", 320 | "2. Adenostoma fasciculatum (Chamise)\n", 321 | "3. Atriplex lentiformis (Quail bush)\n", 322 | "4. Atriplex lentiformis (Quail bush)\n", 323 | "5. Baccharis pilularis (Coyote bush)\n", 324 | "6. Baccharis salicifolia (Mulefat)\n", 325 | "7. Ceanothus 'Concha' (Wild lilac)\n", 326 | "8. Ceanothus 'Concha' seed (Wild lilac)\n", 327 | "9. Ceanothus 'Dark Star' seed (Wild lilac)\n", 328 | "10. Ceanothus 'Frosty Blue' (seed) (Frosty Blue' Ceanothus)\n", 329 | "11. Ceanothus crassifolius (Hoary Leaved Ceanothus)\n", 330 | "12. Ceanothus cuneatus (Buckbrush)\n", 331 | "13. Ceanothus cuneatus (Deerbrush)\n", 332 | "14. Ceanothus integerrimus (Deerbrush)\n", 333 | "15. Ceanothus leucodermis (White bark California lilac)\n", 334 | "16. Ceanothus leucodermis (White bark California lilac)\n", 335 | "17. Ceanothus leucodermis (White bark California lilac)\n", 336 | "18. Ceanothus tomentosus (Woollyleaf Ceanothus)\n", 337 | "19. Ceanothus tomentosus (Woollyleaf Ceanothus)\n", 338 | "20. Ceanothus thyrsiflorus (Blossom ceanothus)\n", 339 | "21. Ceanothus thyrsiflorus (Blossom ceanothus)\n", 340 | "22. Ceanothus thyrsiflorus (Blossom ceanothus)\n", 341 | "23. Ceanothus thyrsiflorus (Blossom ceanothus)\n", 342 | "24. Ceanothus thyrsiflorus (Blossom ceanothus)\n", 343 | "25. Ceanothus thyrsiflorus (Blossom ceanothus)\n", 344 | "26. Cercis occidentalis (Western redbud)\n", 345 | "27. Cercis occidentalis (Western redbud)\n", 346 | "28. Cercis occidentalis (Western redbud)\n", 347 | "29. Cercis occidentalis (Western redbud)\n", 348 | "30. Cercis occidentalis (Western redbud)\n", 349 | "31. Cercis occidentalis (Western redbud)\n", 350 | "32. Cercocarpus betuloides (Mountain Mahogany)\n", 351 | "33. Cercocarpus betuloides (Mountain Mahogany)\n", 352 | "34. Cornus glabrata (Brown dogwood)\n", 353 | "35. Cornus sericea (American dogwood)\n", 354 | "36. Cornus sericea (American dogwood)\n", 355 | "37. Cornus sericea (American dogwood)\n", 356 | "38. Eriodictyon californicum (California Bush Sunflower)\n", 357 | "39. Eriodictyon californicum (Yerba santa)\n", 358 | "40. Eriodictyon californicum (Yerba santa)\n", 359 | "41. Eriodictyon californicum (Yerba santa)\n", 360 | "42. Eriogonum arborescens (Santa Cruz Island)\n", 361 | "43. Eriogonum giganteum (Saint Catherine's Lace)\n", 362 | "44. Eriogonum giganteum (Saint Catherine's Lace)\n", 363 | "45. Eriogonum giganteum (Saint Catherine's Lace)\n", 364 | "46. Frangula californica (California coffeeberry)\n", 365 | "47. Frangula californica (California coffeeberry)\n", 366 | "48. Fremontodendron californicum (Flannelbush)\n", 367 | "49. Heteromeles arbutifolia (Toyon)\n", 368 | "50. Heteromeles arbutifolia (Toyon)\n", 369 | "51. Heteromeles arbutifolia (Toyon)\n", 370 | "52. Heteromeles arbutifolia (Toyon)\n", 371 | "53. Heteromeles arbutifolia (Toyon)\n", 372 | "54. Heteromeles arbutifolia (Toyon)\n", 373 | "55. Holodiscus discolor (Ocean spray)\n", 374 | "56. Holodiscus discolor (Ocean spray)\n", 375 | "57. Keckiella cordifolia (Heart leaved keckiella)\n", 376 | "58. Keckiella cordifolia (Heart leaved keckiella)\n", 377 | "59. Keckiella cordifolia (Heart leaved keckiella)\n", 378 | "60. Keckiella cordifolia (Heart leaved keckiella)\n", 379 | "61. Keckiella cordifolia (Heart leaved keckiella)\n", 380 | "62. Keckiella cordifolia (Heart leaved keckiella)\n", 381 | "63. Keckiella cordifolia (Heart leaved keckiella)\n", 382 | "64. Keckiella cordifolia (Heart leaved keckiella)\n", 383 | "65. Keckiella cordifolia (Heart leaved keckiella)\n", 384 | "66. Keckiella cordifolia (Heart leaved keckiella)\n", 385 | "67. Keckiella cordifolia (Heart leaved keckiella)\n", 386 | "68. Keckiella cordifolia (Heart leaved keckiella)\n", 387 | "69. Keckiella cordifolia (Heart leaved keckiella)\n", 388 | "70. Keckiella cordifolia (Heart leaved keckiella)\n", 389 | "71. Keckiella cordifolia (Heart leaved keckiella)\n", 390 | "72. Keckiella cordifolia (Heart leaved keckiella)\n", 391 | "73. Keckiella cordifolia (Heart leaved keckiella)\n", 392 | "74. Keckiella cordifolia (Heart leaved keckiella)\n", 393 | "75. Keckiella cordifolia (Heart leaved keckiella)\n", 394 | "76. Keckiella cordifolia (Heart leaved keckiella)\n", 395 | "77. Keckiella cordifolia (Heart leaved keckiella)\n", 396 | "78. Keckiella cordifolia (Heart leaved keckiella)\n", 397 | "79. Keckiella cordifolia (Heart leaved keckiella)\n", 398 | "80. Keckiella cordifolia (Heart leaved keckiella)\n", 399 | "81. Keckiella cordifolia (Heart leaved keckiella)\n", 400 | "82. Keckiella cordifolia (Heart leaved keckiella)\n", 401 | "83. Keckiella cordifolia (Heart leaved keckiella)\n", 402 | "84. Keckiella cordifolia (Heart leaved keckiella)\n", 403 | "85. Keckiella cordifolia (Heart leaved keckiella)\n", 404 | "86. Keckiella cordifolia (Heart leaved keckiella)\n", 405 | "87. Keckiella cordifolia (Heart leaved keckiella)\n", 406 | "88. Keckiella cordifolia (Heart leaved keckiella)\n", 407 | "89. Keckiella cordifolia (Heart leaved keckiella)\n", 408 | "90. Keckiella cordifolia (Heart leaved keckiella)\n", 409 | "91. Keckiella cordifolia (Heart leaved keckiella)\n", 410 | "92. Keckiella cordifolia (Heart leaved keckiella)\n", 411 | "93. Keckiella cordifolia (Heart leaved keckiella)\n", 412 | "94. Keckiella cordifolia (Heart leaved keckiella)\n", 413 | "95. Keckiella cordifolia (Heart leaved keckiella)\n", 414 | "96. Keckiella cordifolia (Heart leaved keckiella)\n", 415 | "97. Keckiella cordifolia (Heart leaved keckiella)\n", 416 | "98. Keckiella cordifolia (Heart leaved keckiella)\n", 417 | "99. Keckiella cordifolia (Heart leaved keckiella)\n", 418 | "100. Keckiella cordifolia (Heart leaved keckiella)\n", 419 | "101. Keckiella cordifolia (Heart leaved keckiella)\n", 420 | "102. Keckiella cordifolia (Heart leaved keckiella)\n", 421 | "103. Keckiella cordifolia (Heart leaved keckiella)\n", 422 | "104. Keckiella cordifolia (Heart leaved keckiella)\n", 423 | "105. Keckiella cordifolia (Heart leaved keckiella)\n", 424 | "106. Keckiella cordifolia (Heart leaved keckiella)\n", 425 | "107. Keckiella cordifolia (Heart leaved keckiella)\n", 426 | "108. Keckiella cordifolia (Heart leaved keckiella)\n", 427 | "109. Keckiella cordifolia (Heart leaved keckiella)\n", 428 | "110. Keckiella cordifolia (Heart leaved keckiella)\n", 429 | "111. Keckiella cordifolia (Heart leaved keckiella)\n", 430 | "112. Keckiella cordifolia (Heart leaved keckiella)\n", 431 | "113. Keckiella cordifolia (Heart leaved keckiella)\n", 432 | "114. Keckiella cordifolia (Heart leaved keckiella)\n", 433 | "115. Keckiella cordifolia (Heart leaved keckiella)\n", 434 | "116. Keckiella cordifolia (Heart leaved keckiella)\n", 435 | "117. Keckiella cordifolia (Heart leaved keckiella)\n", 436 | "118. Keckiella cordifolia (Heart leaved keckiella)\n", 437 | "119. Keckiella cordifolia (Heart leaved keckiella)\n", 438 | "120. Keckiella cordifolia (Heart leaved keckiella)\n", 439 | "121. Keckiella cordifolia (Heart leaved keckiella)\n", 440 | "122. Keckiella cordifolia (Heart leaved keckiella)\n", 441 | "123. Keckiella cordifolia (Heart leaved keckiella)\n", 442 | "124. Keckiella cordifolia (Heart leaved keckiella)\n", 443 | "125. Keckiella cordifolia (Heart leaved keckiella)\n", 444 | "126. Keckiella cordifolia (Heart leaved keckiella)\n", 445 | "127. Keckiella\n" 446 | ] 447 | } 448 | ], 449 | "source": [ 450 | "user_content = [{\"text\": \"What plants are listed on these pages?\", \"type\": \"text\"}]\n", 451 | "for i in range(doc.page_count):\n", 452 | " user_content.append({\"image_url\": {\"url\": open_image_as_base64(f\"page_{i}.png\")}, \"type\": \"image_url\"})\n", 453 | "\n", 454 | "response = openai_client.chat.completions.create(\n", 455 | " model=model_name, messages=[{\"role\": \"user\", \"content\": user_content}], temperature=0.5\n", 456 | ")\n", 457 | "\n", 458 | "print(response.choices[0].message.content)" 459 | ] 460 | } 461 | ], 462 | "metadata": { 463 | "kernelspec": { 464 | "display_name": "Python 3", 465 | "language": "python", 466 | "name": "python3" 467 | }, 468 | "language_info": { 469 | "codemirror_mode": { 470 | "name": "ipython", 471 | "version": 3 472 | }, 473 | "file_extension": ".py", 474 | "mimetype": "text/x-python", 475 | "name": "python", 476 | "nbconvert_exporter": "python", 477 | "pygments_lexer": "ipython3", 478 | "version": "3.11.10" 479 | } 480 | }, 481 | "nbformat": 4, 482 | "nbformat_minor": 2 483 | } 484 | -------------------------------------------------------------------------------- /notebooks/chat_vision.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Chat with vision models\n", 8 | "\n", 9 | "**If you're looking for the web application, check the src/ folder.**\n", 10 | "\n", 11 | "This notebook is just provided for manual experimentation with the vision model." 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "## Authenticate to OpenAI\n", 19 | "\n", 20 | "The following code connects to OpenAI, either using an Azure OpenAI account, GitHub models, or local Ollama model. See the README for instruction on configuring the `.env` file." 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 22, 26 | "metadata": {}, 27 | "outputs": [ 28 | { 29 | "name": "stdout", 30 | "output_type": "stream", 31 | "text": [ 32 | "Using GitHub Models with GITHUB_TOKEN as key\n", 33 | "Using model gpt-4o\n" 34 | ] 35 | } 36 | ], 37 | "source": [ 38 | "import os\n", 39 | "\n", 40 | "import azure.identity\n", 41 | "import openai\n", 42 | "from dotenv import load_dotenv\n", 43 | "\n", 44 | "load_dotenv(\".env\", override=True)\n", 45 | "\n", 46 | "openai_host = os.getenv(\"OPENAI_HOST\", \"github\")\n", 47 | "model_name = os.getenv(\"OPENAI_MODEL\", \"gpt-4o\")\n", 48 | "\n", 49 | "if openai_host == \"local\":\n", 50 | " print(\"Using local OpenAI-compatible API with no key\")\n", 51 | " openai_client = openai.OpenAI(api_key=\"no-key-required\", base_url=os.environ[\"LOCAL_OPENAI_ENDPOINT\"])\n", 52 | "elif openai_host == \"github\":\n", 53 | " print(\"Using GitHub Models with GITHUB_TOKEN as key\")\n", 54 | " openai_client = openai.OpenAI(\n", 55 | " api_key=os.environ[\"GITHUB_TOKEN\"],\n", 56 | " base_url=\"https://models.inference.ai.azure.com\",\n", 57 | " )\n", 58 | "elif openai_host == \"azure\" and os.getenv(\"AZURE_OPENAI_KEY_FOR_CHATVISION\"):\n", 59 | " # Authenticate using an Azure OpenAI API key\n", 60 | " # This is generally discouraged, but is provided as a convenience\n", 61 | " print(\"Using Azure OpenAI with key\")\n", 62 | " openai_client = openai.AzureOpenAI(\n", 63 | " api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\") or \"2024-02-15-preview\",\n", 64 | " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", 65 | " api_key=os.environ[\"AZURE_OPENAI_KEY_FOR_CHATVISION\"],\n", 66 | " )\n", 67 | "elif openai_host == \"azure\" and os.getenv(\"AZURE_OPENAI_ENDPOINT\"):\n", 68 | " tenant_id = os.environ[\"AZURE_TENANT_ID\"]\n", 69 | " print(\"Using Azure OpenAI with Azure Developer CLI credential for tenant id\", tenant_id)\n", 70 | " default_credential = azure.identity.AzureDeveloperCliCredential(tenant_id=tenant_id)\n", 71 | " token_provider = azure.identity.get_bearer_token_provider(\n", 72 | " default_credential, \"https://cognitiveservices.azure.com/.default\"\n", 73 | " )\n", 74 | " openai_client = openai.AzureOpenAI(\n", 75 | " api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\") or \"2024-02-15-preview\",\n", 76 | " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", 77 | " azure_ad_token_provider=token_provider,\n", 78 | " )\n", 79 | "print(f\"Using model {model_name}\")" 80 | ] 81 | }, 82 | { 83 | "cell_type": "markdown", 84 | "metadata": {}, 85 | "source": [ 86 | "## Send an image by URL" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "metadata": {}, 93 | "outputs": [ 94 | { 95 | "name": "stdout", 96 | "output_type": "stream", 97 | "text": [ 98 | "No, this is not a unicorn. This is an illustration of an aurochs, an extinct species of large wild cattle that once roamed Europe, Asia, and North Africa. Unicorns are mythical creatures typically depicted with a single horn on their forehead, while this animal clearly has two horns.\n" 99 | ] 100 | } 101 | ], 102 | "source": [ 103 | "messages = [\n", 104 | " {\n", 105 | " \"role\": \"user\",\n", 106 | " \"content\": [\n", 107 | " {\"text\": \"Is this a unicorn?\", \"type\": \"text\"},\n", 108 | " {\n", 109 | " \"image_url\": {\"url\": \"https://upload.wikimedia.org/wikipedia/commons/6/6e/Ur-painting.jpg\"},\n", 110 | " \"type\": \"image_url\",\n", 111 | " },\n", 112 | " ],\n", 113 | " }\n", 114 | "]\n", 115 | "response = openai_client.chat.completions.create(model=model_name, messages=messages, temperature=0.5)\n", 116 | "\n", 117 | "print(response.choices[0].message.content)" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": {}, 123 | "source": [ 124 | "## Send an image by Data URI\n", 125 | "\n" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": 14, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "import base64\n", 135 | "\n", 136 | "\n", 137 | "def open_image_as_base64(filename):\n", 138 | " with open(filename, \"rb\") as image_file:\n", 139 | " image_data = image_file.read()\n", 140 | " image_base64 = base64.b64encode(image_data).decode(\"utf-8\")\n", 141 | " return f\"data:image/png;base64,{image_base64}\"" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": null, 147 | "metadata": {}, 148 | "outputs": [ 149 | { 150 | "name": "stdout", 151 | "output_type": "stream", 152 | "text": [ 153 | "These are crocodiles. You can tell by their slender, V-shaped snouts, which are a characteristic feature of crocodiles, as opposed to the broader, U-shaped snouts of alligators.\n" 154 | ] 155 | } 156 | ], 157 | "source": [ 158 | "response = openai_client.chat.completions.create(\n", 159 | " model=model_name,\n", 160 | " messages=[\n", 161 | " {\n", 162 | " \"role\": \"user\",\n", 163 | " \"content\": [\n", 164 | " {\"text\": \"are these alligators or crocodiles?\", \"type\": \"text\"},\n", 165 | " {\"image_url\": {\"url\": open_image_as_base64(\"mystery_reptile.png\")}, \"type\": \"image_url\"},\n", 166 | " ],\n", 167 | " }\n", 168 | " ],\n", 169 | ")\n", 170 | "\n", 171 | "print(response.choices[0].message.content)" 172 | ] 173 | }, 174 | { 175 | "cell_type": "markdown", 176 | "metadata": {}, 177 | "source": [ 178 | "## Use cases for image analysis" 179 | ] 180 | }, 181 | { 182 | "cell_type": "markdown", 183 | "metadata": {}, 184 | "source": [ 185 | "### Accessibility" 186 | ] 187 | }, 188 | { 189 | "cell_type": "markdown", 190 | "metadata": {}, 191 | "source": [ 192 | "#### Assistance for vision-impaired" 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "execution_count": null, 198 | "metadata": {}, 199 | "outputs": [ 200 | { 201 | "name": "stdout", 202 | "output_type": "stream", 203 | "text": [ 204 | "This menu doesn't seem to have any explicitly vegan dishes, as many options contain meat, seafood, cheese, or other animal-derived ingredients. However, some dishes could potentially be modified to make them vegan-friendly. Here are some dishes that seem like they may be adapted:\n", 205 | "\n", 206 | "### Antipasti:\n", 207 | "- **Spinaci Soffritti (8)**: Fresh spinach sautéed with lemon and garlic. Ensure no butter or animal-based oils are used for preparation.\n", 208 | "- **Bruschetta Trio (18)**: The avocado, smoked salmon, and crème fraiche bruschetta sounds flexible. Ask if it can be served without the salmon and crème fraiche, and confirm the base bread is vegan.\n", 209 | "\n", 210 | "### Zuppe & Insalate:\n", 211 | "- **Panzanella con Fagioli (18)**: A vine tomato and bread salad mixed with onions, beans, cucumbers, and avocado. Ask the kitchen to confirm that Carmine's House Vinaigrette is vegan.\n", 212 | "- **Insalata Di Mista (13)**: Seasonal greens tossed in the house vinaigrette. Double-check if the dressing is vegan and skip the option of adding chicken or steak. \n", 213 | "- **Zuppa Di Fagioli (11)**: Tuscan beans and pasta soup. Ensure it is made with vegetable stock (not meat-based). \n", 214 | "- **Portofino Salad (18)**: Tomatoes, nicoise olives, and vegetables. You’ll need to request no egg, anchovies, or tuna.\n", 215 | "\n", 216 | "### Pasta:\n", 217 | "While none of the listed pasta dishes are vegan, you could inquire with the chef if there is a plain pasta primavera or vegetable pasta option available, made with olive oil and without cheese, cream, or other animal ingredients.\n", 218 | "\n", 219 | "### Final Suggestions:\n", 220 | "Talk with your server to determine if they have vegan customization options or any off-menu items catering to dietary preferences.\n" 221 | ] 222 | } 223 | ], 224 | "source": [ 225 | "response = openai_client.chat.completions.create(\n", 226 | " model=model_name,\n", 227 | " messages=[\n", 228 | " {\n", 229 | " \"role\": \"user\",\n", 230 | " \"content\": [\n", 231 | " {\"text\": \"is there anything good for vegans on this menu?\", \"type\": \"text\"},\n", 232 | " {\"image_url\": {\"url\": open_image_as_base64(\"menu.png\")}, \"type\": \"image_url\"},\n", 233 | " ],\n", 234 | " }\n", 235 | " ],\n", 236 | ")\n", 237 | "\n", 238 | "print(response.choices[0].message.content)" 239 | ] 240 | }, 241 | { 242 | "cell_type": "markdown", 243 | "metadata": {}, 244 | "source": [ 245 | "#### Automated image captioning" 246 | ] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "execution_count": null, 251 | "metadata": {}, 252 | "outputs": [ 253 | { 254 | "name": "stdout", 255 | "output_type": "stream", 256 | "text": [ 257 | "Diagram showing the architecture of an Azure-based deployment for a containerized chat application. The central component is a \"Container App\" connected to various Azure resources. It links to a \"Container Apps Environment\" at the top, Azure Cognitive Services (\"Azure AI Services\") on the top right, and \"Managed Identity\" on the bottom left. Other connected resources include \"Log Analytics Workspace\" for monitoring, a \"Container Registry\" for image storage, and an Azure \"Key Vault\" for secret management. The flow of dependencies is visually represented with arrows between the components, illustrating their interactions.\n" 258 | ] 259 | } 260 | ], 261 | "source": [ 262 | "response = openai_client.chat.completions.create(\n", 263 | " model=model_name,\n", 264 | " messages=[\n", 265 | " {\n", 266 | " \"role\": \"user\",\n", 267 | " \"content\": [\n", 268 | " {\"text\": \"Suggest an alt text for this image\", \"type\": \"text\"},\n", 269 | " {\"image_url\": {\"url\": open_image_as_base64(\"azure_arch.png\")}, \"type\": \"image_url\"},\n", 270 | " ],\n", 271 | " }\n", 272 | " ],\n", 273 | ")\n", 274 | "\n", 275 | "print(response.choices[0].message.content)" 276 | ] 277 | }, 278 | { 279 | "cell_type": "markdown", 280 | "metadata": {}, 281 | "source": [ 282 | "### Business process automation" 283 | ] 284 | }, 285 | { 286 | "cell_type": "markdown", 287 | "metadata": {}, 288 | "source": [ 289 | "#### Insurance claim processing" 290 | ] 291 | }, 292 | { 293 | "cell_type": "code", 294 | "execution_count": null, 295 | "metadata": {}, 296 | "outputs": [ 297 | { 298 | "name": "stdout", 299 | "output_type": "stream", 300 | "text": [ 301 | "The damage shown in this image is not consistent with hail damage. Hail damage is typically characterized by multiple small, round dents across the surface of the vehicle, usually on the hood, roof, and trunk, as well as occasionally cracked or broken windows.\n", 302 | "\n", 303 | "The significant and centralized deformation of the hood, along with substantial damage to the front grille and surrounding areas, suggests impact with a large object (e.g., another vehicle, a tree, or a pole), not hailstones. Based on this observation, the claim is not valid if it is attributed solely to hail.\n" 304 | ] 305 | } 306 | ], 307 | "source": [ 308 | "response = openai_client.chat.completions.create(\n", 309 | " model=model_name,\n", 310 | " messages=[\n", 311 | " {\n", 312 | " \"role\": \"system\",\n", 313 | " \"content\": (\n", 314 | " \"You are an AI assistant that helps auto insurance companies process claims.\"\n", 315 | " \"You accept images of damaged cars that are submitted with claims, and you are able to make judgments \"\n", 316 | " \"about the causes of automobile damage, and the validity of claims regarding that damage.\"\n", 317 | " ),\n", 318 | " },\n", 319 | " {\n", 320 | " \"role\": \"user\",\n", 321 | " \"content\": [\n", 322 | " {\"text\": \"Claim states that this damage is due to hail. Is it valid?\", \"type\": \"text\"},\n", 323 | " {\"image_url\": {\"url\": open_image_as_base64(\"dented_car.jpg\")}, \"type\": \"image_url\"},\n", 324 | " ],\n", 325 | " },\n", 326 | " ],\n", 327 | ")\n", 328 | "\n", 329 | "print(response.choices[0].message.content)" 330 | ] 331 | }, 332 | { 333 | "cell_type": "markdown", 334 | "metadata": {}, 335 | "source": [ 336 | "#### Graph analysis" 337 | ] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "execution_count": 19, 342 | "metadata": {}, 343 | "outputs": [ 344 | { 345 | "name": "stdout", 346 | "output_type": "stream", 347 | "text": [ 348 | "The zone where we are losing the most trees is the **Tropical zone**, represented by the dark green bars in the graph. It consistently shows the largest amount of tree cover loss compared to the Boreal, Temperate, and Subtropical zones.\n" 349 | ] 350 | } 351 | ], 352 | "source": [ 353 | "messages = [\n", 354 | " {\n", 355 | " \"role\": \"user\",\n", 356 | " \"content\": [\n", 357 | " {\"text\": \"What zone are we losing the most trees in?\", \"type\": \"text\"},\n", 358 | " {\n", 359 | " \"image_url\": {\n", 360 | " \"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/1/1f/20210331_Global_tree_cover_loss_-_World_Resources_Institute.svg/1280px-20210331_Global_tree_cover_loss_-_World_Resources_Institute.svg.png\"\n", 361 | " },\n", 362 | " \"type\": \"image_url\",\n", 363 | " },\n", 364 | " ],\n", 365 | " }\n", 366 | "]\n", 367 | "response = openai_client.chat.completions.create(model=os.environ[\"OPENAI_MODEL\"], messages=messages, temperature=0.5)\n", 368 | "\n", 369 | "print(response.choices[0].message.content)" 370 | ] 371 | }, 372 | { 373 | "cell_type": "markdown", 374 | "metadata": {}, 375 | "source": [ 376 | "#### Table analysis" 377 | ] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "execution_count": null, 382 | "metadata": {}, 383 | "outputs": [ 384 | { 385 | "name": "stdout", 386 | "output_type": "stream", 387 | "text": [ 388 | "The cheapest plant listed on the availability sheet is *Agrostis pallens* (Thringrass) under the \"Grass\" category, priced at **$0.58** per stub.\n" 389 | ] 390 | } 391 | ], 392 | "source": [ 393 | "response = openai_client.chat.completions.create(\n", 394 | " model=model_name,\n", 395 | " messages=[\n", 396 | " {\n", 397 | " \"role\": \"user\",\n", 398 | " \"content\": [\n", 399 | " {\"text\": \"What's the cheapest plant?\", \"type\": \"text\"},\n", 400 | " {\"image_url\": {\"url\": open_image_as_base64(\"page_0.png\")}, \"type\": \"image_url\"},\n", 401 | " ],\n", 402 | " }\n", 403 | " ],\n", 404 | ")\n", 405 | "\n", 406 | "print(response.choices[0].message.content)" 407 | ] 408 | }, 409 | { 410 | "cell_type": "markdown", 411 | "metadata": {}, 412 | "source": [ 413 | "#### Appliance support" 414 | ] 415 | }, 416 | { 417 | "cell_type": "code", 418 | "execution_count": null, 419 | "metadata": {}, 420 | "outputs": [ 421 | { 422 | "name": "stdout", 423 | "output_type": "stream", 424 | "text": [ 425 | "To wash the dishes quickly with this Bosch dishwasher, follow these steps:\n", 426 | "\n", 427 | "1. **Turn on the dishwasher**: Press the **\"On/Off\"** button.\n", 428 | "2. **Select the quick program**: Press the **\"Quick 45°\"** button. This is a designated fast wash program, typically for lightly soiled dishes.\n", 429 | "3. **Optional - Use VarioSpeed**: If you want to make the cycle even faster, press the **\"VarioSpeed\"** button, which reduces the cycle time further by increasing energy and water usage.\n", 430 | "4. **Start the dishwasher**: Press the **\"Start\"** button to begin the cycle.\n", 431 | "\n", 432 | "That's it! Your dishwasher will now wash the dishes quickly. Keep in mind that this setting is best suited for lightly soiled dishes and not for heavy loads or tough stains.\n" 433 | ] 434 | } 435 | ], 436 | "source": [ 437 | "response = openai_client.chat.completions.create(\n", 438 | " model=model_name,\n", 439 | " messages=[\n", 440 | " {\n", 441 | " \"role\": \"user\",\n", 442 | " \"content\": [\n", 443 | " {\"text\": \"How do I set this to wash the dishes quickly?\", \"type\": \"text\"},\n", 444 | " {\"image_url\": {\"url\": open_image_as_base64(\"dishwasher.png\")}, \"type\": \"image_url\"},\n", 445 | " ],\n", 446 | " }\n", 447 | " ],\n", 448 | ")\n", 449 | "\n", 450 | "print(response.choices[0].message.content)" 451 | ] 452 | } 453 | ], 454 | "metadata": { 455 | "kernelspec": { 456 | "display_name": "Python 3", 457 | "language": "python", 458 | "name": "python3" 459 | }, 460 | "language_info": { 461 | "codemirror_mode": { 462 | "name": "ipython", 463 | "version": 3 464 | }, 465 | "file_extension": ".py", 466 | "mimetype": "text/x-python", 467 | "name": "python", 468 | "nbconvert_exporter": "python", 469 | "pygments_lexer": "ipython3", 470 | "version": "3.11.10" 471 | } 472 | }, 473 | "nbformat": 4, 474 | "nbformat_minor": 2 475 | } 476 | -------------------------------------------------------------------------------- /notebooks/dented_car.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/dented_car.jpg -------------------------------------------------------------------------------- /notebooks/dishwasher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/dishwasher.png -------------------------------------------------------------------------------- /notebooks/menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/menu.png -------------------------------------------------------------------------------- /notebooks/mystery_reptile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/mystery_reptile.png -------------------------------------------------------------------------------- /notebooks/page_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_0.png -------------------------------------------------------------------------------- /notebooks/page_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_1.png -------------------------------------------------------------------------------- /notebooks/page_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_10.png -------------------------------------------------------------------------------- /notebooks/page_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_11.png -------------------------------------------------------------------------------- /notebooks/page_12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_12.png -------------------------------------------------------------------------------- /notebooks/page_13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_13.png -------------------------------------------------------------------------------- /notebooks/page_14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_14.png -------------------------------------------------------------------------------- /notebooks/page_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_2.png -------------------------------------------------------------------------------- /notebooks/page_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_3.png -------------------------------------------------------------------------------- /notebooks/page_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_4.png -------------------------------------------------------------------------------- /notebooks/page_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_5.png -------------------------------------------------------------------------------- /notebooks/page_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_6.png -------------------------------------------------------------------------------- /notebooks/page_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_7.png -------------------------------------------------------------------------------- /notebooks/page_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_8.png -------------------------------------------------------------------------------- /notebooks/page_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/page_9.png -------------------------------------------------------------------------------- /notebooks/plants.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/plants.pdf -------------------------------------------------------------------------------- /notebooks/ur.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/notebooks/ur.jpg -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.ruff] 2 | target-version = "py39" 3 | line-length = 120 4 | src = ["src"] 5 | 6 | [tool.ruff.lint] 7 | select = ["E", "F", "I", "UP"] 8 | 9 | [tool.ruff.lint.isort] 10 | known-first-party = ["quartapp"] 11 | 12 | [tool.pytest.ini_options] 13 | addopts = "-ra --cov" 14 | pythonpath = ["src"] 15 | 16 | [tool.coverage.report] 17 | show_missing = true 18 | -------------------------------------------------------------------------------- /readme_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/readme_diagram.png -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | -r src/requirements.txt 2 | ruff 3 | pre-commit 4 | pytest 5 | pytest-asyncio 6 | pytest-snapshot 7 | pytest-cov 8 | pip-tools 9 | ipykernel -------------------------------------------------------------------------------- /scripts/write_env.ps1: -------------------------------------------------------------------------------- 1 | # Define the .env file path 2 | $envFilePath = ".env" 3 | 4 | # Clear the contents of the .env file 5 | Set-Content -Path $envFilePath -Value "" 6 | 7 | # Append new values to the .env file 8 | $azureOpenAiEndpoint = azd env get-value AZURE_OPENAI_ENDPOINT 9 | $azureOpenAiDeployment = azd env get-value AZURE_OPENAI_DEPLOYMENT 10 | $azureOpenAiApiVersion = azd env get-value AZURE_OPENAI_API_VERSION 11 | $azureTenantId = azd env get-value AZURE_TENANT_ID 12 | 13 | Add-Content -Path $envFilePath -Value "OPENAI_HOST=azure" 14 | Add-Content -Path $envFilePath -Value "OPENAI_MODEL=$azureOpenAiDeployment" 15 | Add-Content -Path $envFilePath -Value "" 16 | Add-Content -Path $envFilePath -Value "AZURE_OPENAI_ENDPOINT=$azureOpenAiEndpoint" 17 | Add-Content -Path $envFilePath -Value "AZURE_OPENAI_API_VERSION=$azureOpenAiApiVersion" 18 | Add-Content -Path $envFilePath -Value "AZURE_TENANT_ID=$azureTenantId" 19 | Add-Content -Path $envFilePath -Value "" 20 | -------------------------------------------------------------------------------- /scripts/write_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Define the .env file path 4 | ENV_FILE_PATH=".env" 5 | 6 | # Clear the contents of the .env file 7 | > $ENV_FILE_PATH 8 | 9 | echo "OPENAI_HOST=azure" >> $ENV_FILE_PATH 10 | echo "OPENAI_MODEL=$(azd env get-value AZURE_OPENAI_DEPLOYMENT)" >> $ENV_FILE_PATH 11 | echo "" >> $ENV_FILE_PATH 12 | echo "AZURE_OPENAI_ENDPOINT=$(azd env get-value AZURE_OPENAI_ENDPOINT)" >> $ENV_FILE_PATH 13 | echo "AZURE_OPENAI_API_VERSION=$(azd env get-value AZURE_OPENAI_API_VERSION)" >> $ENV_FILE_PATH 14 | echo "AZURE_TENANT_ID=$(azd env get-value AZURE_TENANT_ID)" >> $ENV_FILE_PATH 15 | echo "" >> $ENV_FILE_PATH 16 | -------------------------------------------------------------------------------- /src/.dockerignore: -------------------------------------------------------------------------------- 1 | .git* 2 | .venv/ 3 | **/*.pyc 4 | -------------------------------------------------------------------------------- /src/Dockerfile: -------------------------------------------------------------------------------- 1 | # ------------------- Stage 0: Base Stage ------------------------------ 2 | FROM python:3.11-alpine AS base 3 | 4 | WORKDIR /code 5 | 6 | # Install tini, a tiny init for containers 7 | RUN apk add --update --no-cache tini 8 | 9 | # Install required packages for cryptography package 10 | # https://cryptography.io/en/latest/installation/#building-cryptography-on-linux 11 | RUN apk add gcc musl-dev python3-dev libffi-dev openssl-dev cargo pkgconfig 12 | 13 | # ------------------- Stage 1: Build Stage ------------------------------ 14 | FROM base AS build 15 | 16 | COPY requirements.txt . 17 | 18 | RUN pip3 install -r requirements.txt 19 | 20 | COPY . . 21 | # ------------------- Stage 2: Final Stage ------------------------------ 22 | FROM base AS final 23 | 24 | RUN addgroup -S app && adduser -S app -G app 25 | 26 | COPY --from=build --chown=app:app /usr/local/lib/python3.11 /usr/local/lib/python3.11 27 | COPY --from=build --chown=app:app /usr/local/bin /usr/local/bin 28 | COPY --from=build --chown=app:app /code /code 29 | 30 | USER app 31 | 32 | EXPOSE 50505 33 | 34 | ENTRYPOINT ["tini", "gunicorn", "quartapp:create_app()"] 35 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/src/__init__.py -------------------------------------------------------------------------------- /src/gunicorn.conf.py: -------------------------------------------------------------------------------- 1 | import multiprocessing 2 | import os 3 | 4 | from dotenv import load_dotenv 5 | 6 | load_dotenv(override=True) 7 | 8 | max_requests = 1000 9 | max_requests_jitter = 50 10 | log_file = "-" 11 | bind = "0.0.0.0:50505" 12 | 13 | if not os.getenv("RUNNING_IN_PRODUCTION"): 14 | reload = True 15 | 16 | num_cpus = multiprocessing.cpu_count() 17 | workers = (num_cpus * 2) + 1 18 | worker_class = "uvicorn.workers.UvicornWorker" 19 | 20 | timeout = 120 21 | -------------------------------------------------------------------------------- /src/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "quartapp" 3 | version = "1.0.0" 4 | description = "Create a simple chat app using Quart and OpenAI" 5 | dependencies = [ 6 | "quart", 7 | "werkzeug", 8 | "gunicorn", 9 | "uvicorn[standard]", 10 | "openai", 11 | "azure-identity", 12 | "aiohttp", 13 | "python-dotenv", 14 | "pyyaml" 15 | ] 16 | 17 | [build-system] 18 | requires = ["flit_core<4"] 19 | build-backend = "flit_core.buildapi" 20 | -------------------------------------------------------------------------------- /src/quartapp/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from dotenv import load_dotenv 5 | from quart import Quart 6 | 7 | 8 | def create_app(): 9 | # We do this here in addition to gunicorn.conf.py, since we don't always use gunicorn 10 | load_dotenv(override=True) 11 | if os.getenv("RUNNING_IN_PRODUCTION"): 12 | logging.basicConfig(level=logging.WARNING) 13 | else: 14 | logging.basicConfig(level=logging.INFO) 15 | 16 | app = Quart(__name__) 17 | 18 | from . import chat # noqa 19 | 20 | app.register_blueprint(chat.bp) 21 | 22 | return app 23 | -------------------------------------------------------------------------------- /src/quartapp/chat.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | import azure.identity.aio 5 | import openai 6 | from quart import ( 7 | Blueprint, 8 | Response, 9 | current_app, 10 | render_template, 11 | request, 12 | stream_with_context, 13 | ) 14 | 15 | bp = Blueprint("chat", __name__, template_folder="templates", static_folder="static") 16 | 17 | 18 | @bp.before_app_serving 19 | async def configure_openai(): 20 | openai_host = os.getenv("OPENAI_HOST", "github") 21 | bp.model_name = os.getenv("OPENAI_MODEL", "gpt-4o") 22 | if openai_host == "local": 23 | # Use a local endpoint like llamafile server 24 | current_app.logger.info("Using model %s from local OpenAI-compatible API with no key", bp.model_name) 25 | bp.openai_client = openai.AsyncOpenAI(api_key="no-key-required", base_url=os.getenv("LOCAL_OPENAI_ENDPOINT")) 26 | elif openai_host == "github": 27 | current_app.logger.info("Using model %s from GitHub models with GITHUB_TOKEN as key", bp.model_name) 28 | bp.openai_client = openai.AsyncOpenAI( 29 | api_key=os.environ["GITHUB_TOKEN"], 30 | base_url="https://models.inference.ai.azure.com", 31 | ) 32 | else: 33 | client_args = {} 34 | # Use an Azure OpenAI endpoint instead, 35 | # either with a key or with keyless authentication 36 | if os.getenv("AZURE_OPENAI_KEY_FOR_CHATVISION"): 37 | # Authenticate using an Azure OpenAI API key 38 | # This is generally discouraged, but is provided for developers 39 | # that want to develop locally inside the Docker container. 40 | current_app.logger.info("Using model %s from Azure OpenAI with key", bp.model_name) 41 | client_args["api_key"] = os.getenv("AZURE_OPENAI_KEY_FOR_CHATVISION") 42 | else: 43 | if os.getenv("RUNNING_IN_PRODUCTION"): 44 | client_id = os.getenv("AZURE_CLIENT_ID") 45 | current_app.logger.info( 46 | "Using model %s from Azure OpenAI with managed identity credential for client ID %s", 47 | bp.model_name, 48 | client_id, 49 | ) 50 | azure_credential = azure.identity.aio.ManagedIdentityCredential(client_id=client_id) 51 | else: 52 | tenant_id = os.environ["AZURE_TENANT_ID"] 53 | current_app.logger.info( 54 | "Using model %s from Azure OpenAI with Azure Developer CLI credential for tenant ID: %s", 55 | bp.model_name, 56 | tenant_id, 57 | ) 58 | azure_credential = azure.identity.aio.AzureDeveloperCliCredential(tenant_id=tenant_id) 59 | client_args["azure_ad_token_provider"] = azure.identity.aio.get_bearer_token_provider( 60 | azure_credential, "https://cognitiveservices.azure.com/.default" 61 | ) 62 | bp.openai_client = openai.AsyncAzureOpenAI( 63 | azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], 64 | api_version=os.getenv("AZURE_OPENAI_API_VERSION") or "2024-05-01-preview", 65 | **client_args, 66 | ) 67 | 68 | 69 | @bp.after_app_serving 70 | async def shutdown_openai(): 71 | await bp.openai_client.close() 72 | 73 | 74 | @bp.get("/") 75 | async def index(): 76 | return await render_template("index.html") 77 | 78 | 79 | @bp.post("/chat/stream") 80 | async def chat_handler(): 81 | request_json = await request.get_json() 82 | request_messages = request_json["messages"] 83 | # get the base64 encoded image from the request 84 | image = request_json["context"]["file"] 85 | 86 | @stream_with_context 87 | async def response_stream(): 88 | # This sends all messages, so API request may exceed token limits 89 | all_messages = [ 90 | {"role": "system", "content": "You are a helpful assistant."}, 91 | ] + request_messages[0:-1] 92 | all_messages = request_messages[0:-1] 93 | if image: 94 | user_content = [] 95 | user_content.append({"text": request_messages[-1]["content"], "type": "text"}) 96 | user_content.append({"image_url": {"url": image, "detail": "auto"}, "type": "image_url"}) 97 | all_messages.append({"role": "user", "content": user_content}) 98 | else: 99 | all_messages.append(request_messages[-1]) 100 | 101 | chat_coroutine = bp.openai_client.chat.completions.create( 102 | # Azure Open AI takes the deployment name as the model name 103 | model=bp.model_name, 104 | messages=all_messages, 105 | stream=True, 106 | temperature=request_json.get("temperature", 0.5), 107 | ) 108 | try: 109 | async for event in await chat_coroutine: 110 | event_dict = event.model_dump() 111 | if event_dict["choices"]: 112 | yield json.dumps(event_dict["choices"][0], ensure_ascii=False) + "\n" 113 | except Exception as e: 114 | current_app.logger.error(e) 115 | yield json.dumps({"error": str(e)}, ensure_ascii=False) + "\n" 116 | 117 | return Response(response_stream()) 118 | -------------------------------------------------------------------------------- /src/quartapp/static/speech-input.js: -------------------------------------------------------------------------------- 1 | class SpeechInputButton extends HTMLElement { 2 | constructor() { 3 | super(); 4 | this.isRecording = false; 5 | const SpeechRecognition = 6 | window.SpeechRecognition || window.webkitSpeechRecognition; 7 | if (!SpeechRecognition) { 8 | this.dispatchEvent( 9 | new CustomEvent("speecherror", { 10 | detail: { error: "SpeechRecognition not supported" }, 11 | }) 12 | ); 13 | return; 14 | } 15 | this.speechRecognition = new SpeechRecognition(); 16 | this.speechRecognition.lang = navigator.language || navigator.userLanguage; 17 | this.speechRecognition.interimResults = false; 18 | this.speechRecognition.continuous = true; 19 | this.speechRecognition.maxAlternatives = 1; 20 | } 21 | 22 | connectedCallback() { 23 | this.innerHTML = ` 24 | `; 27 | this.recordButton = this.querySelector("button"); 28 | this.recordButton.addEventListener("click", () => this.toggleRecording()); 29 | document.addEventListener('keydown', this.handleKeydown.bind(this)); 30 | } 31 | 32 | disconnectedCallback() { 33 | document.removeEventListener('keydown', this.handleKeydown.bind(this)); 34 | } 35 | 36 | handleKeydown(event) { 37 | if (event.key === 'Escape') { 38 | this.abortRecording(); 39 | } else if (event.key === ' ' && event.shiftKey) { // Shift + Space 40 | event.preventDefault(); // Prevent default action 41 | this.toggleRecording(); 42 | } 43 | } 44 | 45 | renderButtonOn() { 46 | this.recordButton.classList.add("speech-input-active"); 47 | this.recordButton.innerHTML = ''; 48 | } 49 | 50 | renderButtonOff() { 51 | this.recordButton.classList.remove("speech-input-active"); 52 | this.recordButton.innerHTML = ''; 53 | } 54 | 55 | 56 | toggleRecording() { 57 | if (this.isRecording) { 58 | this.stopRecording(); 59 | } else { 60 | this.startRecording(); 61 | } 62 | } 63 | 64 | startRecording() { 65 | if (this.speechRecognition == null) { 66 | this.dispatchEvent( 67 | new CustomEvent("speech-input-error", { 68 | detail: { error: "SpeechRecognition not supported" }, 69 | }) 70 | ); 71 | } 72 | 73 | this.speechRecognition.onresult = (event) => { 74 | let input = ""; 75 | for (const result of event.results) { 76 | input += result[0].transcript; 77 | } 78 | this.dispatchEvent( 79 | new CustomEvent("speech-input-result", { 80 | detail: { transcript: input }, 81 | }) 82 | ); 83 | }; 84 | 85 | this.speechRecognition.onend = () => { 86 | // NOTE: In some browsers (e.g. Chrome), the recording will stop automatically after a few seconds of silence. 87 | this.isRecording = false; 88 | this.renderButtonOff(); 89 | this.dispatchEvent(new Event("speech-input-end")); 90 | }; 91 | 92 | this.speechRecognition.onerror = (event) => { 93 | if (this.speechRecognition) { 94 | this.speechRecognition.stop(); 95 | if (event.error == "no-speech") { 96 | this.dispatchEvent( 97 | new CustomEvent("speech-input-error", { 98 | detail: { 99 | error: 100 | "No speech was detected. Please check your system audio settings and try again.", 101 | }, 102 | }) 103 | ); 104 | } else if (event.error == "language-not-supported") { 105 | this.dispatchEvent( 106 | new CustomEvent("speech-input-error", { 107 | detail: { 108 | error: 109 | "The selected language is not supported. Please try a different language.", 110 | }, 111 | }) 112 | ); 113 | } else if (event.error != "aborted") { 114 | this.dispatchEvent( 115 | new CustomEvent("speech-input-error", { 116 | detail: { 117 | error: "An error occurred while recording. Please try again: " + event.error, 118 | }, 119 | }) 120 | ); 121 | } 122 | } 123 | }; 124 | 125 | this.speechRecognition.start(); 126 | this.isRecording = true; 127 | this.renderButtonOn(); 128 | } 129 | 130 | stopRecording() { 131 | if (this.speechRecognition) { 132 | this.speechRecognition.stop(); 133 | } 134 | } 135 | 136 | abortRecording() { 137 | if (this.speechRecognition) { 138 | this.speechRecognition.abort(); 139 | } 140 | } 141 | 142 | } 143 | 144 | customElements.define("speech-input-button", SpeechInputButton); 145 | -------------------------------------------------------------------------------- /src/quartapp/static/speech-output.js: -------------------------------------------------------------------------------- 1 | class SpeechOutputButton extends HTMLElement { 2 | static observedAttributes = ["text"]; 3 | 4 | constructor() { 5 | super(); 6 | this.isPlaying = false; 7 | const SpeechSynthesis = 8 | window.speechSynthesis || window.webkitSpeechSynthesis; 9 | if (!SpeechSynthesis) { 10 | this.dispatchEvent( 11 | new CustomEvent("speech-output-error", { 12 | detail: { error: "SpeechSynthesis not supported" }, 13 | }) 14 | ); 15 | return; 16 | } 17 | this.synth = SpeechSynthesis; 18 | this.lngCode = navigator.language || navigator.userLanguage; 19 | } 20 | 21 | connectedCallback() { 22 | this.innerHTML = ` 23 | `; 26 | this.speechButton = this.querySelector("button"); 27 | this.speechButton.addEventListener("click", () => 28 | this.toggleSpeechOutput() 29 | ); 30 | document.addEventListener('keydown', this.handleKeydown.bind(this)); 31 | } 32 | 33 | disconnectedCallback() { 34 | document.removeEventListener('keydown', this.handleKeydown.bind(this)); 35 | } 36 | 37 | handleKeydown(event) { 38 | if (event.key === 'Escape') { 39 | this.stopSpeech(); 40 | } 41 | } 42 | 43 | renderButtonOn() { 44 | this.speechButton.classList.add("speech-output-active"); 45 | this.speechButton.innerHTML = ''; 46 | } 47 | 48 | renderButtonOff() { 49 | this.speechButton.classList.remove("speech-output-active"); 50 | this.speechButton.innerHTML = ''; 51 | } 52 | 53 | toggleSpeechOutput() { 54 | if (!this.isConnected) { 55 | return; 56 | } 57 | const text = this.getAttribute("text"); 58 | if (this.synth != null) { 59 | if (this.isPlaying || text === "") { 60 | this.stopSpeech(); 61 | return; 62 | } 63 | 64 | // Create a new utterance and play it. 65 | const utterance = new SpeechSynthesisUtterance(text); 66 | utterance.lang = this.lngCode; 67 | utterance.volume = 1; 68 | utterance.rate = 1; 69 | utterance.pitch = 1; 70 | 71 | let voice = this.synth 72 | .getVoices() 73 | .filter((voice) => voice.lang === this.lngCode)[0]; 74 | if (!voice) { 75 | voice = this.synth 76 | .getVoices() 77 | .filter((voice) => voice.lang === "en-US")[0]; 78 | } 79 | utterance.voice = voice; 80 | 81 | if (!utterance) { 82 | return; 83 | } 84 | 85 | this.synth.speak(utterance); 86 | 87 | utterance.onstart = () => { 88 | this.isPlaying = true; 89 | this.renderButtonOn(); 90 | }; 91 | 92 | utterance.onend = () => { 93 | this.isPlaying = false; 94 | this.renderButtonOff(); 95 | }; 96 | } 97 | } 98 | 99 | stopSpeech() { 100 | if (this.synth) { 101 | this.synth.cancel(); 102 | this.isPlaying = false; 103 | this.renderButtonOff(); 104 | } 105 | } 106 | } 107 | 108 | customElements.define("speech-output-button", SpeechOutputButton); 109 | -------------------------------------------------------------------------------- /src/quartapp/static/styles.css: -------------------------------------------------------------------------------- 1 | * { 2 | box-sizing: border-box; 3 | } 4 | 5 | html, 6 | body { 7 | height: 100%; 8 | background-color: #f8f9fa; 9 | } 10 | 11 | #messages .toast-container { 12 | margin-bottom: 12px; 13 | } 14 | 15 | #messages .message-file img { 16 | max-width: 100%; 17 | max-height: 400px; 18 | } 19 | 20 | .background-user { 21 | background-color: #4f28b9; 22 | } 23 | 24 | .background-assistant { 25 | background-color: #0080ff; 26 | } 27 | 28 | #image-preview { 29 | max-height: 150px; 30 | float: right; 31 | margin-left: 20px; 32 | } 33 | 34 | #no-messages-heading { 35 | margin-top: 20%; 36 | } 37 | 38 | #chat-area { 39 | background-color: white; 40 | border: 1px solid #c4cad0; 41 | } 42 | 43 | /* Speech input/output buttons */ 44 | 45 | #chat-form speech-input-button button { 46 | border-top-right-radius: 0; 47 | border-bottom-right-radius: 0; 48 | } 49 | 50 | speech-input-button button.speech-input-active, 51 | speech-output-button button.speech-output-active { 52 | border: 1px solid blue; 53 | color: blue; 54 | animation: pulse 1s infinite; 55 | } 56 | 57 | speech-input-button button.speech-input-active i, 58 | speech-output-button button.speech-output-active { 59 | color: blue; 60 | } 61 | 62 | @keyframes pulse { 63 | 0% { 64 | box-shadow: 0 0 0 0 rgba(0, 123, 255, 0.7); 65 | } 66 | 70% { 67 | box-shadow: 0 0 0 6px rgba(0, 123, 255, 0); 68 | } 69 | 100% { 70 | box-shadow: 0 0 0 0 rgba(0, 123, 255, 0); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/quartapp/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | GPT-4o chat on image demo 8 | 10 | 12 | 14 | 15 | 16 | 17 |
18 |
19 |

Chat with your uploaded images

20 | 36 | 37 | 52 |
53 |
54 |
55 |
56 |
57 | 58 | 59 | 60 |
61 | 62 | 63 |
64 | 65 | 66 | 67 |
68 | 69 | 70 | 74 |
75 |
76 | 77 |
78 |
79 |
80 |
81 | 82 | 83 | 84 | 85 | 198 | 199 | 200 | -------------------------------------------------------------------------------- /src/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.11 3 | # by the following command: 4 | # 5 | # pip-compile --output-file=requirements.txt pyproject.toml 6 | # 7 | aiofiles==24.1.0 8 | # via quart 9 | aiohappyeyeballs==2.6.1 10 | # via aiohttp 11 | aiohttp==3.11.13 12 | # via quartapp (pyproject.toml) 13 | aiosignal==1.3.2 14 | # via aiohttp 15 | annotated-types==0.7.0 16 | # via pydantic 17 | anyio==4.8.0 18 | # via 19 | # httpx 20 | # openai 21 | # watchfiles 22 | attrs==25.2.0 23 | # via aiohttp 24 | azure-core==1.32.0 25 | # via azure-identity 26 | azure-identity==1.21.0 27 | # via quartapp (pyproject.toml) 28 | blinker==1.9.0 29 | # via 30 | # flask 31 | # quart 32 | certifi==2025.1.31 33 | # via 34 | # httpcore 35 | # httpx 36 | # requests 37 | cffi==1.17.1 38 | # via cryptography 39 | charset-normalizer==3.4.1 40 | # via requests 41 | click==8.1.8 42 | # via 43 | # flask 44 | # quart 45 | # uvicorn 46 | cryptography==44.0.2 47 | # via 48 | # azure-identity 49 | # msal 50 | # pyjwt 51 | distro==1.9.0 52 | # via openai 53 | flask==3.1.0 54 | # via quart 55 | frozenlist==1.5.0 56 | # via 57 | # aiohttp 58 | # aiosignal 59 | gunicorn==23.0.0 60 | # via quartapp (pyproject.toml) 61 | h11==0.14.0 62 | # via 63 | # httpcore 64 | # hypercorn 65 | # uvicorn 66 | # wsproto 67 | h2==4.2.0 68 | # via hypercorn 69 | hpack==4.1.0 70 | # via h2 71 | httpcore==1.0.7 72 | # via httpx 73 | httptools==0.6.4 74 | # via uvicorn 75 | httpx==0.28.1 76 | # via openai 77 | hypercorn==0.17.3 78 | # via quart 79 | hyperframe==6.1.0 80 | # via h2 81 | idna==3.10 82 | # via 83 | # anyio 84 | # httpx 85 | # requests 86 | # yarl 87 | itsdangerous==2.2.0 88 | # via 89 | # flask 90 | # quart 91 | jinja2==3.1.6 92 | # via 93 | # flask 94 | # quart 95 | jiter==0.9.0 96 | # via openai 97 | markupsafe==3.0.2 98 | # via 99 | # jinja2 100 | # quart 101 | # werkzeug 102 | msal==1.32.0 103 | # via 104 | # azure-identity 105 | # msal-extensions 106 | msal-extensions==1.2.0 107 | # via azure-identity 108 | multidict==6.1.0 109 | # via 110 | # aiohttp 111 | # yarl 112 | openai==1.66.3 113 | # via quartapp (pyproject.toml) 114 | packaging==24.2 115 | # via gunicorn 116 | portalocker==2.10.1 117 | # via msal-extensions 118 | priority==2.0.0 119 | # via hypercorn 120 | propcache==0.3.0 121 | # via 122 | # aiohttp 123 | # yarl 124 | pycparser==2.22 125 | # via cffi 126 | pydantic==2.10.6 127 | # via openai 128 | pydantic-core==2.27.2 129 | # via pydantic 130 | pyjwt[crypto]==2.10.1 131 | # via 132 | # msal 133 | # pyjwt 134 | python-dotenv==1.0.1 135 | # via 136 | # quartapp (pyproject.toml) 137 | # uvicorn 138 | pyyaml==6.0.2 139 | # via 140 | # quartapp (pyproject.toml) 141 | # uvicorn 142 | quart==0.20.0 143 | # via quartapp (pyproject.toml) 144 | requests==2.32.3 145 | # via 146 | # azure-core 147 | # msal 148 | six==1.17.0 149 | # via azure-core 150 | sniffio==1.3.1 151 | # via 152 | # anyio 153 | # openai 154 | tqdm==4.67.1 155 | # via openai 156 | typing-extensions==4.12.2 157 | # via 158 | # anyio 159 | # azure-core 160 | # azure-identity 161 | # openai 162 | # pydantic 163 | # pydantic-core 164 | urllib3==2.3.0 165 | # via requests 166 | uvicorn[standard]==0.34.0 167 | # via quartapp (pyproject.toml) 168 | uvloop==0.21.0 169 | # via uvicorn 170 | watchfiles==1.0.4 171 | # via uvicorn 172 | websockets==15.0.1 173 | # via uvicorn 174 | werkzeug==3.1.3 175 | # via 176 | # flask 177 | # quart 178 | # quartapp (pyproject.toml) 179 | wsproto==1.2.0 180 | # via hypercorn 181 | yarl==1.18.3 182 | # via aiohttp 183 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/openai-chat-vision-quickstart/e7506bf6519064158b6865cdee663bc8348ee55d/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import openai 2 | import pytest 3 | import pytest_asyncio 4 | 5 | import quartapp 6 | 7 | from . import mock_cred 8 | 9 | 10 | @pytest.fixture 11 | def mock_openai_chatcompletion(monkeypatch): 12 | class AsyncChatCompletionIterator: 13 | def __init__(self, answer: str): 14 | self.chunk_index = 0 15 | self.chunks = [ 16 | # This is an Azure-specific chunk solely for prompt_filter_results 17 | openai.types.chat.ChatCompletionChunk( 18 | object="chat.completion.chunk", 19 | choices=[], 20 | id="", 21 | created=0, 22 | model="", 23 | prompt_filter_results=[ 24 | { 25 | "prompt_index": 0, 26 | "content_filter_results": { 27 | "hate": {"filtered": False, "severity": "safe"}, 28 | "self_harm": {"filtered": False, "severity": "safe"}, 29 | "sexual": {"filtered": False, "severity": "safe"}, 30 | "violence": {"filtered": False, "severity": "safe"}, 31 | }, 32 | } 33 | ], 34 | ), 35 | openai.types.chat.ChatCompletionChunk( 36 | id="test-123", 37 | object="chat.completion.chunk", 38 | choices=[ 39 | openai.types.chat.chat_completion_chunk.Choice( 40 | delta=openai.types.chat.chat_completion_chunk.ChoiceDelta(content=None, role="assistant"), 41 | index=0, 42 | finish_reason=None, 43 | # Only Azure includes content_filter_results 44 | content_filter_results={}, 45 | ) 46 | ], 47 | created=1703462735, 48 | model="gpt-35-turbo", 49 | ), 50 | ] 51 | answer_deltas = answer.split(" ") 52 | for answer_index, answer_delta in enumerate(answer_deltas): 53 | # Completion chunks include whitespace, so we need to add it back in 54 | if answer_index > 0: 55 | answer_delta = " " + answer_delta 56 | self.chunks.append( 57 | openai.types.chat.ChatCompletionChunk( 58 | id="test-123", 59 | object="chat.completion.chunk", 60 | choices=[ 61 | openai.types.chat.chat_completion_chunk.Choice( 62 | delta=openai.types.chat.chat_completion_chunk.ChoiceDelta( 63 | role=None, content=answer_delta 64 | ), 65 | finish_reason=None, 66 | index=0, 67 | logprobs=None, 68 | # Only Azure includes content_filter_results 69 | content_filter_results={ 70 | "hate": {"filtered": False, "severity": "safe"}, 71 | "self_harm": {"filtered": False, "severity": "safe"}, 72 | "sexual": {"filtered": False, "severity": "safe"}, 73 | "violence": {"filtered": False, "severity": "safe"}, 74 | }, 75 | ) 76 | ], 77 | created=1703462735, 78 | model="gpt-35-turbo", 79 | ) 80 | ) 81 | self.chunks.append( 82 | openai.types.chat.ChatCompletionChunk( 83 | id="test-123", 84 | object="chat.completion.chunk", 85 | choices=[ 86 | openai.types.chat.chat_completion_chunk.Choice( 87 | delta=openai.types.chat.chat_completion_chunk.ChoiceDelta(content=None, role=None), 88 | index=0, 89 | finish_reason="stop", 90 | # Only Azure includes content_filter_results 91 | content_filter_results={}, 92 | ) 93 | ], 94 | created=1703462735, 95 | model="gpt-35-turbo", 96 | ) 97 | ) 98 | 99 | def __aiter__(self): 100 | return self 101 | 102 | async def __anext__(self): 103 | if self.chunk_index < len(self.chunks): 104 | next_chunk = self.chunks[self.chunk_index] 105 | self.chunk_index += 1 106 | return next_chunk 107 | else: 108 | raise StopAsyncIteration 109 | 110 | async def mock_acreate(*args, **kwargs): 111 | # Only mock a stream=True completion 112 | last_message = kwargs.get("messages")[-1]["content"] 113 | if last_message == "What is the capital of France?": 114 | return AsyncChatCompletionIterator("The capital of France is Paris.") 115 | elif last_message == "What is the capital of Germany?": 116 | return AsyncChatCompletionIterator("The capital of Germany is Berlin.") 117 | else: 118 | raise ValueError(f"Unexpected message: {last_message}") 119 | 120 | monkeypatch.setattr("openai.resources.chat.AsyncCompletions.create", mock_acreate) 121 | 122 | 123 | @pytest.fixture 124 | def mock_defaultazurecredential(monkeypatch): 125 | monkeypatch.setattr("azure.identity.aio.AzureDeveloperCliCredential", mock_cred.MockAzureDeveloperCliCredential) 126 | monkeypatch.setattr("azure.identity.aio.ManagedIdentityCredential", mock_cred.MockManagedIdentityCredential) 127 | 128 | 129 | @pytest_asyncio.fixture 130 | async def client(monkeypatch, mock_openai_chatcompletion, mock_defaultazurecredential): 131 | monkeypatch.setenv("OPENAI_HOST", "azure") 132 | monkeypatch.setenv("AZURE_TENANT_ID", "test-tenant-id") 133 | monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", "test-openai-service.openai.azure.com") 134 | monkeypatch.setenv("OPENAI_MODEL", "test-chatgpt") 135 | 136 | quart_app = quartapp.create_app() 137 | 138 | async with quart_app.test_app() as test_app: 139 | quart_app.config.update({"TESTING": True}) 140 | 141 | yield test_app.test_client() 142 | -------------------------------------------------------------------------------- /tests/mock_cred.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Mapping 2 | from typing import Any, Optional 3 | 4 | import azure.core.credentials_async 5 | 6 | 7 | class MockAzureDeveloperCliCredential(azure.core.credentials_async.AsyncTokenCredential): 8 | def __init__( 9 | self, 10 | *, 11 | tenant_id: str = "", 12 | additionally_allowed_tenants: Optional[list[str]] = None, 13 | process_timeout: int = 10, 14 | ) -> None: 15 | pass 16 | 17 | 18 | # Added as Python 3.13 throws a typing error when using the above code 19 | class MockManagedIdentityCredential(azure.core.credentials_async.AsyncTokenCredential): 20 | def __init__( 21 | self, *, client_id: Optional[str] = None, identity_config: Optional[Mapping[str, str]] = None, **kwargs: Any 22 | ) -> None: 23 | pass 24 | -------------------------------------------------------------------------------- /tests/snapshots/test_app/test_chat_stream_text/result.json: -------------------------------------------------------------------------------- 1 | {"delta": {"content": null, "function_call": null, "refusal": null, "role": "assistant", "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {}} 2 | {"delta": {"content": "The", "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}} 3 | {"delta": {"content": " capital", "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}} 4 | {"delta": {"content": " of", "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}} 5 | {"delta": {"content": " France", "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}} 6 | {"delta": {"content": " is", "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}} 7 | {"delta": {"content": " Paris.", "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}} 8 | {"delta": {"content": null, "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": "stop", "index": 0, "logprobs": null, "content_filter_results": {}} 9 | -------------------------------------------------------------------------------- /tests/snapshots/test_app/test_chat_stream_text_history/result.json: -------------------------------------------------------------------------------- 1 | {"delta": {"content": null, "function_call": null, "refusal": null, "role": "assistant", "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {}} 2 | {"delta": {"content": "The", "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}} 3 | {"delta": {"content": " capital", "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}} 4 | {"delta": {"content": " of", "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}} 5 | {"delta": {"content": " Germany", "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}} 6 | {"delta": {"content": " is", "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}} 7 | {"delta": {"content": " Berlin.", "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null, "content_filter_results": {"hate": {"filtered": false, "severity": "safe"}, "self_harm": {"filtered": false, "severity": "safe"}, "sexual": {"filtered": false, "severity": "safe"}, "violence": {"filtered": false, "severity": "safe"}}} 8 | {"delta": {"content": null, "function_call": null, "refusal": null, "role": null, "tool_calls": null}, "finish_reason": "stop", "index": 0, "logprobs": null, "content_filter_results": {}} 9 | -------------------------------------------------------------------------------- /tests/test_app.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import quartapp 4 | 5 | from . import mock_cred 6 | 7 | 8 | @pytest.mark.asyncio 9 | async def test_index(client): 10 | response = await client.get("/") 11 | assert response.status_code == 200 12 | 13 | 14 | @pytest.mark.asyncio 15 | async def test_chat_stream_text(client, snapshot): 16 | response = await client.post( 17 | "/chat/stream", 18 | json={ 19 | "messages": [ 20 | {"role": "user", "content": "What is the capital of France?"}, 21 | ], 22 | "context": {"file": ""}, 23 | }, 24 | ) 25 | assert response.status_code == 200 26 | result = await response.get_data() 27 | snapshot.assert_match(result, "result.json") 28 | 29 | 30 | @pytest.mark.asyncio 31 | async def test_chat_stream_text_history(client, snapshot): 32 | response = await client.post( 33 | "/chat/stream", 34 | json={ 35 | "messages": [ 36 | {"role": "user", "content": "What is the capital of France?"}, 37 | {"role": "assistant", "content": "Paris"}, 38 | {"role": "user", "content": "What is the capital of Germany?"}, 39 | ], 40 | "context": {"file": ""}, 41 | }, 42 | ) 43 | assert response.status_code == 200 44 | result = await response.get_data() 45 | snapshot.assert_match(result, "result.json") 46 | 47 | 48 | @pytest.mark.asyncio 49 | async def test_openai_key(monkeypatch): 50 | monkeypatch.setenv("OPENAI_HOST", "azure") 51 | monkeypatch.setenv("AZURE_OPENAI_KEY_FOR_CHATVISION", "test-key") 52 | monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", "test-openai-service.openai.azure.com") 53 | monkeypatch.setenv("OPENAI_MODEL", "test-chatgpt") 54 | monkeypatch.setenv("AZURE_OPENAI_VERSION", "2023-10-01-preview") 55 | 56 | quart_app = quartapp.create_app() 57 | 58 | async with quart_app.test_app(): 59 | assert quart_app.blueprints["chat"].openai_client.api_key == "test-key" 60 | assert quart_app.blueprints["chat"].openai_client._azure_ad_token_provider is None 61 | 62 | 63 | @pytest.mark.asyncio 64 | async def test_openai_managedidentity(monkeypatch): 65 | monkeypatch.setenv("OPENAI_HOST", "azure") 66 | monkeypatch.setenv("RUNNING_IN_PRODUCTION", "1") 67 | monkeypatch.setenv("AZURE_CLIENT_ID", "test-client-id") 68 | monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", "test-openai-service.openai.azure.com") 69 | monkeypatch.setenv("OPENAI_MODEL", "test-chatgpt") 70 | monkeypatch.setenv("AZURE_OPENAI_VERSION", "2023-10-01-preview") 71 | 72 | monkeypatch.setattr("azure.identity.aio.ManagedIdentityCredential", mock_cred.MockManagedIdentityCredential) 73 | 74 | quart_app = quartapp.create_app() 75 | 76 | async with quart_app.test_app(): 77 | assert quart_app.blueprints["chat"].openai_client._azure_ad_token_provider is not None 78 | 79 | 80 | @pytest.mark.asyncio 81 | async def test_openai_local(monkeypatch): 82 | monkeypatch.setenv("OPENAI_HOST", "local") 83 | monkeypatch.setenv("LOCAL_OPENAI_ENDPOINT", "http://localhost:8080") 84 | 85 | quart_app = quartapp.create_app() 86 | 87 | async with quart_app.test_app(): 88 | assert quart_app.blueprints["chat"].openai_client.api_key == "no-key-required" 89 | assert quart_app.blueprints["chat"].openai_client.base_url == "http://localhost:8080" 90 | --------------------------------------------------------------------------------