├── README.md ├── LICENSE └── Monitoring_LLM_Systems_in_Production_with_Comet_Opik.ipynb /README.md: -------------------------------------------------------------------------------- 1 | Some supporting code for the ClaireBot porject. 2 | * Learn more about ClaireBot RAG here: https://statistician-in-stilettos.medium.com/meet-clairebot-a-conversational-rag-llm-app-with-social-media-context-data-7346ca47d1ad 3 | * Learn more about the ClaireBot style advisor here: https://www.comet.com/site/blog/ai-chatbot-example-clairebot/ 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Claire Longo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Monitoring_LLM_Systems_in_Production_with_Comet_Opik.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [], 7 | "include_colab_link": true 8 | }, 9 | "kernelspec": { 10 | "name": "python3", 11 | "display_name": "Python 3" 12 | }, 13 | "language_info": { 14 | "name": "python" 15 | } 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "markdown", 20 | "metadata": { 21 | "id": "view-in-github", 22 | "colab_type": "text" 23 | }, 24 | "source": [ 25 | "\"Open" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "source": [ 31 | "#LLM Observability and Monitoring Tutorial with my Recipe Generator App\n", 32 | "\n", 33 | "This example personal project is designed to practice establishing proper montioring best practices for a live LLM app. In this tutorial, I'll walk you through how to build a very simple yet powerful LLM app, and show how to design the monitoring metrics and dashboards to facilitate detecting and root causing issues that might arise in production.\n" 34 | ], 35 | "metadata": { 36 | "id": "4nBkLEnK2yT1" 37 | } 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "source": [ 42 | "## Preparing our environment\n", 43 | "\n", 44 | "First, we will install the required libraries and set up our OpenAI API keys. Check out Opik config parameters here: https://www.comet.com/docs/opik/tracing/sdk_configuration" 45 | ], 46 | "metadata": { 47 | "id": "2eWfFnAJ4iRx" 48 | } 49 | }, 50 | { 51 | "cell_type": "code", 52 | "source": [ 53 | "%pip install -U opik openai --quiet" 54 | ], 55 | "metadata": { 56 | "id": "m080q6EML_Zr", 57 | "colab": { 58 | "base_uri": "https://localhost:8080/" 59 | }, 60 | "outputId": "cec0c636-c5bd-4d5c-f91e-e675ac22d440" 61 | }, 62 | "execution_count": null, 63 | "outputs": [ 64 | { 65 | "output_type": "stream", 66 | "name": "stdout", 67 | "text": [ 68 | "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/371.3 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m371.3/371.3 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 69 | "\u001b[?25h\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/463.1 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m463.1/463.1 kB\u001b[0m \u001b[31m28.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 70 | "\u001b[?25h\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/162.7 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m162.7/162.7 kB\u001b[0m \u001b[31m14.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 71 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m72.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 72 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m61.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 73 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m46.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 74 | "\u001b[?25h" 75 | ] 76 | } 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "source": [ 82 | "import opik\n", 83 | "\n", 84 | "opik.configure(use_local=False)" 85 | ], 86 | "metadata": { 87 | "id": "7uIKdRLv5ueg", 88 | "colab": { 89 | "base_uri": "https://localhost:8080/" 90 | }, 91 | "outputId": "4fe55ff6-9bdb-44d0-e10f-495f635b411c" 92 | }, 93 | "execution_count": null, 94 | "outputs": [ 95 | { 96 | "output_type": "stream", 97 | "name": "stderr", 98 | "text": [ 99 | "OPIK: Your Opik API key is available in your account settings, can be found at https://www.comet.com/api/my/settings/ for Opik cloud\n" 100 | ] 101 | }, 102 | { 103 | "name": "stdout", 104 | "output_type": "stream", 105 | "text": [ 106 | "Please enter your Opik API key:··········\n", 107 | "Do you want to use \"statisticianinstilettos\" workspace? (Y/n)Y\n" 108 | ] 109 | }, 110 | { 111 | "output_type": "stream", 112 | "name": "stderr", 113 | "text": [ 114 | "OPIK: Configuration saved to file: /root/.opik.config\n" 115 | ] 116 | } 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "source": [ 122 | "import os\n", 123 | "import getpass\n", 124 | "\n", 125 | "if \"OPENAI_API_KEY\" not in os.environ:\n", 126 | " os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"Enter your OpenAI API key: \")" 127 | ], 128 | "metadata": { 129 | "colab": { 130 | "base_uri": "https://localhost:8080/" 131 | }, 132 | "id": "5AjvhiY_4hug", 133 | "outputId": "770ee43f-f5bf-4475-8055-331bc1f416a4" 134 | }, 135 | "execution_count": null, 136 | "outputs": [ 137 | { 138 | "name": "stdout", 139 | "output_type": "stream", 140 | "text": [ 141 | "Enter your OpenAI API key: ··········\n" 142 | ] 143 | } 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "source": [ 149 | "from opik import track, opik_context\n", 150 | "from opik.integrations.openai import track_openai\n", 151 | "from openai import OpenAI\n", 152 | "\n", 153 | "os.environ[\"OPIK_PROJECT_NAME\"] = \"recipe-generator\"\n", 154 | "\n", 155 | "client = OpenAI()\n", 156 | "openai_client = track_openai(client)" 157 | ], 158 | "metadata": { 159 | "id": "sXjEXs245jH2" 160 | }, 161 | "execution_count": null, 162 | "outputs": [] 163 | }, 164 | { 165 | "cell_type": "markdown", 166 | "source": [ 167 | "# Building a simple recipe generator\n", 168 | "\n", 169 | "I'm using the OpenAI API to generate a response, and Opik to track and evaluate it." 170 | ], 171 | "metadata": { 172 | "id": "43lvGsyuhd18" 173 | } 174 | }, 175 | { 176 | "cell_type": "code", 177 | "source": [ 178 | "#This function performs the API call to openAI to request the recipe based on the prompt. Simply enter the dish name!\n", 179 | "#Because I'm using a function, I'm using the Opik @track decorator to log my traces.\n", 180 | "\n", 181 | "@opik.track(flush=True)\n", 182 | "def generate_recipe(dish_name):\n", 183 | "\n", 184 | " prompt = f\"\"\"\n", 185 | " You are a world-class home chef and recipe developer. Generate a step-by-step recipe for {dish_name}.\n", 186 | " Include a list of ingredients with quantities, clear cooking instructions, preparation time, cooking time, and serving size.\n", 187 | " If applicable, provide useful tips, optional variations, or suggestions for substitutes.\n", 188 | " Keep the recipe straightforward and easy to follow for a novice at home cook.\n", 189 | " \"\"\"\n", 190 | "\n", 191 | " # Request recipe generation from OpenAI's completions model\n", 192 | " response = openai_client.chat.completions.create(\n", 193 | " model=\"gpt-3.5-turbo\",\n", 194 | " messages=[\n", 195 | " {\"role\": \"system\", \"content\": \"You are a world-class chef and recipe developer.\"}, # system message\n", 196 | " {\"role\": \"user\", \"content\": prompt} # User's input prompt\n", 197 | " ],\n", 198 | " max_tokens=500,\n", 199 | " temperature=0.7,\n", 200 | " n=1\n", 201 | " )\n", 202 | "\n", 203 | " # Add user feedback score to the trace in Opik\n", 204 | " opik_context.update_current_trace(\n", 205 | " feedback_scores=[\n", 206 | " {\"name\": \"user_feedback\", \"value\": 1.0, \"reason\": \"It looks good.\"}\n", 207 | " ]\n", 208 | " )\n", 209 | "\n", 210 | " return response\n", 211 | "\n" 212 | ], 213 | "metadata": { 214 | "id": "o-A0UR8nKEuT" 215 | }, 216 | "execution_count": null, 217 | "outputs": [] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "source": [ 222 | "# Example usage\n", 223 | "dish_name = \"red velet cupcakes\"\n", 224 | "recipe = generate_recipe(dish_name)\n", 225 | "\n", 226 | "#extract just the recipe text and print\n", 227 | "print(recipe.choices[0].to_dict()['message']['content'])" 228 | ], 229 | "metadata": { 230 | "colab": { 231 | "base_uri": "https://localhost:8080/" 232 | }, 233 | "id": "8IGA3UzhLlS5", 234 | "outputId": "6e238873-5148-4484-ea34-fa5fc8bcf1ad" 235 | }, 236 | "execution_count": null, 237 | "outputs": [ 238 | { 239 | "output_type": "stream", 240 | "name": "stderr", 241 | "text": [ 242 | "OPIK: Started logging traces to the \"recipe-generator\" project at https://www.comet.com/opik/statisticianinstilettos/redirect/projects?name=recipe-generator.\n" 243 | ] 244 | }, 245 | { 246 | "output_type": "stream", 247 | "name": "stdout", 248 | "text": [ 249 | "**Cheddar Dill Biscuits**\n", 250 | "\n", 251 | "**Ingredients:**\n", 252 | "- 4 cups all-purpose flour\n", 253 | "- 1 packet instant yeast\n", 254 | "- 3 large eggs\n", 255 | "- 1 cup milk\n", 256 | "- 1 cup melted butter\n", 257 | "- 1 cup shredded cheddar cheese\n", 258 | "- 2 tablespoons dried dill\n", 259 | "- 1 teaspoon salt\n", 260 | "\n", 261 | "**Instructions:**\n", 262 | "\n", 263 | "**Preparation Time:** 20 minutes\n", 264 | "**Cooking Time:** 15-20 minutes\n", 265 | "**Serving Size:** 12 biscuits\n", 266 | "\n", 267 | "1. Preheat your oven to 375°F (190°C) and line a baking sheet with parchment paper.\n", 268 | " \n", 269 | "2. In a large mixing bowl, combine the flour, instant yeast, shredded cheddar cheese, dried dill, and salt. Mix well to combine all the dry ingredients.\n", 270 | "\n", 271 | "3. In a separate bowl, whisk together the eggs, milk, and melted butter until well combined.\n", 272 | "\n", 273 | "4. Make a well in the center of the dry ingredients and pour the wet mixture into it. Use a spatula to gently fold the wet and dry ingredients together until a soft dough forms. Do not overmix.\n", 274 | "\n", 275 | "5. Turn the dough out onto a lightly floured surface and gently knead it a few times until it comes together.\n", 276 | "\n", 277 | "6. Roll out the dough to about 1-inch thickness. Use a round cookie cutter or a glass to cut out biscuits from the dough.\n", 278 | "\n", 279 | "7. Place the biscuits on the prepared baking sheet, leaving a little space between each one.\n", 280 | "\n", 281 | "8. Bake in the preheated oven for 15-20 minutes, or until the biscuits are golden brown and cooked through.\n", 282 | "\n", 283 | "9. Remove the biscuits from the oven and let them cool slightly before serving.\n", 284 | "\n", 285 | "**Tips:**\n", 286 | "- Be gentle when mixing and kneading the dough to ensure a light and fluffy texture.\n", 287 | "- Feel free to add more cheese or dill according to your taste preference.\n", 288 | "- Serve the biscuits warm with butter or your favorite spread.\n", 289 | "\n", 290 | "Enjoy your delicious homemade Cheddar Dill Biscuits that are reminiscent of the ones served at Red Lobster!\n" 291 | ] 292 | } 293 | ] 294 | } 295 | ] 296 | } --------------------------------------------------------------------------------