![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
"
26 | ]
27 | },
28 | {
29 | "cell_type": "markdown",
30 | "source": [
31 | "## Set Up Environment"
32 | ],
33 | "metadata": {
34 | "id": "YhwwvuUggrQG"
35 | }
36 | },
37 | {
38 | "cell_type": "code",
39 | "execution_count": 1,
40 | "metadata": {
41 | "colab": {
42 | "base_uri": "https://localhost:8080/"
43 | },
44 | "id": "K6pNBFzcer0X",
45 | "outputId": "9a7c02e0-6167-43aa-ceba-d6aa21b5558b"
46 | },
47 | "outputs": [
48 | {
49 | "output_type": "stream",
50 | "name": "stdout",
51 | "text": [
52 | "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/337.0 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━\u001b[0m \u001b[32m327.7/337.0 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m337.0/337.0 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
53 | "\u001b[?25h\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/75.6 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
54 | "\u001b[?25h\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/77.9 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
55 | "\u001b[?25h\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/58.3 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
56 | "\u001b[?25h"
57 | ]
58 | }
59 | ],
60 | "source": [
61 | "!pip install --upgrade --quiet openai"
62 | ]
63 | },
64 | {
65 | "cell_type": "code",
66 | "source": [
67 | "from google.colab import userdata\n",
68 | "import os\n",
69 | "\n",
70 | "os.environ[\"OPENAI_API_KEY\"] = userdata.get('OPENAI_API_KEY')"
71 | ],
72 | "metadata": {
73 | "id": "kFaWZqnbfN1k"
74 | },
75 | "execution_count": 2,
76 | "outputs": []
77 | },
78 | {
79 | "cell_type": "code",
80 | "source": [
81 | "import json\n",
82 | "import openai\n",
83 | "import os\n",
84 | "import pandas as pd\n",
85 | "from pprint import pprint\n",
86 | "\n",
87 | "client = openai.OpenAI()"
88 | ],
89 | "metadata": {
90 | "id": "r8r4EUxCfYOE"
91 | },
92 | "execution_count": 3,
93 | "outputs": []
94 | },
95 | {
96 | "cell_type": "code",
97 | "source": [
98 | "recipe_df = pd.read_csv(\"/content/cookbook_recipes_nlg_10k.csv\")\n",
99 | "\n",
100 | "recipe_df.head()"
101 | ],
102 | "metadata": {
103 | "colab": {
104 | "base_uri": "https://localhost:8080/",
105 | "height": 276
106 | },
107 | "id": "sIPRcIwdfiMA",
108 | "outputId": "a61025d1-0862-4e48-8ede-df5a3660a93e"
109 | },
110 | "execution_count": 4,
111 | "outputs": [
112 | {
113 | "output_type": "execute_result",
114 | "data": {
115 | "text/plain": [
116 | " title ingredients \\\n",
117 | "0 No-Bake Nut Cookies [\"1 c. firmly packed brown sugar\", \"1/2 c. eva... \n",
118 | "1 Jewell Ball'S Chicken [\"1 small jar chipped beef, cut up\", \"4 boned ... \n",
119 | "2 Creamy Corn [\"2 (16 oz.) pkg. frozen corn\", \"1 (8 oz.) pkg... \n",
120 | "3 Chicken Funny [\"1 large whole chicken\", \"2 (10 1/2 oz.) cans... \n",
121 | "4 Reeses Cups(Candy) [\"1 c. peanut butter\", \"3/4 c. graham cracker ... \n",
122 | "\n",
123 | " directions \\\n",
124 | "0 [\"In a heavy 2-quart saucepan, mix brown sugar... \n",
125 | "1 [\"Place chipped beef on bottom of baking dish.... \n",
126 | "2 [\"In a slow cooker, combine all ingredients. C... \n",
127 | "3 [\"Boil and debone chicken.\", \"Put bite size pi... \n",
128 | "4 [\"Combine first four ingredients and press in ... \n",
129 | "\n",
130 | " link source \\\n",
131 | "0 www.cookbooks.com/Recipe-Details.aspx?id=44874 www.cookbooks.com \n",
132 | "1 www.cookbooks.com/Recipe-Details.aspx?id=699419 www.cookbooks.com \n",
133 | "2 www.cookbooks.com/Recipe-Details.aspx?id=10570 www.cookbooks.com \n",
134 | "3 www.cookbooks.com/Recipe-Details.aspx?id=897570 www.cookbooks.com \n",
135 | "4 www.cookbooks.com/Recipe-Details.aspx?id=659239 www.cookbooks.com \n",
136 | "\n",
137 | " NER \n",
138 | "0 [\"brown sugar\", \"milk\", \"vanilla\", \"nuts\", \"bu... \n",
139 | "1 [\"beef\", \"chicken breasts\", \"cream of mushroom... \n",
140 | "2 [\"frozen corn\", \"cream cheese\", \"butter\", \"gar... \n",
141 | "3 [\"chicken\", \"chicken gravy\", \"cream of mushroo... \n",
142 | "4 [\"peanut butter\", \"graham cracker crumbs\", \"bu... "
143 | ],
144 | "text/html": [
145 | "\n",
146 | " | \n", 165 | " | title | \n", 166 | "ingredients | \n", 167 | "directions | \n", 168 | "link | \n", 169 | "source | \n", 170 | "NER | \n", 171 | "
|---|---|---|---|---|---|---|
| 0 | \n", 176 | "No-Bake Nut Cookies | \n", 177 | "[\"1 c. firmly packed brown sugar\", \"1/2 c. eva... | \n", 178 | "[\"In a heavy 2-quart saucepan, mix brown sugar... | \n", 179 | "www.cookbooks.com/Recipe-Details.aspx?id=44874 | \n", 180 | "www.cookbooks.com | \n", 181 | "[\"brown sugar\", \"milk\", \"vanilla\", \"nuts\", \"bu... | \n", 182 | "
| 1 | \n", 185 | "Jewell Ball'S Chicken | \n", 186 | "[\"1 small jar chipped beef, cut up\", \"4 boned ... | \n", 187 | "[\"Place chipped beef on bottom of baking dish.... | \n", 188 | "www.cookbooks.com/Recipe-Details.aspx?id=699419 | \n", 189 | "www.cookbooks.com | \n", 190 | "[\"beef\", \"chicken breasts\", \"cream of mushroom... | \n", 191 | "
| 2 | \n", 194 | "Creamy Corn | \n", 195 | "[\"2 (16 oz.) pkg. frozen corn\", \"1 (8 oz.) pkg... | \n", 196 | "[\"In a slow cooker, combine all ingredients. C... | \n", 197 | "www.cookbooks.com/Recipe-Details.aspx?id=10570 | \n", 198 | "www.cookbooks.com | \n", 199 | "[\"frozen corn\", \"cream cheese\", \"butter\", \"gar... | \n", 200 | "
| 3 | \n", 203 | "Chicken Funny | \n", 204 | "[\"1 large whole chicken\", \"2 (10 1/2 oz.) cans... | \n", 205 | "[\"Boil and debone chicken.\", \"Put bite size pi... | \n", 206 | "www.cookbooks.com/Recipe-Details.aspx?id=897570 | \n", 207 | "www.cookbooks.com | \n", 208 | "[\"chicken\", \"chicken gravy\", \"cream of mushroo... | \n", 209 | "
| 4 | \n", 212 | "Reeses Cups(Candy) | \n", 213 | "[\"1 c. peanut butter\", \"3/4 c. graham cracker ... | \n", 214 | "[\"Combine first four ingredients and press in ... | \n", 215 | "www.cookbooks.com/Recipe-Details.aspx?id=659239 | \n", 216 | "www.cookbooks.com | \n", 217 | "[\"peanut butter\", \"graham cracker crumbs\", \"bu... | \n", 218 | "
"
27 | ]
28 | },
29 | {
30 | "cell_type": "markdown",
31 | "source": [
32 | "# OpenAI Assistants and Files API based RAG Application\n",
33 | "\n",
34 | "**Assistants API** allows developers to build AI assistants that can handle the stateful operations required for LLM based applications, including persistent threads and messages, files, and automatic RAG.\n",
35 | "\n",
36 | "An Assistant has instructions and can leverage models, tools, and knowledge to respond to user queries. The Assistants API currently supports three types of tools:\n",
37 | "\n",
38 | "- Code Interpreter\n",
39 | "- Retrieval\n",
40 | "- Function calling\n",
41 | "\n",
42 | "Notice that, when we give `Assistants` access to OpenAI-hosted tools listed above, the usage of the tools comes at an additional fee.\n",
43 | "\n",
44 | "**RAG** (Retrieval Augmented Generation) is a technique used in natural language processing that employes the capabilities of retrieval-based models and generative models to improve the quality and relevance of generated text.\n",
45 | "\n",
46 | "Document based QA bot is a classic use case of RAG. The mainstream LLM frameworks for example `LangChain` and `LlamaIndex` support building such RAG application.\n",
47 | "\n",
48 | "In this tutorial, I will show you how to develop a RAG application with OpenAI `Assistants` and `Files` API. The code may be cleaner, the solution may be more elegant, and it may cost a bit more, considering the extra charge for using OpenAI hosted tool - **`Retrieval`**.\n",
49 | "\n",
50 | "With this solution, you don't have to deal with the following tedious operations:\n",
51 | "\n",
52 | "- Split text with proper strategy\n",
53 | "- Vectorize text chunks\n",
54 | "- Persist vector data set\n",
55 | "- Similarity search\n"
56 | ],
57 | "metadata": {
58 | "id": "Dej4rmcv1qCm"
59 | }
60 | },
61 | {
62 | "cell_type": "markdown",
63 | "source": [
64 | "## Prepare the environment\n",
65 | "\n",
66 | "In order to run the example code below, make sure you have a valid OpenAI API key with access to the model `gpt-4-1106-preview`.\n",
67 | "\n",
68 | "You also should have the `.env` file ready in current directory with the content in the pattern below:\n",
69 | "\n",
70 | "```shell\n",
71 | "OPENAI_API_KEY=sk-xxxxxx\n",
72 | "```\n",
73 | "\n",
74 | "Now let's install the necessary Python packages."
75 | ],
76 | "metadata": {
77 | "id": "ipQHI0Fe7OP7"
78 | }
79 | },
80 | {
81 | "cell_type": "code",
82 | "execution_count": 105,
83 | "metadata": {
84 | "id": "GkF_TXMN2AML"
85 | },
86 | "outputs": [],
87 | "source": [
88 | "!pip install openai python-dotenv -U -q"
89 | ]
90 | },
91 | {
92 | "cell_type": "markdown",
93 | "source": [
94 | "## Coding Time\n",
95 | "\n",
96 | "We can start coding. In this example, we will use the following PDF file as the knowledge base:\n",
97 | "\n",
98 | "[2023 Venture Capital Report](https://www.wilmerhale.com/-/media/files/shared_content/editorial/publications/documents/2023-wilmerhale-vc-report.pdf)\n",
99 | "\n",
100 | "This is a Venture Capital report of 2023 issued by WilmerHale."
101 | ],
102 | "metadata": {
103 | "id": "O2f7XFsu8BIK"
104 | }
105 | },
106 | {
107 | "cell_type": "markdown",
108 | "source": [
109 | "### 1. Load"
110 | ],
111 | "metadata": {
112 | "id": "qUpdc89s8lmd"
113 | }
114 | },
115 | {
116 | "cell_type": "code",
117 | "source": [
118 | "from dotenv import load_dotenv\n",
119 | "load_dotenv()\n",
120 | "\n",
121 | "import os"
122 | ],
123 | "metadata": {
124 | "id": "ywRTtMmR2UKV"
125 | },
126 | "execution_count": 107,
127 | "outputs": []
128 | },
129 | {
130 | "cell_type": "code",
131 | "source": [
132 | "openai_api_key = os.environ[\"OPENAI_API_KEY\"]"
133 | ],
134 | "metadata": {
135 | "id": "GeW4OJ6Y2rR7"
136 | },
137 | "execution_count": 108,
138 | "outputs": []
139 | },
140 | {
141 | "cell_type": "code",
142 | "source": [
143 | "import openai\n",
144 | "\n",
145 | "# You don't have to explicitly fetch the API key from environmental variables and assign it.\n",
146 | "# OpenAI class will load it from env var OPENAI_API_KEY automatically.\n",
147 | "\n",
148 | "openai_client = openai.OpenAI(api_key=openai_api_key)"
149 | ],
150 | "metadata": {
151 | "id": "LPz0oCZkEcwb"
152 | },
153 | "execution_count": 65,
154 | "outputs": []
155 | },
156 | {
157 | "cell_type": "markdown",
158 | "source": [
159 | "### 2. Retrieve existing files\n",
160 | "\n",
161 | "The files uploaded via `OpenAI Files API` are persisted. They can be surely reused by referring to their ids.\n",
162 | "\n",
163 | "In this step, we will retrieve the files already uploaded and see if the PDF file `2023-WilmerHale-VC-Report.pdf` exists.\n",
164 | "\n",
165 | "If so, let's delete and upload again."
166 | ],
167 | "metadata": {
168 | "id": "lNi_L5_V9mZz"
169 | }
170 | },
171 | {
172 | "cell_type": "code",
173 | "source": [
174 | "# Retrieve the file list\n",
175 | "\n",
176 | "uploaded_files = openai_client.files.list()"
177 | ],
178 | "metadata": {
179 | "id": "2LTwlGShzYJd"
180 | },
181 | "execution_count": 127,
182 | "outputs": []
183 | },
184 | {
185 | "cell_type": "code",
186 | "source": [
187 | "uploaded_files.data"
188 | ],
189 | "metadata": {
190 | "colab": {
191 | "base_uri": "https://localhost:8080/"
192 | },
193 | "id": "s0YAls8qzkoH",
194 | "outputId": "a2652496-3e9c-4c03-9ecf-a0277a063109"
195 | },
196 | "execution_count": 128,
197 | "outputs": [
198 | {
199 | "output_type": "execute_result",
200 | "data": {
201 | "text/plain": [
202 | "[FileObject(id='file-KC7h7nJFf2XBmXGJP2Arzzno', bytes=24117248, created_at=1700154133, filename='Standford_AI-Index-Report_2023.pdf', object='file', purpose='assistants', status='processed', status_details=None)]"
203 | ]
204 | },
205 | "metadata": {},
206 | "execution_count": 128
207 | }
208 | ]
209 | },
210 | {
211 | "cell_type": "code",
212 | "source": [
213 | "# Find the file by name\n",
214 | "\n",
215 | "filename_to_find = '2023-WilmerHale-VC-Report.pdf'\n",
216 | "the_file_id = None\n",
217 | "\n",
218 | "file_objects = list(filter(lambda x: x.filename == filename_to_find, uploaded_files.data))\n",
219 | "\n",
220 | "if len(file_objects) > 0:\n",
221 | " the_file_id = file_objects[0].id"
222 | ],
223 | "metadata": {
224 | "id": "aROVPPAl-a-m"
225 | },
226 | "execution_count": 124,
227 | "outputs": []
228 | },
229 | {
230 | "cell_type": "code",
231 | "source": [
232 | "the_file_id"
233 | ],
234 | "metadata": {
235 | "colab": {
236 | "base_uri": "https://localhost:8080/",
237 | "height": 36
238 | },
239 | "id": "m0NMZ59sIC-N",
240 | "outputId": "8287e04c-d9c3-4b5a-837f-64c5ae444d3b"
241 | },
242 | "execution_count": 126,
243 | "outputs": [
244 | {
245 | "output_type": "execute_result",
246 | "data": {
247 | "text/plain": [
248 | "'file-1zy5eXDx5nDotjUp4tybtNGl'"
249 | ],
250 | "application/vnd.google.colaboratory.intrinsic+json": {
251 | "type": "string"
252 | }
253 | },
254 | "metadata": {},
255 | "execution_count": 126
256 | }
257 | ]
258 | },
259 | {
260 | "cell_type": "markdown",
261 | "source": [
262 | "Delete if it already exists.\n",
263 | "\n",
264 | "Notice that, this is for demonstration purpose."
265 | ],
266 | "metadata": {
267 | "id": "IJVmHdvP_wB1"
268 | }
269 | },
270 | {
271 | "cell_type": "code",
272 | "source": [
273 | "if the_file_id:\n",
274 | " delete_status = openai_client.files.delete(the_file_id)\n",
275 | " delete_status"
276 | ],
277 | "metadata": {
278 | "id": "DVouWaqgzr4Y"
279 | },
280 | "execution_count": 125,
281 | "outputs": []
282 | },
283 | {
284 | "cell_type": "markdown",
285 | "source": [
286 | "### 2. Upload the PDF file\n",
287 | "\n",
288 | "This PDF file will be the knowledge base of the RAG application."
289 | ],
290 | "metadata": {
291 | "id": "eBpTSwLQ_2we"
292 | }
293 | },
294 | {
295 | "cell_type": "code",
296 | "source": [
297 | "file = openai_client.files.create(\n",
298 | " file=open(\"2023-WilmerHale-VC-Report.pdf\", \"rb\"),\n",
299 | " purpose='assistants'\n",
300 | ")"
301 | ],
302 | "metadata": {
303 | "id": "MIbK3o4XFH1N"
304 | },
305 | "execution_count": 129,
306 | "outputs": []
307 | },
308 | {
309 | "cell_type": "code",
310 | "source": [
311 | "file"
312 | ],
313 | "metadata": {
314 | "colab": {
315 | "base_uri": "https://localhost:8080/"
316 | },
317 | "id": "4BEJW7lxFXVX",
318 | "outputId": "58cd48e1-3320-49d6-d4c9-64c076ef328e"
319 | },
320 | "execution_count": 130,
321 | "outputs": [
322 | {
323 | "output_type": "execute_result",
324 | "data": {
325 | "text/plain": [
326 | "FileObject(id='file-p1e3aGURKtzkR83g91YUTBhn', bytes=1310948, created_at=1700171663, filename='2023-WilmerHale-VC-Report.pdf', object='file', purpose='assistants', status='processed', status_details=None)"
327 | ]
328 | },
329 | "metadata": {},
330 | "execution_count": 130
331 | }
332 | ]
333 | },
334 | {
335 | "cell_type": "markdown",
336 | "source": [
337 | "### 3. Retrieve the file by id\n",
338 | "\n",
339 | "This is to make sure the file is successfully uploaded."
340 | ],
341 | "metadata": {
342 | "id": "agL44d-1__wU"
343 | }
344 | },
345 | {
346 | "cell_type": "code",
347 | "source": [
348 | "retrieved_file = openai_client.files.retrieve(file.id)\n",
349 | "retrieved_file"
350 | ],
351 | "metadata": {
352 | "colab": {
353 | "base_uri": "https://localhost:8080/"
354 | },
355 | "id": "3vg3-7t3cb2_",
356 | "outputId": "c305d4f6-3174-44d5-a97e-8cefe203d7b2"
357 | },
358 | "execution_count": 131,
359 | "outputs": [
360 | {
361 | "output_type": "execute_result",
362 | "data": {
363 | "text/plain": [
364 | "FileObject(id='file-p1e3aGURKtzkR83g91YUTBhn', bytes=1310948, created_at=1700171663, filename='2023-WilmerHale-VC-Report.pdf', object='file', purpose='assistants', status='processed', status_details=None)"
365 | ]
366 | },
367 | "metadata": {},
368 | "execution_count": 131
369 | }
370 | ]
371 | },
372 | {
373 | "cell_type": "markdown",
374 | "source": [
375 | "### 4. Create an Assistant\n",
376 | "\n",
377 | "This Assistant will use the tool `Retrieval` and get associated with the PDF file uploaded by its id."
378 | ],
379 | "metadata": {
380 | "id": "ju7oOHNfAI2D"
381 | }
382 | },
383 | {
384 | "cell_type": "code",
385 | "source": [
386 | "assistant = openai_client.beta.assistants.create(\n",
387 | " instructions=\"Use the file provided as your knowledge base to best respond to customer queries.\",\n",
388 | " model=\"gpt-4-1106-preview\",\n",
389 | " tools=[\n",
390 | " { \"type\": \"retrieval\" }\n",
391 | " ],\n",
392 | " file_ids=[retrieved_file.id]\n",
393 | ")"
394 | ],
395 | "metadata": {
396 | "id": "iwD1noKvFds9"
397 | },
398 | "execution_count": 132,
399 | "outputs": []
400 | },
401 | {
402 | "cell_type": "markdown",
403 | "source": [
404 | "### 5. Retrieve the created Assistant\n",
405 | "\n",
406 | "It's retrieved by the assistant id.\n",
407 | "\n",
408 | "We must make sure in the response, we can see expected tool and file are associated with it."
409 | ],
410 | "metadata": {
411 | "id": "uAZ0Y-egATbn"
412 | }
413 | },
414 | {
415 | "cell_type": "code",
416 | "source": [
417 | "my_assistant = openai_client.beta.assistants.retrieve(assistant.id)\n",
418 | "my_assistant"
419 | ],
420 | "metadata": {
421 | "colab": {
422 | "base_uri": "https://localhost:8080/"
423 | },
424 | "id": "YU7XgHKfcJmG",
425 | "outputId": "5a8bfafb-ed28-4ce9-b7ee-cd7cf718ca54"
426 | },
427 | "execution_count": 133,
428 | "outputs": [
429 | {
430 | "output_type": "execute_result",
431 | "data": {
432 | "text/plain": [
433 | "Assistant(id='asst_YYN7UKBkyuFs5UghtNfzxO3z', created_at=1700171729, description=None, file_ids=['file-p1e3aGURKtzkR83g91YUTBhn'], instructions='Use the file provided as your knowledge base to best respond to customer queries.', metadata={}, model='gpt-4-1106-preview', name=None, object='assistant', tools=[ToolRetrieval(type='retrieval')])"
434 | ]
435 | },
436 | "metadata": {},
437 | "execution_count": 133
438 | }
439 | ]
440 | },
441 | {
442 | "cell_type": "markdown",
443 | "source": [
444 | "### 6. (Optional) Update the Assistant\n",
445 | "\n",
446 | "I noticed once that the created assistant didn't have the tool and file associated.\n",
447 | "\n",
448 | "When it happens to you, use the `update` function to associate them again."
449 | ],
450 | "metadata": {
451 | "id": "Gyba9nuNAgqu"
452 | }
453 | },
454 | {
455 | "cell_type": "code",
456 | "source": [
457 | "updated_assistant = openai_client.beta.assistants.update(\n",
458 | " assistant.id,\n",
459 | " tools=[{\"type\": \"retrieval\"}],\n",
460 | " file_ids=[retrieved_file.id],\n",
461 | ")\n",
462 | "\n",
463 | "updated_assistant"
464 | ],
465 | "metadata": {
466 | "colab": {
467 | "base_uri": "https://localhost:8080/"
468 | },
469 | "id": "Am6RzSLfeEs3",
470 | "outputId": "a206f689-2af1-4a3d-e6ab-b02a62bbd271"
471 | },
472 | "execution_count": 134,
473 | "outputs": [
474 | {
475 | "output_type": "execute_result",
476 | "data": {
477 | "text/plain": [
478 | "Assistant(id='asst_YYN7UKBkyuFs5UghtNfzxO3z', created_at=1700171729, description=None, file_ids=['file-p1e3aGURKtzkR83g91YUTBhn'], instructions='Use the file provided as your knowledge base to best respond to customer queries.', metadata={}, model='gpt-4-1106-preview', name=None, object='assistant', tools=[ToolRetrieval(type='retrieval')])"
479 | ]
480 | },
481 | "metadata": {},
482 | "execution_count": 134
483 | }
484 | ]
485 | },
486 | {
487 | "cell_type": "markdown",
488 | "source": [
489 | "### 7. Create a Thread"
490 | ],
491 | "metadata": {
492 | "id": "e1pi3ta8AvX4"
493 | }
494 | },
495 | {
496 | "cell_type": "code",
497 | "source": [
498 | "thread = openai_client.beta.threads.create()\n",
499 | "\n",
500 | "thread"
501 | ],
502 | "metadata": {
503 | "colab": {
504 | "base_uri": "https://localhost:8080/"
505 | },
506 | "id": "5H1cR3XdF0R1",
507 | "outputId": "6adcb900-cad0-437b-ee24-15a44ac39504"
508 | },
509 | "execution_count": 135,
510 | "outputs": [
511 | {
512 | "output_type": "execute_result",
513 | "data": {
514 | "text/plain": [
515 | "Thread(id='thread_PIN1poM9aRX4sYjnnohlvZNa', created_at=1700171825, metadata={}, object='thread')"
516 | ]
517 | },
518 | "metadata": {},
519 | "execution_count": 135
520 | }
521 | ]
522 | },
523 | {
524 | "cell_type": "markdown",
525 | "source": [
526 | "### 8. Create a Message\n",
527 | "\n",
528 | "We are going to use a message object to request the Assistant to extract the content architecture out of the PDF file."
529 | ],
530 | "metadata": {
531 | "id": "PiCMxyJNAyEU"
532 | }
533 | },
534 | {
535 | "cell_type": "code",
536 | "source": [
537 | "thread_message = openai_client.beta.threads.messages.create(\n",
538 | " thread_id=thread.id,\n",
539 | " role=\"user\",\n",
540 | " content=\"Please show me the content architecture of this report\",\n",
541 | ")\n",
542 | "thread_message"
543 | ],
544 | "metadata": {
545 | "colab": {
546 | "base_uri": "https://localhost:8080/"
547 | },
548 | "id": "sD4YL8apMoUV",
549 | "outputId": "7c31e5ba-499d-4e3b-a2dd-e05d823987fd"
550 | },
551 | "execution_count": 136,
552 | "outputs": [
553 | {
554 | "output_type": "execute_result",
555 | "data": {
556 | "text/plain": [
557 | "ThreadMessage(id='msg_r00GeAZ3q4zaQYxaYlGsjMmd', assistant_id=None, content=[MessageContentText(text=Text(annotations=[], value='Please show me the content architecture of this report'), type='text')], created_at=1700171844, file_ids=[], metadata={}, object='thread.message', role='user', run_id=None, thread_id='thread_PIN1poM9aRX4sYjnnohlvZNa')"
558 | ]
559 | },
560 | "metadata": {},
561 | "execution_count": 136
562 | }
563 | ]
564 | },
565 | {
566 | "cell_type": "markdown",
567 | "source": [
568 | "### 9. Create a Run\n",
569 | "\n",
570 | "A `Run` triggers the interaction to the LLM."
571 | ],
572 | "metadata": {
573 | "id": "x8jMsm-DBG8R"
574 | }
575 | },
576 | {
577 | "cell_type": "code",
578 | "source": [
579 | "run = openai_client.beta.threads.runs.create(\n",
580 | " thread_id=thread.id,\n",
581 | " assistant_id=updated_assistant.id\n",
582 | ")"
583 | ],
584 | "metadata": {
585 | "id": "Yp1Fr_FEL65n"
586 | },
587 | "execution_count": 137,
588 | "outputs": []
589 | },
590 | {
591 | "cell_type": "markdown",
592 | "source": [
593 | "### 10. Retrieve the Run\n",
594 | "\n",
595 | "The `Run` is done in async mode, so we need to query the status of the Run by id."
596 | ],
597 | "metadata": {
598 | "id": "o7HYAnqABRay"
599 | }
600 | },
601 | {
602 | "cell_type": "code",
603 | "source": [
604 | "retrieved_run = openai_client.beta.threads.runs.retrieve(\n",
605 | " thread_id=thread.id,\n",
606 | " run_id=run.id\n",
607 | ")"
608 | ],
609 | "metadata": {
610 | "id": "rCv4p11TNJUu"
611 | },
612 | "execution_count": 138,
613 | "outputs": []
614 | },
615 | {
616 | "cell_type": "code",
617 | "source": [
618 | "retrieved_run"
619 | ],
620 | "metadata": {
621 | "colab": {
622 | "base_uri": "https://localhost:8080/"
623 | },
624 | "id": "nILYSfeINRnw",
625 | "outputId": "6ca7eb08-dcdd-4d09-c52f-c5556b800fd6"
626 | },
627 | "execution_count": 139,
628 | "outputs": [
629 | {
630 | "output_type": "execute_result",
631 | "data": {
632 | "text/plain": [
633 | "Run(id='run_uGwTGAemL5rqFwWlHc02Wxwl', assistant_id='asst_YYN7UKBkyuFs5UghtNfzxO3z', cancelled_at=None, completed_at=1700171869, created_at=1700171854, expires_at=None, failed_at=None, file_ids=['file-p1e3aGURKtzkR83g91YUTBhn'], instructions='Use the file provided as your knowledge base to best respond to customer queries.', last_error=None, metadata={}, model='gpt-4-1106-preview', object='thread.run', required_action=None, started_at=1700171854, status='completed', thread_id='thread_PIN1poM9aRX4sYjnnohlvZNa', tools=[ToolAssistantToolsRetrieval(type='retrieval')])"
634 | ]
635 | },
636 | "metadata": {},
637 | "execution_count": 139
638 | }
639 | ]
640 | },
641 | {
642 | "cell_type": "markdown",
643 | "source": [
644 | "### 11. Retrieve the message list of the Thread\n",
645 | "\n",
646 | "Until the Run is completed, retrieve the message list and fetch the latest message of the list which is the response of the LLM."
647 | ],
648 | "metadata": {
649 | "id": "fCcSFIXoBc6i"
650 | }
651 | },
652 | {
653 | "cell_type": "code",
654 | "source": [
655 | "thread_messages = openai_client.beta.threads.messages.list(thread.id)\n",
656 | "thread_messages.data"
657 | ],
658 | "metadata": {
659 | "colab": {
660 | "base_uri": "https://localhost:8080/"
661 | },
662 | "id": "dAbqv9S7Ne3n",
663 | "outputId": "37a14842-5c11-47db-e177-d5ef1ae63a09"
664 | },
665 | "execution_count": 140,
666 | "outputs": [
667 | {
668 | "output_type": "execute_result",
669 | "data": {
670 | "text/plain": [
671 | "[ThreadMessage(id='msg_UYQX4LrKPLpVNjhjBQRUGO07', assistant_id='asst_YYN7UKBkyuFs5UghtNfzxO3z', content=[MessageContentText(text=Text(annotations=[TextAnnotationFileCitation(end_index=817, file_citation=TextAnnotationFileCitationFileCitation(file_id='file-p1e3aGURKtzkR83g91YUTBhn', quote='2023 Venture Capital Report – What’s Inside\\n\\n\\n2 US Market Review and Outlook\\n\\n\\n6 Regional Market Review and Outlook\\n\\n\\n10 Selected WilmerHale Venture Capital Financings\\n\\n\\n12 New Law Requires Federal Reporting of Private Company Ownership\\nMany Startups and Life Sciences Companies Will be Subject to Beneficial Ownership\\nReporting\\n\\n\\n13 Show Me the Money\\nWhat Employers Need to Know About New Salary Disclosure Laws\\n\\n\\n14 Navigating the Quiet Period Shoals\\nSafe Harbors Aid Compliance With Quiet Period Requirements\\n\\n\\n16 State Taxation of Qualified Small Business Stock\\nFederal Tax Exclusion Not Always Replicated at State Level\\n\\n\\n17 Trends in VC-Backed Company M&A Deal Terms\\n\\n\\n18 Trends in Convertible Note and SAFE Terms\\n\\n\\n19 Trends in Venture Capital Financing Terms'), start_index=807, text='【7†source】', type='file_citation')], value='The content architecture of the 2023 Venture Capital Report includes the following sections:\\n\\n1. US Market Review and Outlook\\n2. Regional Market Review and Outlook\\n3. Selected WilmerHale Venture Capital Financings\\n4. New Law Requires Federal Reporting of Private Company Ownership (Many Startups and Life Sciences Companies Will be Subject to Beneficial Ownership Reporting)\\n5. Show Me the Money (What Employers Need to Know About New Salary Disclosure Laws)\\n6. Navigating the Quiet Period Shoals (Safe Harbors Aid Compliance With Quiet Period Requirements)\\n7. State Taxation of Qualified Small Business Stock (Federal Tax Exclusion Not Always Replicated at State Level)\\n8. Trends in VC-Backed Company M&A Deal Terms\\n9. Trends in Convertible Note and SAFE Terms\\n10. Trends in Venture Capital Financing Terms【7†source】.'), type='text')], created_at=1700171861, file_ids=[], metadata={}, object='thread.message', role='assistant', run_id='run_uGwTGAemL5rqFwWlHc02Wxwl', thread_id='thread_PIN1poM9aRX4sYjnnohlvZNa'),\n",
672 | " ThreadMessage(id='msg_r00GeAZ3q4zaQYxaYlGsjMmd', assistant_id=None, content=[MessageContentText(text=Text(annotations=[], value='Please show me the content architecture of this report'), type='text')], created_at=1700171844, file_ids=[], metadata={}, object='thread.message', role='user', run_id=None, thread_id='thread_PIN1poM9aRX4sYjnnohlvZNa')]"
673 | ]
674 | },
675 | "metadata": {},
676 | "execution_count": 140
677 | }
678 | ]
679 | },
680 | {
681 | "cell_type": "code",
682 | "source": [
683 | "print(thread_messages.data[0].content[0].text.value)"
684 | ],
685 | "metadata": {
686 | "colab": {
687 | "base_uri": "https://localhost:8080/"
688 | },
689 | "id": "cxVMc18Tem5u",
690 | "outputId": "51201c38-9374-4157-c38d-7ae718b87a77"
691 | },
692 | "execution_count": 141,
693 | "outputs": [
694 | {
695 | "output_type": "stream",
696 | "name": "stdout",
697 | "text": [
698 | "The content architecture of the 2023 Venture Capital Report includes the following sections:\n",
699 | "\n",
700 | "1. US Market Review and Outlook\n",
701 | "2. Regional Market Review and Outlook\n",
702 | "3. Selected WilmerHale Venture Capital Financings\n",
703 | "4. New Law Requires Federal Reporting of Private Company Ownership (Many Startups and Life Sciences Companies Will be Subject to Beneficial Ownership Reporting)\n",
704 | "5. Show Me the Money (What Employers Need to Know About New Salary Disclosure Laws)\n",
705 | "6. Navigating the Quiet Period Shoals (Safe Harbors Aid Compliance With Quiet Period Requirements)\n",
706 | "7. State Taxation of Qualified Small Business Stock (Federal Tax Exclusion Not Always Replicated at State Level)\n",
707 | "8. Trends in VC-Backed Company M&A Deal Terms\n",
708 | "9. Trends in Convertible Note and SAFE Terms\n",
709 | "10. Trends in Venture Capital Financing Terms【7†source】.\n"
710 | ]
711 | }
712 | ]
713 | }
714 | ]
715 | }
--------------------------------------------------------------------------------
/openai_meta_prompt.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "nbformat": 4,
3 | "nbformat_minor": 0,
4 | "metadata": {
5 | "colab": {
6 | "provenance": [],
7 | "authorship_tag": "ABX9TyP45+rZjFXKdMVZSc80F91m",
8 | "include_colab_link": true
9 | },
10 | "kernelspec": {
11 | "name": "python3",
12 | "display_name": "Python 3"
13 | },
14 | "language_info": {
15 | "name": "python"
16 | }
17 | },
18 | "cells": [
19 | {
20 | "cell_type": "markdown",
21 | "metadata": {
22 | "id": "view-in-github",
23 | "colab_type": "text"
24 | },
25 | "source": [
26 | ""
27 | ]
28 | },
29 | {
30 | "cell_type": "code",
31 | "execution_count": 30,
32 | "metadata": {
33 | "id": "l4_Mh6q4lmNo"
34 | },
35 | "outputs": [],
36 | "source": [
37 | "!pip install openai -qU"
38 | ]
39 | },
40 | {
41 | "cell_type": "code",
42 | "source": [
43 | "from google.colab import userdata\n",
44 | "OPENAI_API_KEY = userdata.get('OPENAI_API_KEY')"
45 | ],
46 | "metadata": {
47 | "id": "89_EbmbBlt_W"
48 | },
49 | "execution_count": 31,
50 | "outputs": []
51 | },
52 | {
53 | "cell_type": "code",
54 | "source": [
55 | "from openai import OpenAI\n",
56 | "\n",
57 | "client = OpenAI(api_key=OPENAI_API_KEY)\n",
58 | "\n",
59 | "META_PROMPT = \"\"\"\n",
60 | "Given a task description or existing prompt, produce a detailed system prompt to guide a language model in completing the task effectively.\n",
61 | "\n",
62 | "# Guidelines\n",
63 | "\n",
64 | "- Understand the Task: Grasp the main objective, goals, requirements, constraints, and expected output.\n",
65 | "- Minimal Changes: If an existing prompt is provided, improve it only if it's simple. For complex prompts, enhance clarity and add missing elements without altering the original structure.\n",
66 | "- Reasoning Before Conclusions**: Encourage reasoning steps before any conclusions are reached. ATTENTION! If the user provides examples where the reasoning happens afterward, REVERSE the order! NEVER START EXAMPLES WITH CONCLUSIONS!\n",
67 | " - Reasoning Order: Call out reasoning portions of the prompt and conclusion parts (specific fields by name). For each, determine the ORDER in which this is done, and whether it needs to be reversed.\n",
68 | " - Conclusion, classifications, or results should ALWAYS appear last.\n",
69 | "- Examples: Include high-quality examples if helpful, using placeholders [in brackets] for complex elements.\n",
70 | " - What kinds of examples may need to be included, how many, and whether they are complex enough to benefit from placeholders.\n",
71 | "- Clarity and Conciseness: Use clear, specific language. Avoid unnecessary instructions or bland statements.\n",
72 | "- Formatting: Use markdown features for readability. DO NOT USE ``` CODE BLOCKS UNLESS SPECIFICALLY REQUESTED.\n",
73 | "- Preserve User Content: If the input task or prompt includes extensive guidelines or examples, preserve them entirely, or as closely as possible. If they are vague, consider breaking down into sub-steps. Keep any details, guidelines, examples, variables, or placeholders provided by the user.\n",
74 | "- Constants: DO include constants in the prompt, as they are not susceptible to prompt injection. Such as guides, rubrics, and examples.\n",
75 | "- Output Format: Explicitly the most appropriate output format, in detail. This should include length and syntax (e.g. short sentence, paragraph, JSON, etc.)\n",
76 | " - For tasks outputting well-defined or structured data (classification, JSON, etc.) bias toward outputting a JSON.\n",
77 | " - JSON should never be wrapped in code blocks (```) unless explicitly requested.\n",
78 | "\n",
79 | "The final prompt you output should adhere to the following structure below. Do not include any additional commentary, only output the completed system prompt. SPECIFICALLY, do not include any additional messages at the start or end of the prompt. (e.g. no \"---\")\n",
80 | "\n",
81 | "[Concise instruction describing the task - this should be the first line in the prompt, no section header]\n",
82 | "\n",
83 | "[Additional details as needed.]\n",
84 | "\n",
85 | "[Optional sections with headings or bullet points for detailed steps.]\n",
86 | "\n",
87 | "# Steps [optional]\n",
88 | "\n",
89 | "[optional: a detailed breakdown of the steps necessary to accomplish the task]\n",
90 | "\n",
91 | "# Output Format\n",
92 | "\n",
93 | "[Specifically call out how the output should be formatted, be it response length, structure e.g. JSON, markdown, etc]\n",
94 | "\n",
95 | "# Examples [optional]\n",
96 | "\n",
97 | "[Optional: 1-3 well-defined examples with placeholders if necessary. Clearly mark where examples start and end, and what the input and output are. User placeholders as necessary.]\n",
98 | "[If the examples are shorter than what a realistic example is expected to be, make a reference with () explaining how real examples should be longer / shorter / different. AND USE PLACEHOLDERS! ]\n",
99 | "\n",
100 | "# Notes [optional]\n",
101 | "\n",
102 | "[optional: edge cases, details, and an area to call or repeat out specific important considerations]\n",
103 | "\"\"\".strip()\n",
104 | "\n",
105 | "def generate_prompt(task_or_prompt: str):\n",
106 | " completion = client.chat.completions.create(\n",
107 | " model=\"gpt-4o\",\n",
108 | " messages=[\n",
109 | " {\n",
110 | " \"role\": \"system\",\n",
111 | " \"content\": META_PROMPT,\n",
112 | " },\n",
113 | " {\n",
114 | " \"role\": \"user\",\n",
115 | " \"content\": \"Task, Goal, or Current Prompt:\\n\" + task_or_prompt,\n",
116 | " },\n",
117 | " ],\n",
118 | " )\n",
119 | "\n",
120 | " return completion.choices[0].message.content"
121 | ],
122 | "metadata": {
123 | "id": "PytAbZNslrQx"
124 | },
125 | "execution_count": 32,
126 | "outputs": []
127 | },
128 | {
129 | "cell_type": "code",
130 | "source": [
131 | "prompt = generate_prompt(\"Optimize the prompt that users enter for image generation with Flux 1.1 model\")"
132 | ],
133 | "metadata": {
134 | "id": "GyNBRYEImMbD"
135 | },
136 | "execution_count": 33,
137 | "outputs": []
138 | },
139 | {
140 | "cell_type": "code",
141 | "source": [
142 | "from IPython.display import Markdown, display\n",
143 | "\n",
144 | "def print_md(string):\n",
145 | " display(Markdown(string))"
146 | ],
147 | "metadata": {
148 | "id": "TlNRtAJe3Nww"
149 | },
150 | "execution_count": 35,
151 | "outputs": []
152 | },
153 | {
154 | "cell_type": "code",
155 | "source": [
156 | "print_md(prompt)"
157 | ],
158 | "metadata": {
159 | "colab": {
160 | "base_uri": "https://localhost:8080/",
161 | "height": 558
162 | },
163 | "id": "ua48AmG4mdBK",
164 | "outputId": "0a5c77c5-f3ae-489b-d368-a7f2fb7bb602"
165 | },
166 | "execution_count": 36,
167 | "outputs": [
168 | {
169 | "output_type": "display_data",
170 | "data": {
171 | "text/plain": [
172 | ""
27 | ]
28 | },
29 | {
30 | "cell_type": "markdown",
31 | "source": [
32 | "# PandasAI\n",
33 | "\n",
34 | "PandasAI is a Python library that makes it easy to ask questions to your data (CSV, XLSX, PostgreSQL, MySQL, BigQuery, Databrick, Snowflake, etc.) in natural language.\n",
35 | "\n",
36 | "It helps you to explore, clean, and analyze your data using generative AI.\n",
37 | "\n",
38 | "GitHub: [https://github.com/Sinaptik-AI/pandas-ai](https://github.com/Sinaptik-AI/pandas-ai)\n",
39 | "\n",
40 | "Request Your Panda API Key: [https://www.pandabi.ai](https://www.pandabi.ai)\n"
41 | ],
42 | "metadata": {
43 | "id": "ZGfZ9MyAsWJ3"
44 | }
45 | },
46 | {
47 | "cell_type": "code",
48 | "execution_count": 53,
49 | "metadata": {
50 | "colab": {
51 | "base_uri": "https://localhost:8080/"
52 | },
53 | "id": "JG-2haLbZuDt",
54 | "outputId": "6cafa7b1-3725-419b-fc59-2bc67c5c592f"
55 | },
56 | "outputs": [
57 | {
58 | "output_type": "stream",
59 | "name": "stdout",
60 | "text": [
61 | "Requirement already satisfied: pandasai in /usr/local/lib/python3.10/dist-packages (2.0.30)\n",
62 | "Requirement already satisfied: astor<0.9.0,>=0.8.1 in /usr/local/lib/python3.10/dist-packages (from pandasai) (0.8.1)\n",
63 | "Requirement already satisfied: duckdb<1 in /usr/local/lib/python3.10/dist-packages (from pandasai) (0.9.2)\n",
64 | "Requirement already satisfied: faker<20.0.0,>=19.12.0 in /usr/local/lib/python3.10/dist-packages (from pandasai) (19.13.0)\n",
65 | "Requirement already satisfied: jinja2<4.0.0,>=3.1.3 in /usr/local/lib/python3.10/dist-packages (from pandasai) (3.1.3)\n",
66 | "Requirement already satisfied: matplotlib<4.0.0,>=3.7.1 in /usr/local/lib/python3.10/dist-packages (from pandasai) (3.7.1)\n",
67 | "Requirement already satisfied: openai<2 in /usr/local/lib/python3.10/dist-packages (from pandasai) (1.16.2)\n",
68 | "Requirement already satisfied: pandas==1.5.3 in /usr/local/lib/python3.10/dist-packages (from pandasai) (1.5.3)\n",
69 | "Requirement already satisfied: pillow<11.0.0,>=10.1.0 in /usr/local/lib/python3.10/dist-packages (from pandasai) (10.3.0)\n",
70 | "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from pandasai) (2.6.4)\n",
71 | "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from pandasai) (1.0.1)\n",
72 | "Requirement already satisfied: requests<3.0.0,>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from pandasai) (2.31.0)\n",
73 | "Requirement already satisfied: scipy<2.0.0,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from pandasai) (1.11.4)\n",
74 | "Requirement already satisfied: sqlalchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from pandasai) (2.0.29)\n",
75 | "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas==1.5.3->pandasai) (2.8.2)\n",
76 | "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas==1.5.3->pandasai) (2023.4)\n",
77 | "Requirement already satisfied: numpy>=1.21.0 in /usr/local/lib/python3.10/dist-packages (from pandas==1.5.3->pandasai) (1.25.2)\n",
78 | "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2<4.0.0,>=3.1.3->pandasai) (2.1.5)\n",
79 | "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4.0.0,>=3.7.1->pandasai) (1.2.1)\n",
80 | "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4.0.0,>=3.7.1->pandasai) (0.12.1)\n",
81 | "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4.0.0,>=3.7.1->pandasai) (4.50.0)\n",
82 | "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4.0.0,>=3.7.1->pandasai) (1.4.5)\n",
83 | "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4.0.0,>=3.7.1->pandasai) (23.2)\n",
84 | "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib<4.0.0,>=3.7.1->pandasai) (3.1.2)\n",
85 | "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai<2->pandasai) (3.7.1)\n",
86 | "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai<2->pandasai) (1.7.0)\n",
87 | "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai<2->pandasai) (0.27.0)\n",
88 | "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai<2->pandasai) (1.3.1)\n",
89 | "Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai<2->pandasai) (4.66.2)\n",
90 | "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from openai<2->pandasai) (4.10.0)\n",
91 | "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->pandasai) (0.6.0)\n",
92 | "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->pandasai) (2.16.3)\n",
93 | "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.31.0->pandasai) (3.3.2)\n",
94 | "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.31.0->pandasai) (3.6)\n",
95 | "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.31.0->pandasai) (2.0.7)\n",
96 | "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.31.0->pandasai) (2024.2.2)\n",
97 | "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from sqlalchemy<3,>=1.4->pandasai) (3.0.3)\n",
98 | "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai<2->pandasai) (1.2.0)\n",
99 | "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai<2->pandasai) (1.0.5)\n",
100 | "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai<2->pandasai) (0.14.0)\n",
101 | "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas==1.5.3->pandasai) (1.16.0)\n"
102 | ]
103 | }
104 | ],
105 | "source": [
106 | "!pip install pandasai -U"
107 | ]
108 | },
109 | {
110 | "cell_type": "code",
111 | "source": [
112 | "\n",
113 | "import os\n",
114 | "from pandasai import SmartDataframe\n",
115 | "\n",
116 | "# By default, unless you choose a different LLM, it will use BambooLLM.\n",
117 | "# You can get your free API key signing up at https://pandabi.ai (you can also configure it in your .env file)\n",
118 | "\n",
119 | "from google.colab import userdata\n",
120 | "\n",
121 | "os.environ[\"PANDASAI_API_KEY\"] = userdata.get('PANDASAI_API_KEY')\n",
122 | "\n",
123 | "# You can instantiate a SmartDataframe with a path to a CSV file\n",
124 | "sdf = SmartDataframe(\"Yulu.csv\", config={\"verbose\": True})\n",
125 | "\n",
126 | "response = sdf.chat(\"How many seasons are included in this dataset?\")\n",
127 | "print(response)"
128 | ],
129 | "metadata": {
130 | "colab": {
131 | "base_uri": "https://localhost:8080/"
132 | },
133 | "id": "CLNhtpiqjvJR",
134 | "outputId": "6f6163d3-4661-4370-9174-7bf88fbb345f"
135 | },
136 | "execution_count": 54,
137 | "outputs": [
138 | {
139 | "output_type": "stream",
140 | "name": "stdout",
141 | "text": [
142 | "The number of seasons included in this dataset is: 4.\n"
143 | ]
144 | }
145 | ]
146 | },
147 | {
148 | "cell_type": "code",
149 | "source": [
150 | "print(sdf.last_code_executed)"
151 | ],
152 | "metadata": {
153 | "colab": {
154 | "base_uri": "https://localhost:8080/"
155 | },
156 | "id": "oim8y5JaooiE",
157 | "outputId": "bb858520-a408-4a1d-870b-d25782709663"
158 | },
159 | "execution_count": 56,
160 | "outputs": [
161 | {
162 | "output_type": "stream",
163 | "name": "stdout",
164 | "text": [
165 | "seasons_count = dfs[0]['season'].nunique()\n",
166 | "result = {'type': 'string', 'value': f'The number of seasons included in this dataset is: {seasons_count}.'}\n"
167 | ]
168 | }
169 | ]
170 | },
171 | {
172 | "cell_type": "code",
173 | "source": [
174 | "response = sdf.chat(\"Plot the comparison between registered and casual users on seasons level in 2011\")\n",
175 | "print(response)"
176 | ],
177 | "metadata": {
178 | "colab": {
179 | "base_uri": "https://localhost:8080/",
180 | "height": 582
181 | },
182 | "id": "a4DJIycrkhHd",
183 | "outputId": "2d902dbc-bdad-4f19-b41e-4b5c6b769c6b"
184 | },
185 | "execution_count": 57,
186 | "outputs": [
187 | {
188 | "output_type": "display_data",
189 | "data": {
190 | "text/plain": [
191 | "