├── .github
├── CODEOWNERS
└── workflows
│ ├── project.yml
│ ├── publish_cookbook.yml
│ └── verify_index.yml
├── README.md
├── data
├── multilingual_rag_podcast
│ └── podcast_transcript_whisper_small.txt
├── prometheus2_evaluation
│ └── gemma_2b_rag_results.json
└── rag-with-nims
│ ├── ChipNeMo.pdf
│ ├── embeddings-1.png
│ └── embeddings-2.png
├── index.toml
├── notebooks
├── agent_with_human_in_the_loop.ipynb
├── amazon_bedrock_for_documentation_qa.ipynb
├── amazon_sagemaker_and_chroma_for_qa.ipynb
├── apify_haystack_instagram_comments_analysis.ipynb
├── apify_haystack_rag.ipynb
├── apify_haystack_rag_web_browser.ipynb
├── astradb_haystack_integration.ipynb
├── async_pipeline.ipynb
├── auto_merging_retriever.ipynb
├── charactercodex_llamafile.ipynb
├── chat_with_SQL_3_ways.ipynb
├── chroma-indexing-and-rag-examples.ipynb
├── cohere-v3-for-multilingual-qa.ipynb
├── conversational_rag_using_memory.ipynb
├── evaluating_ai_with_haystack.ipynb
├── extracting_metadata_filters_from_a_user_query.ipynb
├── feedback-analysis-agent-with-AzureAISearch.ipynb
├── function_calling_with_OpenAIChatGenerator.ipynb
├── gemma_chat_rag.ipynb
├── github_issue_resolver_agent.ipynb
├── hackernews-custom-component-rag.ipynb
├── haystack-1.x
│ ├── cohere-for-multilingual-qa.ipynb
│ ├── gpt4-weaviate-custom-documentation-qa.ipynb
│ └── whisper-and-weaviate-for-youtube-rag.ipynb
├── hybrid_rag_pipeline_with_breakpoints.ipynb
├── hybrid_retrieval_bm42.ipynb
├── improve-retrieval-by-embedding-metadata.ipynb
├── information-extraction-gorilla.ipynb
├── information_extraction_raven.ipynb
├── jina-embeddings-v2-legal-analysis-rag.ipynb
├── keyword-extraction.ipynb
├── llama32_agentic_rag.ipynb
├── llama3_rag.ipynb
├── metadata_enrichment.ipynb
├── metadata_extraction_with_llm_metadata_extractor.ipynb
├── mixtral-8x7b-for-web-qa.ipynb
├── model_explorer_streaming.ipynb
├── multilingual_rag_podcast.ipynb
├── multimodal_agent_with_fastrag_haystack.ipynb
├── multimodal_intro.ipynb
├── newsletter-agent.ipynb
├── openapitool.ipynb
├── prometheus2_evaluation.ipynb
├── prompt_customization_for_Anthropic.ipynb
├── prompt_optimization_with_dspy.ipynb
├── query-expansion.ipynb
├── query_decomposition.ipynb
├── rag-with-nims.ipynb
├── rag_eval_deep_eval.ipynb
├── rag_eval_ragas.ipynb
├── rag_eval_uptrain.ipynb
├── rag_fastembed.ipynb
├── sparse_embedding_retrieval.ipynb
├── swarm.ipynb
├── techcrunch_news_digest_titanml_takeoff.ipynb
├── tools_support.ipynb
├── using_hyde_for_improved_retrieval.ipynb
├── using_speaker_diarization_with_assemblyai.ipynb
├── vertexai-gemini-examples.ipynb
├── vllm_inference_engine.ipynb
├── web_enhanced_self_reflecting_agent.ipynb
└── zephyr-7b-beta-for-rag.ipynb
├── requirements.txt
└── scripts
├── generate_markdowns.py
└── verify_index.py
/.github/CODEOWNERS:
--------------------------------------------------------------------------------
1 | # See https://help.github.com/articles/about-codeowners/ for syntax
2 |
3 | # DevRel Team will be the default owner for everything
4 | # in the repo. Unless a later match takes precedence,
5 | # @deepset-ai/devrel will be requested for review
6 | # when someone opens a pull request.
7 | * @deepset-ai/devrel
8 |
--------------------------------------------------------------------------------
/.github/workflows/project.yml:
--------------------------------------------------------------------------------
1 | name: Track issues with Github project
2 |
3 | on:
4 | issues:
5 | types:
6 | - opened
7 |
8 | jobs:
9 | add-to-project:
10 | name: Add new issues to project for triage
11 | runs-on: ubuntu-latest
12 | steps:
13 | - uses: actions/add-to-project@v1.0.2
14 | with:
15 | project-url: https://github.com/orgs/deepset-ai/projects/5
16 | github-token: ${{ secrets.GH_PROJECT_PAT }}
17 |
--------------------------------------------------------------------------------
/.github/workflows/publish_cookbook.yml:
--------------------------------------------------------------------------------
1 | name: Publish cookbook on Haystack Home
2 |
3 | on:
4 | workflow_dispatch:
5 | push:
6 | branches:
7 | - main
8 | paths:
9 | - "notebooks/*.ipynb"
10 | - "index.toml"
11 |
12 | jobs:
13 | publish-cookbook:
14 | runs-on: ubuntu-latest
15 |
16 | steps:
17 | - name: trigger-hook
18 | run: |
19 | curl -X POST ${{ secrets.VERCEL_DEPLOY_HOOK }}
20 |
--------------------------------------------------------------------------------
/.github/workflows/verify_index.yml:
--------------------------------------------------------------------------------
1 | name: Verify index
2 |
3 | on:
4 | pull_request:
5 | paths:
6 | - "notebook/*.ipynb"
7 | - "index.toml"
8 |
9 | jobs:
10 | verify-index:
11 | runs-on: ubuntu-latest
12 |
13 | steps:
14 | - uses: actions/checkout@v4
15 |
16 | - uses: actions/setup-python@v5
17 | with:
18 | python-version: "3.12"
19 |
20 | - name: Verify index content
21 | run: python scripts/verify_index.py
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # 👩🏻🍳 Haystack Cookbook
2 |
3 |
4 |
5 |
![](\"https://docs.vllm.ai/en/latest/_images/vllm-logo-text-light.png\")
\n",
12 | "\n",
13 | "*Notebook by [Stefano Fiorucci](https://github.com/anakin87)*\n",
14 | "\n",
15 | "This notebook shows how to use the [vLLM inference engine](https://docs.vllm.ai/en/latest/) with Haystack."
16 | ]
17 | },
18 | {
19 | "cell_type": "markdown",
20 | "metadata": {
21 | "id": "t9GskGf6wGgE"
22 | },
23 | "source": [
24 | "## Install vLLM + Haystack\n",
25 | "\n",
26 | "- we install vLLM using pip ([docs](https://docs.vllm.ai/en/latest/getting_started/installation.html))\n",
27 | "- for production use cases, there are many other options, including Docker ([docs](https://docs.vllm.ai/en/latest/serving/deploying_with_docker.html))"
28 | ]
29 | },
30 | {
31 | "cell_type": "code",
32 | "execution_count": 12,
33 | "metadata": {
34 | "colab": {
35 | "base_uri": "https://localhost:8080/"
36 | },
37 | "id": "46yIpiO_b16C",
38 | "outputId": "5b451759-19e6-4381-a588-e40face7563c"
39 | },
40 | "outputs": [
41 | {
42 | "name": "stdout",
43 | "output_type": "stream",
44 | "text": [
45 | "nvcc: NVIDIA (R) Cuda compiler driver\n",
46 | "Copyright (c) 2005-2023 NVIDIA Corporation\n",
47 | "Built on Tue_Aug_15_22:02:13_PDT_2023\n",
48 | "Cuda compilation tools, release 12.2, V12.2.140\n",
49 | "Build cuda_12.2.r12.2/compiler.33191640_0\n"
50 | ]
51 | }
52 | ],
53 | "source": [
54 | "# we check that CUDA is >=12.1 (https://docs.vllm.ai/en/latest/getting_started/installation.html#install-with-pip)\n",
55 | "! nvcc --version"
56 | ]
57 | },
58 | {
59 | "cell_type": "code",
60 | "execution_count": null,
61 | "metadata": {
62 | "colab": {
63 | "base_uri": "https://localhost:8080/"
64 | },
65 | "id": "RXZsGS9YbUr-",
66 | "outputId": "eedf455a-cbe1-477c-b62d-51b832d9e8cc"
67 | },
68 | "outputs": [],
69 | "source": [
70 | "! pip install vllm haystack-ai"
71 | ]
72 | },
73 | {
74 | "cell_type": "markdown",
75 | "metadata": {
76 | "id": "qt__RIekwTX1"
77 | },
78 | "source": [
79 | "## Run a vLLM OpenAI-compatible server in Colab\n",
80 | "\n",
81 | "vLLM can be deployed as a server that implements the OpenAI API protocol. This allows vLLM to be used as a drop-in replacement for applications using OpenAI API. Read more [in the docs](https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server).\n",
82 | "\n",
83 | "In Colab, we start the OpenAI-compatible server using Python.\n",
84 | "For environments that support Docker, we can run the server using Docker ([docs](https://docs.vllm.ai/en/latest/serving/deploying_with_docker.html)).\n",
85 | "\n",
86 | "*Significant parameters:*\n",
87 | "- **model**: [TheBloke/notus-7B-v1-AWQ](https://huggingface.co/TheBloke/notus-7B-v1-AWQ) is the AWQ quantized version of a good LLM by Argilla. Several model architectures are supported; models are automatically downloaded from Hugging Face as needed. For a comprehensive list of the supported models, see the [docs](https://docs.vllm.ai/en/latest/models/supported_models.html).\n",
88 | "\n",
89 | "- **quantization**: awq. AWQ is a quantization method that allows LLMs to run (fast) when GPU resources are limited. [Simple blogpost on quantization techniques](https://www.maartengrootendorst.com/blog/quantization/#awq-activation-aware-weight-quantization)\n",
90 | "- **max_model_len**: we specify a [maximum context length](https://docs.vllm.ai/en/latest/models/engine_args.html), which consists of the maximum number of tokens (prompt + response). Otherwise, the model does not fit in Colab and we get an OOM error.\n"
91 | ]
92 | },
93 | {
94 | "cell_type": "code",
95 | "execution_count": 1,
96 | "metadata": {
97 | "colab": {
98 | "base_uri": "https://localhost:8080/"
99 | },
100 | "id": "wqceUZ4yj2IT",
101 | "outputId": "f94b1e26-42d7-4362-d5f1-d9c6ce0a5968"
102 | },
103 | "outputs": [
104 | {
105 | "name": "stdout",
106 | "output_type": "stream",
107 | "text": [
108 | "nohup: redirecting stderr to stdout\n"
109 | ]
110 | }
111 | ],
112 | "source": [
113 | "# we prepend \"nohup\" and postpend \"&\" to make the Colab cell run in background\n",
114 | "! nohup python -m vllm.entrypoints.openai.api_server \\\n",
115 | " --model TheBloke/notus-7B-v1-AWQ \\\n",
116 | " --quantization awq \\\n",
117 | " --max-model-len 2048 \\\n",
118 | " > vllm.log &"
119 | ]
120 | },
121 | {
122 | "cell_type": "code",
123 | "execution_count": 2,
124 | "metadata": {
125 | "colab": {
126 | "base_uri": "https://localhost:8080/"
127 | },
128 | "id": "obZ83mNDp2M3",
129 | "outputId": "247812a7-ea57-475c-d3b0-817f2d9f0589"
130 | },
131 | "outputs": [
132 | {
133 | "name": "stdout",
134 | "output_type": "stream",
135 | "text": [
136 | "INFO 02-16 10:57:39 llm_engine.py:72] Initializing an LLM engine with config: model='TheBloke/notus-7B-v1-AWQ', tokenizer='TheBloke/notus-7B-v1-AWQ', tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.float16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, disable_custom_all_reduce=False, quantization=awq, enforce_eager=False, kv_cache_dtype=auto, seed=0)\n",
137 | "INFO 02-16 10:57:43 weight_utils.py:164] Using model weights format ['*.safetensors']\n",
138 | "INFO 02-16 10:57:43 weight_utils.py:164] Using model weights format ['*.safetensors']\n",
139 | "INFO 02-16 10:57:55 llm_engine.py:322] # GPU blocks: 4108, # CPU blocks: 2048\n",
140 | "INFO 02-16 10:57:58 model_runner.py:636] CUDA graphs can take additional 1~3 GiB memory per GPU. If you are running out of memory, consider decreasing `gpu_memory_utilization` or enforcing eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage.\n",
141 | "INFO 02-16 10:57:58 model_runner.py:636] CUDA graphs can take additional 1~3 GiB memory per GPU. If you are running out of memory, consider decreasing `gpu_memory_utilization` or enforcing eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage.\n",
142 | "INFO 02-16 10:57:58 model_runner.py:636] CUDA graphs can take additional 1~3 GiB memory per GPU. If you are running out of memory, consider decreasing `gpu_memory_utilization` or enforcing eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage.\n"
143 | ]
144 | }
145 | ],
146 | "source": [
147 | "# we check the logs until the server has been started correctly\n",
148 | "!while ! grep -q \"Application startup complete\" vllm.log; do tail -n 1 vllm.log; sleep 5; done"
149 | ]
150 | },
151 | {
152 | "cell_type": "markdown",
153 | "metadata": {
154 | "id": "MgQbgCcA1jQ6"
155 | },
156 | "source": [
157 | "## Chat with the model using OpenAIChatGenerator\n",
158 | "\n",
159 | "Once we have launched the vLLM-compatible OpenAI server,\n",
160 | "we can simply initialize an `OpenAIChatGenerator` pointing to the vLLM server URL and start chatting!"
161 | ]
162 | },
163 | {
164 | "cell_type": "code",
165 | "execution_count": 3,
166 | "metadata": {
167 | "id": "ypVbte5DrkWO"
168 | },
169 | "outputs": [],
170 | "source": [
171 | "from haystack.components.generators.chat import OpenAIChatGenerator\n",
172 | "from haystack.dataclasses import ChatMessage\n",
173 | "from haystack.utils import Secret\n",
174 | "\n",
175 | "generator = OpenAIChatGenerator(\n",
176 | " api_key=Secret.from_token(\"VLLM-PLACEHOLDER-API-KEY\"), # for compatibility with the OpenAI API, a placeholder api_key is needed\n",
177 | " model=\"TheBloke/notus-7B-v1-AWQ\",\n",
178 | " api_base_url=\"http://localhost:8000/v1\",\n",
179 | " generation_kwargs = {\"max_tokens\": 512}\n",
180 | ")"
181 | ]
182 | },
183 | {
184 | "cell_type": "code",
185 | "execution_count": 17,
186 | "metadata": {
187 | "colab": {
188 | "base_uri": "https://localhost:8080/"
189 | },
190 | "id": "45xJjlEoCB7v",
191 | "outputId": "7f939707-5b25-4d3f-fefc-0368d15f236a"
192 | },
193 | "outputs": [
194 | {
195 | "name": "stdout",
196 | "output_type": "stream",
197 | "text": [
198 | "Enter your message or Q to exit\n",
199 | "🧑 hello. can you help planning my next travel to Italy?\n",
200 | "🤖 Certainly! I'd be happy to help you plan your next trip to Italy. Here are some steps to help you plan your trip:\n",
201 | "\n",
202 | "1. Determine your travel dates: Decide when you want to travel to Italy. Keep in mind that peak season is from June to August, so prices may be higher, and crowds may be larger.\n",
203 | "\n",
204 | "2. Decide on your destination: Italy is a large country with many beautiful destinations. Consider which cities and regions you would like to visit. Some popular destinations include Rome, Florence, Venice, Amalfi Coast, Tuscany, and the Italian Lakes.\n",
205 | "\n",
206 | "3. Research flights and transportation: Look for flights that fit your budget and travel dates. If you're planning on traveling between cities, research trains and buses. Familiarize yourself with the transportation options in your destination cities.\n",
207 | "\n",
208 | "4. Consider accommodation: Research different types of accommodations, such as hotels, vacation rentals, or hostels. Look for a place that fits your budget and travel style.\n",
209 | "\n",
210 | "5. Plan your itinerary: Make a list of the things you want to see and do in each destination. Consider how much time you want to spend in each location.\n",
211 | "\n",
212 | "6. Research tours and activities: Look for tours and activities that interest you. This could include food tours, wine tastings, museum visits, or tours of historic sites.\n",
213 | "\n",
214 | "7. Learn some Italian: It's always a good idea to learn some Italian before your trip, even if it's just basic phrases. This will help you communicate with locals and make your trip more enjoyable.\n",
215 | "\n",
216 | "8. Pack appropriately: Make sure to pack appropriately for the season and destination. Check the weather forecast before you pack.\n",
217 | "\n",
218 | "9. Exchange currency: Research the best place to exchange currency in your destination. Also, check if your bank or credit card offers no-fee withdrawals.\n",
219 | "\n",
220 | "10. Purchase travel insurance: Consider purchasing travel insurance to protect yourself in case of unforeseen circumstances, like a cancelled flight or lost luggage.\n",
221 | "\n",
222 | "I hope these steps help you plan your next trip to Italy! Let me know if you have any questions or if you need further assistance.\n",
223 | "Enter your message or Q to exit\n",
224 | "🧑 Please list 5 places I should absolutely visit\n",
225 | "🤖 Certainly! Here are 5 places in Italy that I think should be on your must-visit list:\n",
226 | "\n",
227 | "1. Rome - The Eternal City is home to ancient ruins like the Colosseum and the Roman Forum, as well as iconic landmarks like the Vatican City, the Pantheon, and the Trevi Fountain. Rome is also known for its delicious food and vibrant nightlife.\n",
228 | "\n",
229 | "2. Florence - Known as the birthplace of the Renaissance, Florence is home to incredible art and architecture, including the Uffizi Gallery, the Duomo, and the Palazzo Vecchio. Be sure to try some of Florence's famous gelato while you're there!\n",
230 | "\n",
231 | "3. Venice - A city built on water, Venice is known for its canals, palaces, and bridges. Don't miss a gondola ride through the city's canals or a visit to St. Mark's Basilica and the Doge's Palace.\n",
232 | "\n",
233 | "4. Amalfi Coast - This scenic coastline is home to charming towns like Positano, Ravello, and Amalfi, as well as stunning cliffs, beaches, and seaside restaurants.\n",
234 | "\n",
235 | "5. Cinque Terre - This UNESCO World Heritage Site is a series of five colorful villages on the Italian Riviera. Hike between the villages or take a boat ride to see the beautiful coastline.\n",
236 | "\n",
237 | "These are just a few of the many incredible places to visit in Italy, but I hope this helps you start planning your itinerary!\n",
238 | "Enter your message or Q to exit\n",
239 | "🧑 Nice! Can you expand the fith point?\n",
240 | "🤖 Absolutely! The Cinque Terre, which translates to \"Five Lands,\" is a series of five picturesque villages located along the Italian Riviera. The villages are Monterosso al Mare, Vernazza, Corniglia, Manarola, and Riomaggiore. Here are some must-see highlights of each village:\n",
241 | "\n",
242 | "1. Monterosso al Mare - This village is the largest of the five, and is known for its long sandy beach, medieval castle ruins, and the Church of San Francesco, which offers stunning views from its bell tower.\n",
243 | "\n",
244 | "2. Vernazza - This village is often considered the jewel of the Cinque Terre. Don't miss the stunning views from the top of the castle ruins, and be sure to try some of Vernazza's famous seafood dishes.\n",
245 | "\n",
246 | "3. Corniglia - This village is perched on a cliff and can only be reached by foot or by a narrow stairway. Once you reach the top, you can visit the Church of San Lorenzo, which offers stunning views of the sea.\n",
247 | "\n",
248 | "4. Manarola - This village is known for its colorful houses that cascade down the cliffs, as well as its ancient church, San Carlo, and the Church of St. Lawrence.\n",
249 | "\n",
250 | "5. Riomaggiore - This village is the southernmost of the five and is known for its narrow streets, colorful houses, and seaside restaurants. Don't miss the Church of San Giovanni Battista, which sits at the edge of the village and offers stunning views of the sea.\n",
251 | "\n",
252 | "The best way to experience the Cinque Terre is by hiking between the villages along the Sentiero Azzurro (Blue Trail). This hike offers incredible views of the coastline and the villages below. Be sure to check the weather forecast before you go, as the trail can be challenging in rainy or windy conditions.\n",
253 | "\n",
254 | "I hope this helps you plan your visit to the stunning Cinque Terre!\n",
255 | "Enter your message or Q to exit\n",
256 | "🧑 Q\n"
257 | ]
258 | }
259 | ],
260 | "source": [
261 | "messages = []\n",
262 | "\n",
263 | "while True:\n",
264 | " msg = input(\"Enter your message or Q to exit\\n🧑 \")\n",
265 | " if msg==\"Q\":\n",
266 | " break\n",
267 | " messages.append(ChatMessage.from_user(msg))\n",
268 | " response = generator.run(messages=messages)\n",
269 | " assistant_resp = response['replies'][0]\n",
270 | " print(\"🤖 \"+assistant_resp.text)\n",
271 | " messages.append(assistant_resp)"
272 | ]
273 | }
274 | ],
275 | "metadata": {
276 | "accelerator": "GPU",
277 | "colab": {
278 | "collapsed_sections": [
279 | "t9GskGf6wGgE"
280 | ],
281 | "gpuType": "T4",
282 | "provenance": []
283 | },
284 | "kernelspec": {
285 | "display_name": "Python 3",
286 | "name": "python3"
287 | },
288 | "language_info": {
289 | "name": "python"
290 | }
291 | },
292 | "nbformat": 4,
293 | "nbformat_minor": 0
294 | }
295 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | nbconvert
2 |
--------------------------------------------------------------------------------
/scripts/generate_markdowns.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | from pathlib import Path
3 | from subprocess import check_output
4 | import re
5 |
6 | import tomllib
7 | from nbconvert import MarkdownExporter
8 | from nbconvert.filters.strings import get_lines
9 | from nbformat import read, NO_CONVERT
10 |
11 |
12 | def generate_frontmatter(notebook):
13 | last_commit_date = (
14 | check_output(f'git log -1 --pretty=format:"%cs" {notebook["file"]}'.split())
15 | .decode()
16 | .strip()
17 | )
18 | first_commit_date = (
19 | check_output(
20 | f'git log --reverse --pretty=format:"%cs" {notebook["file"]}'.split()
21 | )
22 | .decode()
23 | .strip()
24 | .splitlines()[0]
25 | )
26 |
27 | frontmatter = f"""---
28 | layout: cookbook
29 | featured_image: /images/cookbook-thumbnail.png
30 | images: ["/images/cookbook-thumbnail.png"]
31 | sitemap_exclude: {notebook.get("sitemap_exclude", False)}
32 | colab: {notebook.get("colab")}
33 | toc: True
34 | title: "{notebook["title"]}"
35 | lastmod: {last_commit_date}
36 | created_at: {first_commit_date}
37 | download: /downloads/{notebook["file"].name}
38 | featured: {notebook.get("featured", False)}
39 | experimental: {notebook.get("experimental", False)}
40 | discuss: {notebook.get("discuss", False)}
41 | hidden: {notebook.get("hidden", False)}
42 | new: {notebook.get("new", False)}
43 | topics: {notebook.get("topics", False)}
44 | ---
45 | """
46 | return frontmatter
47 |
48 |
49 | def clean_colab_dataframe_cells(notebook):
50 | """
51 | When there ara dataframes, Colab notebooks contain text/html data
52 | that include style and script, not properly rendered in markdown.
53 | This function removes this data from the notebook.
54 | """
55 |
56 | for cell in notebook.cells:
57 | if cell.cell_type == "code" and "outputs" in cell:
58 | for output in cell.outputs:
59 | # we recognize this type of data by the key 'application/vnd.google.colaboratory.intrinsic+json'
60 | # and the type 'dataframe'
61 | if 'application/vnd.google.colaboratory.intrinsic+json' in output.get('data', {}):
62 | if output['data']['application/vnd.google.colaboratory.intrinsic+json'].get('type') == 'dataframe':
63 | output['data'].pop('text/html', None)
64 |
65 | return notebook
66 |
67 | def generate_markdown_from_notebook(notebook, output_path):
68 | frontmatter = generate_frontmatter(notebook)
69 | md_exporter = MarkdownExporter(exclude_output=False)
70 |
71 | with open(notebook["file"], "r", encoding="utf-8") as f:
72 | nb = read(f, as_version=NO_CONVERT)
73 |
74 | # process the notebook to clean Colab dataframe cells
75 | cleaned_nb = clean_colab_dataframe_cells(nb)
76 |
77 | body, _ = md_exporter.from_notebook_node(cleaned_nb)
78 | body = get_lines(body, start=1)
79 |
80 | # remove output images from the markdown: they are not handled properly
81 | img_pattern=r'^!\[png\]\(.*\.png\)$'
82 | body = re.sub(img_pattern, '', body, flags=re.MULTILINE)
83 |
84 | print(f"Processing {notebook['file']}")
85 | filename = notebook["file"].stem
86 | with open(f"{output_path}/{filename}.md", "w", encoding="utf-8") as f:
87 | try:
88 | f.write(frontmatter + "\n\n")
89 | except IndexError as e:
90 | raise IndexError(
91 | "Can't find the header for this tutorial. Have you added it in 'scripts/generate_markdowns.py'?"
92 | ) from e
93 | f.write(body)
94 |
95 |
96 | if __name__ == "__main__":
97 | parser = argparse.ArgumentParser()
98 | parser.add_argument("--output", dest="output", default="markdowns")
99 | args = parser.parse_args()
100 | root_path = Path(__file__).parent.parent
101 |
102 | readme_file = root_path / "README.md"
103 | readme_content = readme_file.read_text()
104 |
105 | index_file = root_path / "index.toml"
106 | index_data = tomllib.loads(index_file.read_text())
107 |
108 | if not Path(args.output).exists():
109 | Path(args.output).mkdir(parents=True, exist_ok=True)
110 |
111 | for cookbook_data in index_data["cookbook"]:
112 | data = {
113 | "file": root_path / "notebooks" / cookbook_data["notebook"],
114 | "title": cookbook_data["title"],
115 | "colab": f"{index_data['config']['colab']}/{cookbook_data['notebook']}",
116 | "featured": cookbook_data.get("featured", False),
117 | "experimental": cookbook_data.get("experimental", False),
118 | "discuss": cookbook_data.get("discuss", False),
119 | "hidden": cookbook_data.get("hidden", False),
120 | "new": cookbook_data.get("new", False),
121 | "topics": cookbook_data.get("topics", False),
122 | }
123 | generate_markdown_from_notebook(data, args.output)
--------------------------------------------------------------------------------
/scripts/verify_index.py:
--------------------------------------------------------------------------------
1 | from pathlib import Path
2 |
3 | import tomllib
4 |
5 | if __name__ == "__main__":
6 | root_path = Path(__file__).parent.parent
7 |
8 | index_file = root_path / "index.toml"
9 | index_data = tomllib.loads(index_file.read_text())
10 |
11 | notebooks = list(root_path.glob("notebooks/*.ipynb"))
12 |
13 | failed = False
14 | indexed_notebooks = [
15 | notebook["notebook"]
16 | for notebook in index_data["cookbook"]
17 | if "notebook" in notebook
18 | ]
19 | for notebook in notebooks:
20 | if notebook.name not in indexed_notebooks:
21 | print(f"Notebook '{notebook.name}' not found in 'index.toml'")
22 | failed = True
23 |
24 | for notebook in index_data["cookbook"]:
25 | if "notebook" not in notebook:
26 | print(f"Notebook '{notebook}' has no file")
27 | failed = True
28 | continue
29 |
30 | if not (root_path / f"notebooks/{notebook['notebook']}").exists():
31 | print(f"Notebook '{notebook['notebook']}' file not found")
32 | failed = True
33 |
34 | if "title" not in notebook:
35 | print(f"Notebook '{notebook['notebook']}' has no title")
36 | failed = True
37 |
38 | if not notebook.get("topics"):
39 | print(f"Notebook '{notebook['notebook']}' has no topics")
40 | failed = True
41 |
42 | if failed:
43 | exit(1)
44 |
--------------------------------------------------------------------------------