├── .devcontainer ├── Containerfile └── devcontainer.json ├── .gitattributes ├── .github └── workflows │ ├── README.md │ ├── chatbot.yaml │ ├── codegen.yaml │ ├── instructlab.yaml │ ├── manual_build_trigger.yaml │ ├── mirror_repository.yaml │ ├── model_converter.yaml │ ├── model_servers.yaml │ ├── models.yaml │ ├── object_detection.yaml │ ├── rag.yaml │ ├── summarizer.yaml │ ├── test-trace-steps.yaml │ ├── testing_framework.yaml │ ├── training-e2e.yaml │ └── training_bootc.yaml ├── .gitignore ├── CODEOWNERS ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── ailab-images.md ├── assets ├── ai_lab_recipes_logo.png ├── chatbot_nodejs_ui.png ├── chatbot_ui.png ├── codegen_ui.png ├── function_calling_nodejs_ui.png ├── image_analysis.png ├── install_continue_extension.png ├── model_converter.png ├── object_detection.png ├── rag_nodejs.png ├── rag_ui.png ├── summarizer_ui.png └── whisper.png ├── ci └── trace-steps.py ├── convert_models ├── Containerfile ├── README.md ├── download_huggingface.py ├── requirements.txt ├── run.sh └── ui.py ├── data ├── fake_meeting.pdf ├── fake_meeting.txt └── jfk.wav ├── eval ├── embeddings │ └── custom_eval_set.py └── promptfoo │ ├── README.md │ ├── base │ ├── Containerfile │ └── promptfooconfig.yaml │ └── evals │ └── README.md ├── hooks └── pre-commit ├── install-hooks.sh ├── model_servers ├── common │ └── Makefile.common ├── llamacpp_python │ ├── Makefile │ ├── README.md │ ├── base │ │ └── Containerfile │ ├── cuda │ │ └── Containerfile │ ├── src │ │ ├── requirements.txt │ │ └── run.sh │ ├── tests │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── requirements.txt │ │ └── test_alive.py │ ├── tooling_options.ipynb │ └── vulkan │ │ ├── amd64 │ │ └── Containerfile │ │ └── arm64 │ │ └── Containerfile ├── object_detection_python │ ├── Makefile │ ├── README.md │ ├── base │ │ └── Containerfile │ ├── src │ │ ├── object_detection_server.py │ │ ├── requirements-unlocked.txt │ │ ├── requirements.txt │ │ └── run.sh │ └── tests │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── requirements.txt │ │ └── test_alive.py ├── ollama │ ├── README.md │ └── base │ │ └── Containerfile └── whispercpp │ ├── Makefile │ ├── README.md │ ├── base │ └── Containerfile │ ├── src │ └── run.sh │ └── tests │ ├── __init__.py │ ├── conftest.py │ ├── requirements.txt │ └── test_alive.py ├── models ├── Containerfile ├── Makefile ├── README.md └── download_hf_models.py ├── recipes ├── audio │ └── audio_to_text │ │ ├── Makefile │ │ ├── README.md │ │ ├── ai-lab.yaml │ │ ├── app │ │ ├── Containerfile │ │ ├── requirements.txt │ │ └── whisper_client.py │ │ ├── bootc │ │ ├── Containerfile │ │ ├── Containerfile.nocache │ │ └── README.md │ │ └── quadlet │ │ ├── README.md │ │ ├── audio-to-text.image │ │ ├── audio-to-text.kube │ │ └── audio-to-text.yaml ├── common │ ├── Makefile.common │ ├── README.md │ ├── README_bootc_image_builder.md │ ├── bin │ │ └── .gitkeep │ ├── quadlet │ │ └── app.image │ └── usr │ │ ├── lib │ │ └── systemd │ │ │ └── system │ │ │ ├── bootc-generic-growpart.service │ │ │ └── local-fs.target.wants │ │ │ └── bootc-generic-growpart.service │ │ └── libexec │ │ └── bootc-generic-growpart ├── computer_vision │ ├── object_detection │ │ ├── Makefile │ │ ├── README.md │ │ ├── ai-lab.yaml │ │ └── app │ │ │ ├── Containerfile │ │ │ ├── object_detection_client.py │ │ │ └── requirements.txt │ └── tests │ │ ├── conftest.py │ │ ├── functional │ │ ├── __init__.py │ │ ├── conftest.py │ │ └── test_app.py │ │ ├── integration │ │ ├── __init__.py │ │ ├── conftest.py │ │ └── test_app.py │ │ └── requirements.txt ├── multimodal │ └── image_understanding │ │ ├── README.md │ │ ├── ai-lab.yaml │ │ └── app │ │ ├── Containerfile │ │ ├── image_understanding.py │ │ └── requirements.txt └── natural_language_processing │ ├── agents │ ├── Makefile │ ├── README.md │ ├── ai-lab.yaml │ └── app │ │ ├── Containerfile │ │ ├── react-agent-app.py │ │ └── requirements.txt │ ├── chatbot-java-quarkus │ ├── README.md │ ├── ai-lab.yaml │ └── app │ │ ├── .mvn │ │ └── wrapper │ │ │ └── maven-wrapper.properties │ │ ├── Containerfile │ │ ├── mvnw │ │ ├── mvnw.cmd │ │ ├── pom.xml │ │ └── src │ │ └── main │ │ ├── java │ │ └── io │ │ │ └── quarkiverse │ │ │ └── langchain4j │ │ │ └── sample │ │ │ └── chatbot │ │ │ ├── Bot.java │ │ │ ├── ChatBotWebSocket.java │ │ │ └── ImportmapResource.java │ │ └── resources │ │ ├── META-INF │ │ └── resources │ │ │ ├── components │ │ │ ├── demo-chat.js │ │ │ └── demo-title.js │ │ │ ├── fonts │ │ │ └── red-hat-font.min.css │ │ │ ├── images │ │ │ └── chatbot-architecture.png │ │ │ └── index.html │ │ └── application.properties │ ├── chatbot-nodejs │ ├── Makefile │ ├── README.md │ ├── ai-lab.yaml │ ├── app │ │ ├── .gitignore │ │ ├── Containerfile │ │ ├── app │ │ │ ├── layout.tsx │ │ │ └── page.js │ │ ├── next.config.js │ │ ├── package-lock.json │ │ ├── package.json │ │ └── pages │ │ │ └── api │ │ │ └── socket.io │ │ │ └── index.js │ ├── bootc │ │ ├── Containerfile │ │ ├── Containerfile.nocache │ │ └── README.md │ └── quadlet │ │ ├── README.md │ │ ├── chatbot-nodejs.image │ │ ├── chatbot-nodejs.kube │ │ └── chatbot-nodejs.yaml │ ├── chatbot-pydantic-ai │ ├── Makefile │ ├── README.md │ ├── ai-lab.yaml │ ├── app │ │ ├── Containerfile │ │ ├── chatbot-pydantic-ai.py │ │ └── requirements.txt │ ├── bootc │ │ ├── Containerfile │ │ ├── Containerfile.nocache │ │ └── README.md │ ├── provision │ │ ├── playbook.yml │ │ └── requirements.yml │ └── quadlet │ │ ├── chatbot-pydantic-ai.image │ │ ├── chatbot-pydantic-ai.kube │ │ └── chatbot-pydantic-ai.yaml │ ├── chatbot │ ├── Makefile │ ├── README.md │ ├── ai-lab.yaml │ ├── app │ │ ├── Containerfile │ │ ├── chatbot_ui.py │ │ └── requirements.txt │ ├── bootc │ │ ├── Containerfile │ │ ├── Containerfile.nocache │ │ └── README.md │ ├── provision │ │ ├── playbook.yml │ │ └── requirements.yml │ └── quadlet │ │ ├── README.md │ │ ├── chatbot.image │ │ ├── chatbot.kube │ │ └── chatbot.yaml │ ├── codegen │ ├── Makefile │ ├── README.md │ ├── ai-lab.yaml │ ├── app │ │ ├── Containerfile │ │ ├── codegen-app.py │ │ └── requirements.txt │ ├── bootc │ │ ├── Containerfile │ │ ├── Containerfile.nocache │ │ └── README.md │ ├── llms-vscode-integration.md │ ├── provision │ │ ├── playbook.yml │ │ └── requirements.yml │ └── quadlet │ │ ├── README.md │ │ ├── codegen.image │ │ ├── codegen.kube │ │ └── codegen.yaml │ ├── function-calling-nodejs │ ├── Makefile │ ├── README.md │ ├── ai-lab.yaml │ ├── app │ │ ├── .dockerignore │ │ ├── .gitignore │ │ ├── Containerfile │ │ ├── ai │ │ │ ├── tools │ │ │ │ └── weather.mjs │ │ │ └── weather-prompt.mjs │ │ ├── package-lock.json │ │ ├── package.json │ │ ├── public │ │ │ ├── app.js │ │ │ └── index.html │ │ ├── routes │ │ │ └── temperatures.mjs │ │ └── server.mjs │ ├── bootc │ │ ├── Containerfile │ │ ├── Containerfile.nocache │ │ └── README.md │ └── quadlet │ │ ├── README.md │ │ ├── function-calling-nodejs.image │ │ ├── function-calling-nodejs.kube │ │ └── function-calling-nodejs.yaml │ ├── function_calling │ ├── Makefile │ ├── README.md │ ├── ai-lab.yaml │ ├── app │ │ ├── Containerfile │ │ ├── app.py │ │ └── requirements.txt │ ├── bootc │ │ ├── Containerfile │ │ ├── Containerfile.nocache │ │ └── README.md │ ├── provision │ │ ├── playbook.yml │ │ └── requirements.yml │ └── quadlet │ │ ├── README.md │ │ ├── function_calling.kube │ │ └── function_calling.yaml │ ├── graph-rag │ ├── Makefile │ ├── README.md │ ├── ai-lab.yaml │ ├── app │ │ ├── Containerfile │ │ ├── rag_app.py │ │ └── requirements.txt │ ├── bootc │ │ ├── Containerfile │ │ ├── Containerfile.nocache │ │ └── README.md │ ├── provision │ │ ├── playbook.yaml │ │ └── requirements.yml │ └── quadlet │ │ ├── README.md │ │ ├── graph-rag.image │ │ ├── graph-rag.kube │ │ └── graph-rag.yaml │ ├── rag-nodejs │ ├── Makefile │ ├── README.md │ ├── ai-lab.yaml │ ├── app │ │ ├── .gitignore │ │ ├── Containerfile │ │ ├── app │ │ │ ├── api │ │ │ │ └── upload │ │ │ │ │ └── route.js │ │ │ ├── layout.tsx │ │ │ └── page.js │ │ ├── next-env.d.ts │ │ ├── next.config.js │ │ ├── package-lock.json │ │ ├── package.json │ │ ├── page.js │ │ ├── pages │ │ │ └── api │ │ │ │ ├── delete │ │ │ │ └── index.js │ │ │ │ └── socket.io │ │ │ │ └── index.js │ │ └── tsconfig.json │ ├── bootc │ │ ├── Containerfile │ │ ├── Containerfile.nocache │ │ └── README.md │ ├── quadlet │ │ ├── README.md │ │ ├── rag-nodejs.image │ │ ├── rag-nodejs.kube │ │ └── rag-nodejs.yaml │ └── sample-data │ │ └── fake_meeting.txt │ ├── rag │ ├── Makefile │ ├── README.md │ ├── ai-lab.yaml │ ├── app │ │ ├── Containerfile │ │ ├── manage_vectordb.py │ │ ├── rag_app.py │ │ └── requirements.txt │ ├── bootc │ │ ├── Containerfile │ │ ├── Containerfile.nocache │ │ └── README.md │ ├── provision │ │ ├── playbook.yml │ │ └── requirements.yml │ ├── quadlet │ │ ├── README.md │ │ ├── rag.image │ │ ├── rag.kube │ │ └── rag.yaml │ └── sample-data │ │ └── fake_meeting.txt │ ├── summarizer │ ├── Makefile │ ├── README.md │ ├── ai-lab.yaml │ ├── app │ │ ├── Containerfile │ │ ├── requirements.txt │ │ └── summarizer.py │ ├── bootc │ │ ├── Containerfile │ │ ├── Containerfile.nocache │ │ └── README.md │ ├── provision │ │ ├── playbook.yml │ │ └── requirements.yml │ └── quadlet │ │ ├── README.md │ │ ├── summarizer.image │ │ ├── summarizer.kube │ │ └── summarizer.yaml │ └── tests │ ├── conftest.py │ ├── functional │ ├── __init__.py │ ├── conftest.py │ └── test_app.py │ ├── integration │ ├── __init__.py │ ├── conftest.py │ └── test_app.py │ └── requirements.txt ├── renovate.json ├── requirements-test.txt ├── training ├── Makefile ├── README.md ├── amd-bootc │ ├── Containerfile │ ├── Makefile │ ├── containers-storage.conf │ ├── duplicated │ │ └── ilab-wrapper │ │ │ └── ilab │ └── repos.d │ │ ├── RPM-GPG-KEY-AMD-ROCM │ │ ├── amdgpu.repo │ │ └── rocm.repo ├── cloud │ ├── Containerfile │ ├── Makefile │ ├── README.md │ ├── aws │ │ ├── Makefile.env │ │ ├── README.md │ │ ├── cloud-setup.sh │ │ ├── config.toml │ │ └── files │ │ │ ├── etc │ │ │ ├── X11 │ │ │ │ └── xorg.conf.d │ │ │ │ │ └── 00-keyboard.conf │ │ │ ├── locale.conf │ │ │ ├── localtime │ │ │ ├── systemd │ │ │ │ └── system │ │ │ │ │ ├── NetworkManager.service.wants │ │ │ │ │ └── nm-cloud-setup.service │ │ │ │ │ ├── sshd-keygen@.service.d │ │ │ │ │ └── disable-sshd-keygen-if-cloud-init-active.conf │ │ │ │ │ └── timers.target.wants │ │ │ │ │ └── nm-cloud-setup.timer │ │ │ └── vconsole.conf │ │ │ └── usr │ │ │ └── lib │ │ │ ├── bootc │ │ │ └── install │ │ │ │ └── 05-cloud-kargs.toml │ │ │ └── systemd │ │ │ ├── logind.conf.d │ │ │ └── 00-getty-fixes.conf │ │ │ └── system │ │ │ └── nm-cloud-setup.service.d │ │ │ └── 10-rh-enable-for-ec2.conf │ ├── azure │ │ ├── Makefile.env │ │ ├── README.md │ │ ├── cloud-setup.sh │ │ ├── config.toml │ │ └── files │ │ │ ├── etc │ │ │ ├── X11 │ │ │ │ └── xorg.conf.d │ │ │ │ │ └── 00-keyboard.conf │ │ │ ├── cloud │ │ │ │ └── cloud.cfg.d │ │ │ │ │ ├── 10-azure-kvp.cfg │ │ │ │ │ └── 91-azure_datasource.cfg │ │ │ ├── locale.conf │ │ │ ├── localtime │ │ │ ├── modprobe.d │ │ │ │ ├── blacklist-floppy.conf │ │ │ │ ├── blacklist-intel-cstate.conf │ │ │ │ ├── blacklist-nouveau.conf │ │ │ │ └── blacklist-skylake-edac.conf │ │ │ ├── systemd │ │ │ │ └── system │ │ │ │ │ ├── NetworkManager.service.wants │ │ │ │ │ └── nm-cloud-setup.service │ │ │ │ │ ├── multi-user.target.wants │ │ │ │ │ └── waagent.service │ │ │ │ │ └── timers.target.wants │ │ │ │ │ └── nm-cloud-setup.timer │ │ │ ├── udev │ │ │ │ └── rules.d │ │ │ │ │ └── 68-azure-sriov-nm-unmanaged.rules │ │ │ └── vconsole.conf │ │ │ └── usr │ │ │ └── lib │ │ │ ├── bootc │ │ │ └── install │ │ │ │ └── 05-cloud-kargs.toml │ │ │ └── systemd │ │ │ └── system │ │ │ └── nm-cloud-setup.service.d │ │ │ └── 10-rh-enable-for-azure.conf │ ├── gcp │ │ ├── Makefile.env │ │ ├── README.md │ │ ├── cloud-setup.sh │ │ ├── config.toml │ │ └── files │ │ │ ├── etc │ │ │ ├── X11 │ │ │ │ └── xorg.conf.d │ │ │ │ │ └── 00-keyboard.conf │ │ │ ├── default │ │ │ │ └── instance_configs.cfg │ │ │ ├── locale.conf │ │ │ ├── localtime │ │ │ ├── modprobe.d │ │ │ │ └── blacklist-floppy.conf │ │ │ └── vconsole.conf │ │ │ └── usr │ │ │ └── lib │ │ │ └── bootc │ │ │ └── install │ │ │ └── 05-cloud-kargs.toml │ └── ibm │ │ ├── Makefile.env │ │ ├── README.md │ │ ├── cloud-setup.sh │ │ ├── config.toml │ │ └── files │ │ └── etc │ │ ├── X11 │ │ └── xorg.conf.d │ │ │ └── 00-keyboard.conf │ │ ├── locale.conf │ │ ├── localtime │ │ └── vconsole.conf ├── common │ ├── Makefile.common │ ├── driver-toolkit │ │ └── Containerfile │ └── usr │ │ ├── lib │ │ └── systemd │ │ │ └── system │ │ │ ├── basic.target.wants │ │ │ └── upgrade-informer.service │ │ │ ├── timers.target.wants │ │ │ └── upgrade-informer.timer │ │ │ ├── upgrade-informer.service │ │ │ └── upgrade-informer.timer │ │ └── libexec │ │ └── upgrade-informer ├── deepspeed │ ├── Containerfile │ └── Makefile ├── ilab-wrapper │ ├── ilab │ ├── ilab-qlora │ └── ilab-training-launcher ├── instructlab │ └── Makefile ├── intel-bootc │ ├── Containerfile │ ├── Makefile │ ├── duplicated │ │ ├── common │ │ │ └── usr │ │ │ │ ├── lib │ │ │ │ └── systemd │ │ │ │ │ └── system │ │ │ │ │ ├── basic.target.wants │ │ │ │ │ └── upgrade-informer.service │ │ │ │ │ ├── timers.target.wants │ │ │ │ │ └── upgrade-informer.timer │ │ │ │ │ ├── upgrade-informer.service │ │ │ │ │ └── upgrade-informer.timer │ │ │ │ └── libexec │ │ │ │ └── upgrade-informer │ │ └── ilab-wrapper │ │ │ └── ilab │ └── scripts │ │ └── os_dependencies.sh ├── model │ ├── Containerfile │ ├── Makefile │ ├── entrypoint.sh │ └── generate-model-cfile.py ├── nvidia-bootc │ ├── Containerfile │ ├── Makefile │ ├── containers-storage.conf │ ├── duplicated │ │ ├── common │ │ │ └── usr │ │ │ │ ├── lib │ │ │ │ └── systemd │ │ │ │ │ └── system │ │ │ │ │ ├── basic.target.wants │ │ │ │ │ └── upgrade-informer.service │ │ │ │ │ ├── timers.target.wants │ │ │ │ │ └── upgrade-informer.timer │ │ │ │ │ ├── upgrade-informer.service │ │ │ │ │ └── upgrade-informer.timer │ │ │ │ └── libexec │ │ │ │ └── upgrade-informer │ │ └── ilab-wrapper │ │ │ └── ilab │ ├── nvidia-toolkit-firstboot.service │ └── x509-configuration.ini ├── tests │ ├── ansible.cfg │ ├── e2e-tests │ │ └── playbook.yml │ └── provision │ │ ├── playbook.yml │ │ ├── requirements.yml │ │ └── templates │ │ └── Containerfile.j2 └── vllm │ ├── Containerfile │ ├── Makefile │ └── mixtral.jinja └── vector_dbs ├── README.md ├── chromadb ├── Containerfile └── Makefile └── milvus ├── Containerfile ├── Makefile ├── embedEtcd.yaml └── volumes └── milvus └── .gitkeep /.devcontainer/Containerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/containers/podman:v5.0.2 2 | 3 | USER root 4 | 5 | COPY requirements-test.txt . 6 | 7 | RUN dnf install -y python3.11 python3-pip buildah git make && \ 8 | dnf clean all && \ 9 | pip3 install -r requirements-test.txt 10 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "recipes", 3 | "build": { 4 | "dockerfile": "Containerfile", 5 | "context": ".." 6 | }, 7 | "privileged": true, 8 | "containerEnv": { 9 | "REGISTRY": "ghcr.io", 10 | "IMAGE_NAME": "ai-lab-recipes/playground" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf -------------------------------------------------------------------------------- /.github/workflows/mirror_repository.yaml: -------------------------------------------------------------------------------- 1 | name: Mirror Repository 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | workflow_dispatch: 9 | 10 | concurrency: 11 | group: ${{ github.workflow }} 12 | cancel-in-progress: false 13 | 14 | jobs: 15 | mirror-repository: 16 | if: github.repository == 'containers/ai-lab-recipes' 17 | runs-on: ubuntu-24.04 18 | steps: 19 | - uses: actions/checkout@v4.1.7 20 | with: 21 | fetch-depth: 0 22 | 23 | - uses: pixta-dev/repository-mirroring-action@v1.1.1 24 | with: 25 | target_repo_url: 26 | git@github.com:containers-mirror/ai-lab-recipes.git 27 | ssh_private_key: 28 | ${{ secrets.SSH_PRIVATE_KEY }} 29 | 30 | - name: Publish Job Results to Slack 31 | id: slack 32 | if: always() 33 | uses: slackapi/slack-github-action@v1.26.0 34 | with: 35 | payload: | 36 | { 37 | "text": "${{ github.workflow }} workflow status: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" 38 | } 39 | env: 40 | SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gguf 2 | *.bin 3 | *_pycache_* 4 | port_check.lock 5 | *build 6 | models/* 7 | !models/Makefile 8 | !models/README.md 9 | !models/download_hf_models.py 10 | convert_models/converted_models 11 | recipes/common/bin/* 12 | */.venv/ 13 | **/venv/** 14 | training/cloud/examples 15 | training/instructlab/instructlab 16 | vector_dbs/milvus/volumes/milvus/* 17 | .idea 18 | **/volumes/** 19 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @MichaelClifford @rhatdan @sallyom @cgwalters @Gregory-Pereira @jeffmaury 2 | -------------------------------------------------------------------------------- /ailab-images.md: -------------------------------------------------------------------------------- 1 | ## Model Server Images (amd64, arm64) currently built from GH Actions in this repository 2 | 3 | - quay.io/ai-lab/llamacpp_python:latest 4 | - quay.io/ai-lab/llamacpp-python-cuda:latest 5 | - quay.io/ai-lab/llamacpp-python-vulkan:latest 6 | - quay.io/redhat-et/locallm-object-detection-server:latest 7 | 8 | ## Recipe Images (amd64, arm64) 9 | - quay.io/ai-lab/summarizer:latest 10 | - quay.io/ai-lab/chatbot:latest 11 | - quay.io/ai-lab/rag:latest 12 | - quay.io/ai-lab/codegen:latest 13 | - quay.io/redhat-et/locallm-object-detection-client:latest 14 | 15 | ## Dependency images (amd64) 16 | 17 | Images used in the `Bootc` aspect of this repo or tooling images 18 | 19 | - quay.io/ai-lab/chromadb:latest 20 | - quay.io/ai-lab/model-converter:latest 21 | 22 | ## Model Images (amd64, arm64) 23 | 24 | - quay.io/ai-lab/merlinite-7b-lab:latest 25 | - [model download link](https://huggingface.co/instructlab/merlinite-7b-lab-GGUF/resolve/main/merlinite-7b-lab-Q4_K_M.gguf) 26 | - quay.io/ai-lab/granite-7b-lab:latest 27 | - [model download link](https://huggingface.co/instructlab/granite-7b-lab-GGUF/resolve/main/granite-7b-lab-Q4_K_M.gguf) 28 | - quay.io/ai-lab/mistral-7b-instruct:latest 29 | - [model download link](https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_M.gguf) 30 | - quay.io/ai-lab/mistral-7b-code-16k-qlora:latest 31 | - [model download link](https://huggingface.co/TheBloke/Mistral-7B-Code-16K-qlora-GGUF/resolve/main/mistral-7b-code-16k-qlora.Q4_K_M.gguf) 32 | - quay.io/ai-lab/whisper-small:latest 33 | - [model download link](https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin) 34 | 35 | -------------------------------------------------------------------------------- /assets/ai_lab_recipes_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/assets/ai_lab_recipes_logo.png -------------------------------------------------------------------------------- /assets/chatbot_nodejs_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/assets/chatbot_nodejs_ui.png -------------------------------------------------------------------------------- /assets/chatbot_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/assets/chatbot_ui.png -------------------------------------------------------------------------------- /assets/codegen_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/assets/codegen_ui.png -------------------------------------------------------------------------------- /assets/function_calling_nodejs_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/assets/function_calling_nodejs_ui.png -------------------------------------------------------------------------------- /assets/image_analysis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/assets/image_analysis.png -------------------------------------------------------------------------------- /assets/install_continue_extension.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/assets/install_continue_extension.png -------------------------------------------------------------------------------- /assets/model_converter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/assets/model_converter.png -------------------------------------------------------------------------------- /assets/object_detection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/assets/object_detection.png -------------------------------------------------------------------------------- /assets/rag_nodejs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/assets/rag_nodejs.png -------------------------------------------------------------------------------- /assets/rag_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/assets/rag_ui.png -------------------------------------------------------------------------------- /assets/summarizer_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/assets/summarizer_ui.png -------------------------------------------------------------------------------- /assets/whisper.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/assets/whisper.png -------------------------------------------------------------------------------- /convert_models/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | WORKDIR /opt/app-root/src/converter 3 | USER root 4 | RUN chown -R default:root /opt/app-root/src/converter 5 | USER default 6 | RUN git clone https://github.com/ggerganov/llama.cpp.git 7 | RUN cd llama.cpp/ && make 8 | RUN pip install -r llama.cpp/requirements.txt 9 | COPY . /opt/app-root/src/converter/ 10 | ENTRYPOINT ["sh", "run.sh"] 11 | -------------------------------------------------------------------------------- /convert_models/download_huggingface.py: -------------------------------------------------------------------------------- 1 | from huggingface_hub import snapshot_download 2 | import argparse 3 | 4 | parser = argparse.ArgumentParser() 5 | parser.add_argument("-m", "--model") 6 | parser.add_argument("-t", "--token") 7 | args = parser.parse_args() 8 | 9 | snapshot_download(repo_id=args.model, 10 | token=args.token, 11 | local_dir=f"converted_models/{args.model}", 12 | local_dir_use_symlinks=True, 13 | cache_dir=f"converted_models/cache") -------------------------------------------------------------------------------- /convert_models/requirements.txt: -------------------------------------------------------------------------------- 1 | huggingface_hub -------------------------------------------------------------------------------- /convert_models/run.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | hf_model_url=${HF_MODEL_URL} 4 | hf_token=${HF_TOKEN:="None"} 5 | model_org=$(echo $hf_model_url | sed -n 's/\(.*\)\/\(.*\)/\1/p') 6 | model_name=$(echo $hf_model_url | sed -n 's/\(.*\)\/\(.*\)/\2/p') 7 | keep_orgi=${KEEP_ORIGINAL_MODEL} 8 | 9 | if [ -e "/opt/app-root/src/converter/converted_models/gguf/$model_org-$model_name-${QUANTIZATION}.gguf" ]; then 10 | echo "$model_org-$model_name-${QUANTIZATION}.gguf already exists... skipping" 11 | exit 0 12 | fi 13 | 14 | if [ -e "/opt/app-root/src/converter/converted_models/cache/models--$model_org--$model_name" ]; then 15 | echo "$hf_model_url present in cache... skipping download" 16 | fi 17 | 18 | echo "Downloading $hf_model_url" 19 | python download_huggingface.py --model $hf_model_url --token $hf_token 20 | python llama.cpp/examples/convert_legacy_llama.py /opt/app-root/src/converter/converted_models/$hf_model_url 21 | python llama.cpp/convert_hf_to_gguf.py /opt/app-root/src/converter/converted_models/$hf_model_url 22 | mkdir -p /opt/app-root/src/converter/converted_models/gguf/ 23 | llama.cpp/llama-quantize /opt/app-root/src/converter/converted_models/$hf_model_url/ggml-model-f16.gguf /opt/app-root/src/converter/converted_models/gguf/$model_org-$model_name-${QUANTIZATION}.gguf ${QUANTIZATION} 24 | rm -rf /opt/app-root/src/converter/converted_models/$model_org 25 | 26 | if [ $keep_orgi = "False" ]; then 27 | rm -rf /opt/app-root/src/converter/converted_models/cache 28 | fi 29 | 30 | echo "Converted and quantized model written to /opt/app-root/src/converter/converted_models/gguf/$model_org-$model_name.gguf" 31 | echo "$ ls /opt/app-root/src/converter/converted_models/gguf/" 32 | ls /opt/app-root/src/converter/converted_models/gguf/ 33 | -------------------------------------------------------------------------------- /data/fake_meeting.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/data/fake_meeting.pdf -------------------------------------------------------------------------------- /data/jfk.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/data/jfk.wav -------------------------------------------------------------------------------- /eval/promptfoo/README.md: -------------------------------------------------------------------------------- 1 | # LLM Evaluation with Promptfoo 2 | 3 | We are using the [Promptfoo.dev](https://www.promptfoo.dev/) project for LLM model evaluation. 4 | 5 | ``` 6 | podman build -t promptfoo eval/promptfoo/build 7 | ``` 8 | 9 | Make sure you are running an LLM before starting the promptfoo container. 10 | 11 | ``` 12 | podman run -it -p 15500:15500 -v /locallm/eval/promptfoo/evals/:/promptfoo/evals:ro promptfoo 13 | ``` 14 | 15 | Go to `http://0.0.0.0:15500/setup/` to set up your tests. -------------------------------------------------------------------------------- /eval/promptfoo/base/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/nodejs-20-minimal:1-63.1725851021 2 | WORKDIR /promptfoo 3 | RUN npm install promptfoo 4 | ENV PROMPTFOO_DISABLE_TELEMETRY=1 5 | RUN mkdir evals 6 | ENV PROMPTFOO_CONFIG_DIR=/promptfoo/evals 7 | COPY promptfooconfig.yaml /promptfoo 8 | ENTRYPOINT [ "npx", "promptfoo@latest", "view", "--yes" ] 9 | -------------------------------------------------------------------------------- /eval/promptfoo/base/promptfooconfig.yaml: -------------------------------------------------------------------------------- 1 | # This configuration compares LLM output of 2 prompts x 2 GPT models across 3 test cases. 2 | # Learn more: https://promptfoo.dev/docs/configuration/guide 3 | description: 'My first eval' 4 | 5 | prompts: 6 | - "Write a tweet about {{topic}}" 7 | - "Write a very concise, funny tweet about {{topic}}" 8 | 9 | providers: 10 | - openai:gpt-3.5-turbo-0613 11 | - openai:gpt-4 12 | 13 | tests: 14 | - vars: 15 | topic: bananas 16 | 17 | - vars: 18 | topic: avocado toast 19 | assert: 20 | # For more information on assertions, see https://promptfoo.dev/docs/configuration/expected-outputs 21 | - type: icontains 22 | value: avocado 23 | - type: javascript 24 | value: 1 / (output.length + 1) # prefer shorter outputs 25 | 26 | - vars: 27 | topic: new york city 28 | assert: 29 | # For more information on model-graded evals, see https://promptfoo.dev/docs/configuration/expected-outputs/model-graded 30 | - type: llm-rubric 31 | value: ensure that the output is funny 32 | -------------------------------------------------------------------------------- /eval/promptfoo/evals/README.md: -------------------------------------------------------------------------------- 1 | Directory to store evaluation runs locally -------------------------------------------------------------------------------- /hooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SOURCE_FILE="training/ilab-wrapper/ilab" 4 | DEST_FILE="training/nvidia-bootc/duplicated/ilab-wrapper/ilab" 5 | 6 | if [[ -f "$SOURCE_FILE" ]]; then 7 | mkdir -p "$(dirname "$DEST_FILE")" 8 | cp "$SOURCE_FILE" "$DEST_FILE" 9 | git add "$DEST_FILE" 10 | else 11 | echo "Source file $SOURCE_FILE does not exist. Aborting commit." 12 | exit 1 13 | fi 14 | -------------------------------------------------------------------------------- /install-hooks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | HOOKS_DIR="hooks" 4 | GIT_HOOKS_DIR=".git/hooks" 5 | 6 | cp "$HOOKS_DIR/pre-commit" "$GIT_HOOKS_DIR/pre-commit" 7 | 8 | echo "Hooks installed successfully." 9 | -------------------------------------------------------------------------------- /model_servers/common/Makefile.common: -------------------------------------------------------------------------------- 1 | CONTAINER_TOOL ?= podman 2 | REGISTRY ?= quay.io 3 | REGISTRY_ORG ?= ai-lab 4 | COMPONENT ?= model_servers 5 | CHAT_FORMAT ?= 6 | 7 | BIND_MOUNT_OPTIONS := ro 8 | OS := $(shell uname -s) 9 | ifeq ($(OS),Linux) 10 | BIND_MOUNT_OPTIONS := Z,ro 11 | endif 12 | 13 | .PHONY: build 14 | build: 15 | podman build --squash-all --build-arg PORT=$(PORT) -t $(IMAGE) . -f base/Containerfile 16 | 17 | .PHONY: install 18 | install: 19 | pip install -r tests/requirements.txt 20 | 21 | .PHONY: test 22 | test: 23 | @if [ ! -f "../../models/$(MODEL_NAME)" ]; then \ 24 | echo "Model file -- $(MODEL_NAME) -- not present in the models directory."; \ 25 | exit 1; \ 26 | else \ 27 | if [ ! -f "./$(MODEL_NAME)" ]; then \ 28 | ln -s ../../models/$(MODEL_NAME) ./$(MODEL_NAME); \ 29 | fi; \ 30 | REGISTRY=$(REGISTRY) IMAGE_NAME=$(IMAGE_NAME) MODEL_NAME=$(MODEL_NAME) MODEL_PATH=$(MODEL_PATH) PORT=$(PORT) pytest -vvv -s ; \ 31 | fi; 32 | 33 | .PHONY: clean 34 | clean: 35 | - rm ./$(MODEL_NAME) &> /dev/null 36 | 37 | .PHONY: run 38 | run: 39 | cd ../../models && \ 40 | podman run -it \ 41 | -d \ 42 | -p $(PORT):$(PORT) \ 43 | -v ./$(MODEL_NAME):$(MODELS_PATH)/$(MODEL_NAME):$(BIND_MOUNT_OPTIONS) \ 44 | -e MODEL_PATH=$(MODELS_PATH)/$(MODEL_NAME) \ 45 | -e HOST=0.0.0.0 \ 46 | -e PORT=$(PORT) \ 47 | $(CHAT_FORMAT:%=-e CHAT_FORMAT=${CHAT_FORMAT}) \ 48 | $(IMAGE) 49 | 50 | .PHONY: podman-clean 51 | podman-clean: 52 | @container_ids=$$(podman ps --format "{{.ID}} {{.Image}}" | awk '$$2 == "$(IMAGE)" {print $$1}'); \ 53 | echo "removing all containers with IMAGE=$(IMAGE)"; \ 54 | for id in $$container_ids; do \ 55 | echo "Removing container: $$id,"; \ 56 | podman rm -f $$id; \ 57 | done 58 | -------------------------------------------------------------------------------- /model_servers/llamacpp_python/Makefile: -------------------------------------------------------------------------------- 1 | APP := llamacpp_python 2 | PORT ?= 8001 3 | CHAT_FORMAT ?= 4 | 5 | include ../common/Makefile.common 6 | 7 | IMAGE_NAME ?= $(REGISTRY_ORG)/$(COMPONENT)/$(APP):latest 8 | IMAGE := $(REGISTRY)/$(IMAGE_NAME) 9 | CUDA_IMAGE := $(REGISTRY)/$(REGISTRY_ORG)/$(COMPONENT)/$(APP)_cuda:latest 10 | VULKAN_IMAGE := $(REGISTRY)/$(REGISTRY_ORG)/$(COMPONENT)/$(APP)_vulkan:latest 11 | 12 | MODELS_PATH := /locallm/models 13 | MODEL_NAME ?= granite-7b-lab-Q4_K_M.gguf 14 | 15 | .Phony: all 16 | all: build download-model-granite run 17 | 18 | .PHONY: build-cuda 19 | build-cuda: 20 | "${CONTAINER_TOOL}" build --squash-all -t $(CUDA_IMAGE) . -f cuda/Containerfile 21 | 22 | .PHONY: build-vulkan-amd64 build-vulkan-arm64 23 | build-vulkan-amd64: 24 | "${CONTAINER_TOOL}" build --squash-all -t $(VULKAN_IMAGE) . -f vulkan/amd64/Containerfile 25 | build-vulkan-arm64: 26 | "${CONTAINER_TOOL}" build --squash-all -t $(VULKAN_IMAGE) . -f vulkan/arm64/Containerfile 27 | 28 | .PHONY: download-model-granite # default model 29 | download-model-granite: 30 | cd ../../models/ && \ 31 | make download-model-granite 32 | -------------------------------------------------------------------------------- /model_servers/llamacpp_python/base/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | WORKDIR /locallm 3 | COPY src . 4 | USER root 5 | RUN dnf install -y gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ 6 | USER 1001 7 | RUN CC="/opt/rh/gcc-toolset-13/root/usr/bin/gcc" CXX="/opt/rh/gcc-toolset-13/root/usr/bin/g++" pip install --no-cache-dir --verbose -r ./requirements.txt 8 | EXPOSE 8001 9 | ENTRYPOINT [ "sh", "./run.sh" ] 10 | -------------------------------------------------------------------------------- /model_servers/llamacpp_python/cuda/Containerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/opendatahub/workbench-images:cuda-ubi9-python-3.9-20231206 2 | USER root 3 | RUN dnf install -y gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ 4 | USER 1001 5 | WORKDIR /locallm 6 | COPY src . 7 | ENV CMAKE_ARGS="-DGGML_CUDA=on -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF -DLLAMA_F16C=OFF" 8 | ENV FORCE_CMAKE=1 9 | RUN CC="/opt/rh/gcc-toolset-13/root/usr/bin/gcc" CXX="/opt/rh/gcc-toolset-13/root/usr/bin/g++" pip install --no-cache-dir -r ./requirements.txt 10 | ENTRYPOINT [ "sh", "run.sh" ] 11 | -------------------------------------------------------------------------------- /model_servers/llamacpp_python/src/requirements.txt: -------------------------------------------------------------------------------- 1 | llama-cpp-python[server]==0.2.90 2 | transformers==4.41.2 3 | pip==24.0 4 | -------------------------------------------------------------------------------- /model_servers/llamacpp_python/src/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ ${CONFIG_PATH} ] || [[ ${MODEL_PATH} && ${CONFIG_PATH} ]]; then 3 | python -m llama_cpp.server --config_file ${CONFIG_PATH} 4 | exit 0 5 | fi 6 | 7 | if [ "${MODEL_HF_PRETRAINED_MODEL}" == "None" ]; then 8 | MODEL_HF_PRETRAINED_MODEL="" 9 | fi 10 | 11 | if [ ${MODEL_PATH} ]; then 12 | python -m llama_cpp.server \ 13 | --model ${MODEL_PATH} \ 14 | --host ${HOST:=0.0.0.0} \ 15 | --port ${PORT:=8001} \ 16 | --n_gpu_layers ${GPU_LAYERS:=0} \ 17 | --clip_model_path ${CLIP_MODEL_PATH:=None} \ 18 | --chat_format ${MODEL_CHAT_FORMAT:=llama-2} \ 19 | ${PRETRAINED_MODEL_PATH:=} \ 20 | ${MODEL_HF_PRETRAINED_MODEL:+--hf_pretrained_model_name_or_path ${MODEL_HF_PRETRAINED_MODEL}} \ 21 | --interrupt_requests ${INTERRUPT_REQUESTS:=False} 22 | exit 0 23 | fi 24 | 25 | echo "Please set either a CONFIG_PATH or a MODEL_PATH" 26 | exit 1 27 | 28 | -------------------------------------------------------------------------------- /model_servers/llamacpp_python/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/model_servers/llamacpp_python/tests/__init__.py -------------------------------------------------------------------------------- /model_servers/llamacpp_python/tests/requirements.txt: -------------------------------------------------------------------------------- 1 | pip==24.0 2 | pytest-container==0.4.0 3 | pytest-selenium==4.1.0 4 | pytest-testinfra==10.1.0 5 | pytest==8.1.1 6 | requests==2.31.0 7 | selenium==4.19.0 8 | tenacity==8.2.3 9 | -------------------------------------------------------------------------------- /model_servers/llamacpp_python/tests/test_alive.py: -------------------------------------------------------------------------------- 1 | import pytest_container 2 | from .conftest import MS 3 | import tenacity 4 | import os 5 | 6 | CONTAINER_IMAGES = [MS] 7 | 8 | def test_etc_os_release_present(auto_container: pytest_container.container.ContainerData): 9 | assert auto_container.connection.file("/etc/os-release").exists 10 | 11 | @tenacity.retry(stop=tenacity.stop_after_attempt(5), wait=tenacity.wait_exponential()) 12 | def test_alive(auto_container: pytest_container.container.ContainerData, host): 13 | host.run_expect([0],f"curl http://localhost:{auto_container.forwarded_ports[0].host_port}",).stdout.strip() 14 | -------------------------------------------------------------------------------- /model_servers/llamacpp_python/vulkan/amd64/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | USER 0 3 | RUN dnf install -y python3-dnf-plugin-versionlock 4 | RUN dnf install -y mesa-vulkan-drivers-24.1.2-3.el9.x86_64 5 | RUN dnf versionlock mesa-vulkan-drivers-24.1.2-3.el9.x86_64 6 | RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm 7 | RUN dnf install -y git cmake ninja-build gcc gcc-c++ 8 | RUN dnf copr enable -y ligenix/enterprise-sandbox epel-9-x86_64 9 | RUN dnf install -y vulkan-headers vulkan-tools 10 | USER 1001 11 | WORKDIR /locallm 12 | COPY src . 13 | RUN pip install --upgrade pip 14 | ENV CMAKE_ARGS="-DLLAMA_VULKAN=on" 15 | ENV FORCE_CMAKE=1 16 | RUN pip install --no-cache-dir --upgrade -r /locallm/requirements.txt 17 | ENTRYPOINT [ "sh", "run.sh" ] 18 | -------------------------------------------------------------------------------- /model_servers/llamacpp_python/vulkan/arm64/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | USER 0 3 | RUN dnf install -y python3-dnf-plugin-versionlock && \ 4 | dnf install -y \ 5 | https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \ 6 | dnf copr enable -y slp/mesa-krunkit epel-9-aarch64 && \ 7 | dnf install -y mesa-vulkan-drivers-24.1.2-101.el9.aarch64 && \ 8 | dnf versionlock mesa-vulkan-drivers-24.1.2-101.el9.aarch64 && \ 9 | dnf install -y git cmake ninja-build gcc gcc-c++ vulkan-loader-devel vulkan-tools 10 | USER 1001 11 | WORKDIR /locallm 12 | COPY src . 13 | RUN pip install --upgrade pip 14 | ENV CMAKE_ARGS="-DLLAMA_VULKAN=on" 15 | ENV FORCE_CMAKE=1 16 | RUN pip install --no-cache-dir --upgrade -r /locallm/requirements.txt 17 | ENTRYPOINT [ "sh", "run.sh" ] 18 | -------------------------------------------------------------------------------- /model_servers/object_detection_python/Makefile: -------------------------------------------------------------------------------- 1 | APP := object_detection_python 2 | PORT ?= 8000 3 | 4 | REGISTRY ?= ghcr.io 5 | REGISTRY_ORG ?= containers 6 | 7 | MODEL_NAME ?= facebook/detr-resnet-101 8 | MODELS_DIR := /app/models 9 | 10 | include ../common/Makefile.common 11 | 12 | IMAGE_NAME ?= $(REGISTRY_ORG)/$(APP):latest 13 | IMAGE := $(REGISTRY)/$(IMAGE_NAME) 14 | 15 | # Run override required because of the multi-directory models and model_path vs models_dir 16 | .PHONY: run 17 | run: 18 | cd ../../models && \ 19 | podman run -it -d -p $(PORT):$(PORT) -v ./$(MODEL_NAME):$(MODELS_DIR)/$(MODEL_NAME):$(BIND_MOUNT_OPTIONS) -e MODEL_PATH=$(MODELS_DIR)/$(MODEL_NAME) -e HOST=0.0.0.0 -e PORT=$(PORT) $(IMAGE) 20 | 21 | .PHONY: all 22 | all: build download-model-facebook-detr-resnet-101 run 23 | 24 | .PHONY: download-model-facebook-detr-resnet-101 25 | download-model-facebook-detr-resnet-101: 26 | cd ../../models && \ 27 | make download-model-facebook-detr-resnet-101 28 | 29 | .PHONY: test 30 | test: 31 | pip install -r ../../convert_models/requirements.txt 32 | cp -r ../../models/facebook ./ 33 | REGISTRY=$(REGISTRY) MODEL_NAME=$(MODEL_NAME) MODELS_DIR=$(MODELS_DIR) IMAGE_NAME=$(IMAGE_NAME) PORT=$(PORT) pytest -s -vvv 34 | -------------------------------------------------------------------------------- /model_servers/object_detection_python/base/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | ARG PORT=8000 3 | WORKDIR /app 4 | COPY src . 5 | RUN pip install --upgrade pip && \ 6 | pip install --no-cache-dir --upgrade -r requirements.txt 7 | EXPOSE $PORT 8 | ENTRYPOINT [ "sh", "./run.sh" ] 9 | -------------------------------------------------------------------------------- /model_servers/object_detection_python/src/requirements-unlocked.txt: -------------------------------------------------------------------------------- 1 | fastapi 2 | pillow 3 | pydantic 4 | requests 5 | transformers 6 | torch 7 | uvicorn 8 | timm 9 | -------------------------------------------------------------------------------- /model_servers/object_detection_python/src/requirements.txt: -------------------------------------------------------------------------------- 1 | annotated-types==0.7.0 2 | anyio==4.4.0 3 | certifi==2024.6.2 4 | charset-normalizer==3.3.2 5 | click==8.1.8 6 | dnspython==2.6.1 7 | email_validator==2.2.0 8 | fastapi==0.111.1 9 | fastapi-cli==0.0.7 10 | filelock==3.15.4 11 | fsspec==2024.6.1 12 | h11==0.16.0 13 | httpcore==1.0.9 14 | httptools==0.6.4 15 | httpx==0.27.2 16 | huggingface-hub==0.23.4 17 | idna==3.7 18 | Jinja2==3.1.6 19 | markdown-it-py==3.0.0 20 | MarkupSafe==2.1.5 21 | mdurl==0.1.2 22 | mpmath==1.3.0 23 | networkx==3.3 24 | numpy==2.0.1 25 | orjson==3.10.18 26 | packaging==24.1 27 | pillow==10.3.0 28 | pydantic==2.7.4 29 | pydantic_core==2.18.4 30 | Pygments==2.18.0 31 | python-dotenv==1.0.1 32 | python-multipart==0.0.20 33 | PyYAML==6.0.2 34 | regex==2024.5.15 35 | requests==2.32.3 36 | rich==13.7.1 37 | safetensors==0.4.5 38 | shellingham==1.5.4 39 | sniffio==1.3.1 40 | starlette==0.37.2 41 | sympy==1.12.1 42 | timm==1.0.15 43 | tokenizers==0.19.1 44 | torch==2.3.1 45 | torchvision==0.18.1 46 | tqdm==4.66.5 47 | transformers==4.41.2 48 | typer==0.12.5 49 | typing_extensions==4.12.2 50 | ujson==5.10.0 51 | urllib3==2.2.3 52 | uvicorn==0.30.6 53 | uvloop==0.19.0 54 | watchfiles==0.22.0 55 | websockets==12.0 56 | -------------------------------------------------------------------------------- /model_servers/object_detection_python/src/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ${MODEL_PATH} ]; then 4 | PORT=${PORT} MODEL_PATH=${MODEL_PATH} uvicorn object_detection_server:app --port ${PORT:=8000} --host ${HOST:=0.0.0.0} 5 | exit 0 6 | fi 7 | 8 | echo "Please set either a MODEL_PATH" 9 | exit 1 10 | -------------------------------------------------------------------------------- /model_servers/object_detection_python/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/model_servers/object_detection_python/tests/__init__.py -------------------------------------------------------------------------------- /model_servers/object_detection_python/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest_container 2 | import os 3 | 4 | REGISTRY = os.getenv("REGISTRY", "ghcr.io") 5 | IMAGE_NAME = os.getenv("IMAGE_NAME", "containers/object_detection_python:latest") 6 | MODEL_NAME = os.getenv("MODEL_NAME", "facebook/detr-resnet-101") 7 | MODELS_DIR = os.getenv("MODELS_DIR", "/app/models") 8 | 9 | MODEL_PATH = f"{MODELS_DIR}/{MODEL_NAME}" 10 | 11 | PORT = os.getenv("PORT", 8000) 12 | if type(PORT) == str: 13 | try: 14 | PORT = int(PORT) 15 | except: 16 | PORT = 8000 17 | 18 | MS = pytest_container.Container( 19 | url=f"containers-storage:{REGISTRY}/{IMAGE_NAME}", 20 | volume_mounts=[ 21 | pytest_container.container.BindMount( 22 | container_path=f"{MODEL_PATH}", 23 | host_path=f"./{MODEL_NAME}", 24 | flags=["ro"] 25 | ) 26 | ], 27 | extra_environment_variables={ 28 | "MODEL_PATH": f"{MODEL_PATH}", 29 | "HOST": "0.0.0.0", 30 | "PORT": f"{PORT}", 31 | "IMAGE_NAME": f"{IMAGE_NAME}", 32 | "REGISTRY": f"{REGISTRY}" 33 | }, 34 | forwarded_ports=[ 35 | pytest_container.PortForwarding( 36 | container_port=PORT, 37 | host_port=PORT 38 | ) 39 | ], 40 | ) 41 | 42 | def pytest_generate_tests(metafunc): 43 | pytest_container.auto_container_parametrize(metafunc) 44 | 45 | def pytest_addoption(parser): 46 | pytest_container.add_logging_level_options(parser) 47 | -------------------------------------------------------------------------------- /model_servers/object_detection_python/tests/requirements.txt: -------------------------------------------------------------------------------- 1 | pip==24.0 2 | pytest-container==0.4.0 3 | pytest-selenium==4.1.0 4 | pytest-testinfra==10.1.0 5 | pytest==8.1.1 6 | requests==2.31.0 7 | selenium==4.19.0 8 | tenacity==8.2.3 9 | -------------------------------------------------------------------------------- /model_servers/object_detection_python/tests/test_alive.py: -------------------------------------------------------------------------------- 1 | import pytest_container 2 | from .conftest import MS 3 | import tenacity 4 | 5 | CONTAINER_IMAGES = [MS] 6 | 7 | def test_etc_os_release_present(auto_container: pytest_container.container.ContainerData): 8 | assert auto_container.connection.file("/etc/os-release").exists 9 | 10 | @tenacity.retry(stop=tenacity.stop_after_attempt(5), wait=tenacity.wait_exponential()) 11 | def test_alive(auto_container: pytest_container.container.ContainerData, host): 12 | host.run_expect([0],f"curl http://localhost:{auto_container.forwarded_ports[0].host_port}",).stdout.strip() 13 | -------------------------------------------------------------------------------- /model_servers/ollama/base/Containerfile: -------------------------------------------------------------------------------- 1 | FROM ollama/ollama 2 | -------------------------------------------------------------------------------- /model_servers/whispercpp/Makefile: -------------------------------------------------------------------------------- 1 | APP := whispercpp 2 | PORT ?= 8001 3 | 4 | include ../common/Makefile.common 5 | 6 | IMAGE_NAME ?= $(REGISTRY_ORG)/$(COMPONENT)/$(APP):latest 7 | IMAGE ?= $(REGISTRY)/$(IMAGE_NAME) 8 | # CUDA_IMAGE_NAME := $(REGISTRY)/$(BASE_IMAGE_NAME)/$(APP)_cuda:latest 9 | # VULKAN_IMAGE := $(REGISTRY)/$(BASE_IMAGE_NAME)/$(APP)_vulkan:latest 10 | 11 | MODELS_PATH := /app/models 12 | MODEL_NAME ?= ggml-small.bin 13 | 14 | .PHONY: all 15 | all: build download-model-whisper-small run 16 | 17 | .PHONY: download-model-whisper-small 18 | download-model-whisper-small: 19 | cd ../../models && \ 20 | make download-model-whisper-small 21 | -------------------------------------------------------------------------------- /model_servers/whispercpp/base/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/ubi:latest as builder 2 | 3 | WORKDIR /app 4 | RUN dnf install -y git make gcc gcc-c++ 5 | RUN mkdir whisper && cd whisper && git clone https://github.com/ggerganov/whisper.cpp.git . && \ 6 | git checkout tags/v1.5.4 && \ 7 | make && \ 8 | cp main /app/main && \ 9 | cp server /app/server && \ 10 | cp samples/jfk.wav /app/jfk.wav && \ 11 | cd ../ && rm -rf whisper 12 | 13 | FROM registry.access.redhat.com/ubi9-minimal:latest 14 | WORKDIR /app 15 | COPY --from=builder /app /app 16 | 17 | # https://github.com/wader/static-ffmpeg 18 | # https://hub.docker.com/r/mwader/static-ffmpeg/ 19 | COPY --from=mwader/static-ffmpeg:6.1.1 /ffmpeg /bin/ 20 | COPY --from=mwader/static-ffmpeg:6.1.1 /ffprobe /bin/ 21 | 22 | COPY --chown=0:0 --chmod=755 src /app 23 | RUN chown 1001:1001 /app 24 | 25 | USER 1001 26 | 27 | ENV AUDIO_FILE=/app/jfk.wav 28 | ENTRYPOINT ["sh", "run.sh"] 29 | -------------------------------------------------------------------------------- /model_servers/whispercpp/src/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ./server -tr --model ${MODEL_PATH} --convert --host ${HOST:=0.0.0.0} --port ${PORT:=8001} 4 | -------------------------------------------------------------------------------- /model_servers/whispercpp/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/model_servers/whispercpp/tests/__init__.py -------------------------------------------------------------------------------- /model_servers/whispercpp/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest_container 2 | import os 3 | 4 | if not 'REGISTRY' in os.environ: 5 | REGISTRY = 'ghcr.io' 6 | else: 7 | REGISTRY = os.environ['REGISTRY'] 8 | 9 | if not 'IMAGE_NAME' in os.environ: 10 | IMAGE_NAME = 'containers/whispercpp:latest' 11 | else: 12 | IMAGE_NAME = os.environ['IMAGE_NAME'] 13 | 14 | if not 'MODEL_NAME' in os.environ: 15 | MODEL_NAME = 'ggml-small.bin' 16 | else: 17 | MODEL_NAME = os.environ['MODEL_NAME'] 18 | 19 | if not 'MODEL_PATH' in os.environ: 20 | MODEL_PATH = "/app/models" 21 | else: 22 | MODEL_PATH = os.environ['MODEL_PATH'] 23 | 24 | if not 'PORT' in os.environ: 25 | PORT = 8001 26 | else: 27 | PORT = os.environ['PORT'] 28 | try: 29 | PORT = int(PORT) 30 | except: 31 | PORT = 8001 32 | 33 | MS = pytest_container.Container( 34 | url=f"containers-storage:{REGISTRY}/{IMAGE_NAME}", 35 | volume_mounts=[ 36 | pytest_container.container.BindMount( 37 | container_path="{MODEL_PATH}/{MODEL_NAME}".format(MODEL_PATH=MODEL_PATH, MODEL_NAME=MODEL_NAME), 38 | host_path=f"./{MODEL_NAME}", 39 | flags=["ro"] 40 | ) 41 | ], 42 | extra_environment_variables={ 43 | "MODEL_PATH": "{MODEL_PATH}/{MODEL_NAME}".format(MODEL_PATH=MODEL_PATH, MODEL_NAME=MODEL_NAME), 44 | "HOST": "0.0.0.0", 45 | "PORT": f"{PORT}" 46 | }, 47 | forwarded_ports=[ 48 | pytest_container.PortForwarding( 49 | container_port=PORT, 50 | host_port=PORT 51 | ) 52 | ], 53 | ) 54 | 55 | def pytest_generate_tests(metafunc): 56 | pytest_container.auto_container_parametrize(metafunc) 57 | 58 | def pytest_addoption(parser): 59 | pytest_container.add_logging_level_options(parser) 60 | -------------------------------------------------------------------------------- /model_servers/whispercpp/tests/requirements.txt: -------------------------------------------------------------------------------- 1 | pip==24.0 2 | pytest-container==0.4.0 3 | pytest-selenium==4.1.0 4 | pytest-testinfra==10.1.0 5 | pytest==8.1.1 6 | requests==2.31.0 7 | selenium==4.19.0 8 | tenacity==8.2.3 9 | -------------------------------------------------------------------------------- /model_servers/whispercpp/tests/test_alive.py: -------------------------------------------------------------------------------- 1 | import pytest_container 2 | from .conftest import MS 3 | import tenacity 4 | 5 | CONTAINER_IMAGES = [MS] 6 | 7 | def test_etc_os_release_present(auto_container: pytest_container.container.ContainerData): 8 | assert auto_container.connection.file("/etc/os-release").exists 9 | 10 | @tenacity.retry(stop=tenacity.stop_after_attempt(5), wait=tenacity.wait_exponential()) 11 | def test_alive(auto_container: pytest_container.container.ContainerData, host): 12 | host.run_expect([0],f"curl http://localhost:{auto_container.forwarded_ports[0].host_port}",).stdout.strip() 13 | -------------------------------------------------------------------------------- /models/Containerfile: -------------------------------------------------------------------------------- 1 | # Suggested alternative open AI Models 2 | # https://huggingface.co/instructlab/granite-7b-lab-GGUF/resolve/main/granite-7b-lab-Q4_K_M.gguf (Default) 3 | # https://huggingface.co/instructlab/merlinite-7b-lab-GGUF/resolve/main/merlinite-7b-lab-Q4_K_M.gguf 4 | # https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_S.gguf 5 | # https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_M.gguf 6 | # https://huggingface.co/TheBloke/CodeLlama-7B-Instruct-GGUF/resolve/main/codellama-7b-instruct.Q4_K_M.gguf 7 | # https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin 8 | # podman build --build-arg="MODEL_URL=https://..." -t quay.io/yourimage . 9 | # 10 | FROM registry.access.redhat.com/ubi9/ubi-micro:9.4-15 11 | 12 | # Can be substituted using the --build-arg defined above 13 | ARG MODEL_URL=https://huggingface.co/instructlab/granite-7b-lab-GGUF/resolve/main/granite-7b-lab-Q4_K_M.gguf 14 | 15 | # By default the Model Server container image uses the AI Model stored in the model/model.file file. 16 | WORKDIR /model 17 | 18 | ADD $MODEL_URL /model/model.file 19 | -------------------------------------------------------------------------------- /models/README.md: -------------------------------------------------------------------------------- 1 | # Directory to store model files 2 | 3 | The models directory stores models and provides automation around downloading models. 4 | 5 | Want to try one of our tested models? Try one or all of the following: 6 | 7 | ```bash 8 | make download-model-granite 9 | make download-model-merlinite 10 | make download-model-mistral 11 | make download-model-mistral-code 12 | make download-model-whisper-small 13 | ``` 14 | 15 | Want to download and run a model you don't see listed? This is supported with the `MODEL_NAME` and `MODEL_URL` params: 16 | 17 | ```bash 18 | make download-model MODEL_URL=https://huggingface.co/TheBloke/openchat-3.5-0106-GGUF/resolve/main/openchat-3.5-0106.Q4_K_M.gguf MODEL_NAME=openchat-3.5-0106.Q4_K_M.gguf 19 | ``` 20 | -------------------------------------------------------------------------------- /models/download_hf_models.py: -------------------------------------------------------------------------------- 1 | from huggingface_hub import snapshot_download, hf_hub_download ,HfFileSystem 2 | import argparse 3 | 4 | parser = argparse.ArgumentParser() 5 | parser.add_argument("-m", "--model") 6 | parser.add_argument("-o", "--output", default="./models") 7 | parser.add_argument("-q", "--quantization", default="Q4_K_M") 8 | args = parser.parse_args() 9 | 10 | gguf = False 11 | safetensor = False 12 | ignore_patterns = ["*.md", ".gitattributes"] 13 | 14 | fs = HfFileSystem() 15 | files = fs.ls(args.model, detail=False) 16 | 17 | for f in files: 18 | if ".gguf" in f: 19 | gguf = True 20 | break 21 | if ".safetensor" in f: 22 | safetensor = True 23 | break 24 | 25 | if gguf: 26 | file_name = [x for x in files if args.quantization in x][0] 27 | file_name_parts = file_name.split("/") 28 | local_dir = f"{args.output}/{file_name_parts[1]}" 29 | hf_hub_download(repo_id=f"{file_name_parts[0]}/{file_name_parts[1]}", 30 | filename=file_name_parts[2], 31 | local_dir=local_dir, 32 | local_dir_use_symlinks=False 33 | ) 34 | else: 35 | if safetensor: 36 | ignore_patterns.append("*.bin") 37 | 38 | file_name_parts = args.model.split("/") 39 | local_dir = f"{args.output}/{file_name_parts[1]}" 40 | snapshot_download(repo_id=args.model, 41 | local_dir=local_dir, 42 | local_dir_use_symlinks=False, 43 | ignore_patterns=ignore_patterns, 44 | ) -------------------------------------------------------------------------------- /recipes/audio/audio_to_text/Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | APP ?= audio-to-text 3 | PORT ?= 8501 4 | 5 | include ../../common/Makefile.common 6 | 7 | RECIPE_BINARIES_PATH := $(shell realpath ../../common/bin) 8 | RELATIVE_MODELS_PATH := ../../../models 9 | RELATIVE_TESTS_PATH := ../tests 10 | -------------------------------------------------------------------------------- /recipes/audio/audio_to_text/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: audio 4 | name: Audio Transcription 5 | description: Transcribe audio files via speech recognition. 6 | containers: 7 | - name: whispercpp-server 8 | contextdir: ../../../model_servers/whispercpp 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - whisper-cpp 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8001 18 | image: quay.io/ai-lab/whispercpp:latest 19 | - name: whispercpp-app 20 | contextdir: app 21 | containerfile: Containerfile 22 | arch: 23 | - arm64 24 | - amd64 25 | ports: 26 | - 8501 27 | image: quay.io/redhat-et/locallm-whisper-client:latest 28 | -------------------------------------------------------------------------------- /recipes/audio/audio_to_text/app/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | WORKDIR /locallm 3 | COPY requirements.txt /locallm/requirements.txt 4 | RUN pip install --upgrade pip && \ 5 | pip install --no-cache-dir --upgrade -r requirements.txt 6 | COPY whisper_client.py whisper_client.py 7 | EXPOSE 8501 8 | ENTRYPOINT [ "streamlit", "run", "whisper_client.py" ] 9 | -------------------------------------------------------------------------------- /recipes/audio/audio_to_text/app/requirements.txt: -------------------------------------------------------------------------------- 1 | streamlit 2 | requests 3 | ffmpeg -------------------------------------------------------------------------------- /recipes/audio/audio_to_text/app/whisper_client.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | import requests 3 | import os 4 | import ffmpeg 5 | 6 | st.set_page_config(page_title="Whisper Speech Recognition", page_icon=":studio_microphone:") 7 | st.title(":studio_microphone: Speech Recognition") 8 | st.markdown("Upload an audio file you wish to have translated") 9 | endpoint = os.getenv("MODEL_ENDPOINT", default="http://0.0.0.0:8001") 10 | endpoint = f"{endpoint}/inference" 11 | endpoint_bearer = os.getenv("MODEL_ENDPOINT_BEARER") 12 | request_kwargs = {} 13 | if endpoint_bearer is not None: 14 | request_kwargs["headers"] = {"Authorization": f"Bearer {endpoint_bearer}"} 15 | audio = st.file_uploader("", type=["wav","mp3","mp4","flac"], accept_multiple_files=False) 16 | # read audio file 17 | if audio: 18 | audio_bytes = audio.read() 19 | st.audio(audio_bytes, format='audio/wav', start_time=0) 20 | request_kwargs["files"] = {'file': audio_bytes} 21 | response = requests.post(endpoint, **request_kwargs) 22 | response_json = response.json() 23 | st.subheader(f"Translated Text") 24 | st.text_area(label="", value=response_json['text'], height=300) 25 | else: 26 | st.write("Input not provided") 27 | -------------------------------------------------------------------------------- /recipes/audio/audio_to_text/bootc/Containerfile.nocache: -------------------------------------------------------------------------------- 1 | # Example: an AI powered sample application is embedded as a systemd service 2 | # via Podman quadlet files in /usr/share/containers/systemd 3 | # 4 | # from recipes/natural_language_processing/audio-to-text, run 5 | # 'make bootc' 6 | 7 | FROM quay.io/centos-bootc/centos-bootc:stream9 8 | ARG SSHPUBKEY 9 | 10 | # The --build-arg "SSHPUBKEY=$(cat ~/.ssh/id_rsa.pub)" option inserts your 11 | # public key into the image, allowing root access via ssh. 12 | RUN set -eu; mkdir -p /usr/ssh && \ 13 | echo 'AuthorizedKeysFile /usr/ssh/%u.keys .ssh/authorized_keys .ssh/authorized_keys2' >> /etc/ssh/sshd_config.d/30-auth-system.conf && \ 14 | echo ${SSHPUBKEY} > /usr/ssh/root.keys && chmod 0600 /usr/ssh/root.keys 15 | 16 | ARG RECIPE=audio-to-text 17 | 18 | # Add quadlet files to setup system to automatically run AI application on boot 19 | COPY build/${RECIPE}.image build/${RECIPE}.kube build/${RECIPE}.yaml /usr/share/containers/systemd 20 | 21 | # Added for running as an OCI Container to prevent Overlay on Overlay issues. 22 | VOLUME /var/lib/containers 23 | 24 | EXPOSE 8501 25 | -------------------------------------------------------------------------------- /recipes/audio/audio_to_text/quadlet/README.md: -------------------------------------------------------------------------------- 1 | ### Run audio-to-text as a systemd service 2 | 3 | There are pre-built images and a pod definition to run this audio-to-text example application. 4 | This sample converts an audio waveform (.wav) file to text. 5 | 6 | To run locally, 7 | 8 | ```bash 9 | podman kube play ./build/audio-to-text.yaml 10 | ``` 11 | To monitor locally, 12 | 13 | ```bash 14 | podman pod list 15 | podman ps 16 | podman logs 17 | ``` 18 | 19 | The application should be accessible at `http://localhost:8501`. It will take a few minutes for the model to load. 20 | 21 | ### Run audio-to-text as a systemd service 22 | 23 | ```bash 24 | (cd ../;make quadlet) 25 | sudo cp ../build/audio-to-text.yaml ../build/audio-to-text.kube ../build/audio-to-text.image /usr/share/containers/systemd/ 26 | /usr/libexec/podman/quadlet --dryrun (optional) 27 | systemctl daemon-reload 28 | systemctl start audio-to-text 29 | ``` 30 | -------------------------------------------------------------------------------- /recipes/audio/audio_to_text/quadlet/audio-to-text.image: -------------------------------------------------------------------------------- 1 | [Install] 2 | WantedBy=audio-to-text.service 3 | 4 | [Image] 5 | Image=APP_IMAGE 6 | Image=MODEL_IMAGE 7 | Image=SERVER_IMAGE 8 | -------------------------------------------------------------------------------- /recipes/audio/audio_to_text/quadlet/audio-to-text.kube: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Python script to run against downloaded LLM 3 | Documentation=man:podman-generate-systemd(1) 4 | Wants=network-online.target 5 | After=network-online.target 6 | RequiresMountsFor=%t/containers 7 | 8 | [Kube] 9 | # Point to the yaml file in the same directory 10 | Yaml=audio-to-text.yaml 11 | 12 | [Service] 13 | Restart=always 14 | 15 | [Install] 16 | WantedBy=default.target 17 | -------------------------------------------------------------------------------- /recipes/audio/audio_to_text/quadlet/audio-to-text.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | app: audio-to-text 6 | name: audio-to-text 7 | spec: 8 | initContainers: 9 | - name: model-file 10 | image: MODEL_IMAGE 11 | command: ['/usr/bin/install', "/model/ggml-small.bin", "/shared/"] 12 | volumeMounts: 13 | - name: model-file 14 | mountPath: /shared 15 | containers: 16 | - env: 17 | - name: MODEL_ENDPOINT 18 | value: http://0.0.0.0:8001 19 | image: APP_IMAGE 20 | name: audio-to-text 21 | ports: 22 | - containerPort: 8501 23 | hostPort: 8501 24 | securityContext: 25 | runAsNonRoot: true 26 | - env: 27 | - name: HOST 28 | value: 0.0.0.0 29 | - name: PORT 30 | value: 8001 31 | - name: MODEL_PATH 32 | value: /model/model.file 33 | image: SERVER_IMAGE 34 | name: whisper-model-service 35 | ports: 36 | - containerPort: 8001 37 | hostPort: 8001 38 | securityContext: 39 | runAsNonRoot: true 40 | volumeMounts: 41 | - name: model-file 42 | mountPath: /model 43 | volumes: 44 | - name: model-file 45 | emptyDir: {} 46 | -------------------------------------------------------------------------------- /recipes/common/README_bootc_image_builder.md: -------------------------------------------------------------------------------- 1 | This tools allows you to build and deploy disk-images from bootc container inputs. 2 | 3 | The following image disk types are currently available: 4 | 5 | | Image type | Target environment | 6 | |-----------------------|---------------------------------------------------------------------------------------| 7 | | `ami` | [Amazon Machine Image](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) | 8 | | `qcow2` **(default)** | [QEMU](https://www.qemu.org/) | 9 | | `vmdk` | [VMDK](https://en.wikipedia.org/wiki/VMDK) usable in vSphere, among others | 10 | | `anaconda-iso` | An unattended Anaconda installer that installs to the first disk found. | 11 | | `raw` | Unformatted [raw disk](https://en.wikipedia.org/wiki/Rawdisk). | 12 | 13 | The recipe Makefile can be used to automatically generate a disk image from the bootc image. The following 14 | command will create an qcow2 image file from the default bootc image in the build subdirectory. 15 | 16 | `make bootc-image-builder DISK_TYPE=qcow2` 17 | -------------------------------------------------------------------------------- /recipes/common/bin/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/recipes/common/bin/.gitkeep -------------------------------------------------------------------------------- /recipes/common/quadlet/app.image: -------------------------------------------------------------------------------- 1 | [Install] 2 | WantedBy=APP.service 3 | 4 | [Service] 5 | TimeoutStartSec=infinity 6 | 7 | [Image] 8 | Image=APP_IMAGE 9 | Image=MODEL_IMAGE 10 | Image=SERVER_IMAGE 11 | -------------------------------------------------------------------------------- /recipes/common/usr/lib/systemd/system/bootc-generic-growpart.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Bootc Fallback Root Filesystem Grow 3 | Documentation=https://gitlab.com/fedora/bootc/docs 4 | # For now we skip bare metal cases, and we also have nothing to do 5 | # for containers. 6 | ConditionVirtualization=vm 7 | # This helps verify that we're running in a bootc/ostree based target. 8 | ConditionPathIsMountPoint=/sysroot 9 | # We want to run before any e.g. large container images might be pulled. 10 | DefaultDependencies=no 11 | Requires=sysinit.target 12 | After=sysinit.target 13 | Before=basic.target 14 | 15 | [Service] 16 | ExecStart=/usr/libexec/bootc-generic-growpart 17 | # So we can temporarily remount the sysroot writable 18 | MountFlags=slave 19 | # Just to auto-cleanup our temporary files 20 | PrivateTmp=yes 21 | -------------------------------------------------------------------------------- /recipes/common/usr/lib/systemd/system/local-fs.target.wants/bootc-generic-growpart.service: -------------------------------------------------------------------------------- 1 | ../bootc-generic-growpart.service -------------------------------------------------------------------------------- /recipes/common/usr/libexec/bootc-generic-growpart: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eu 3 | 4 | backing_device=$(findmnt -vno SOURCE /sysroot) 5 | echo "Backing device: ${backing_device}" 6 | syspath=/sys/class/block/$(basename "${backing_device}") 7 | if ! test -d "${syspath}"; then 8 | echo "failed to find backing device ${syspath}"; exit 1 9 | fi 10 | 11 | # Handling devicemapper targets is a whole other thing 12 | case $backing_device in 13 | /dev/mapper/*) "Not growing $backing_device"; exit 0 ;; 14 | esac 15 | 16 | # Note that we expect that the rootfs is on a partition 17 | partition=$(cat "${syspath}"/partition) 18 | 19 | # Walk up to find the parent blockdev 20 | parentpath=$(dirname "$(realpath "${syspath}")") 21 | devmajmin=$(cat "${parentpath}"/dev) 22 | parent="/dev/block/${devmajmin}" 23 | 24 | # Grow the partition 25 | tmpf=$(mktemp) 26 | # Ignore errors because growpart exits 1 if nothing changed; 27 | # we need to check the output for NOCHANGE: 28 | if ! /usr/bin/growpart "${parent}" "${partition}" > "${tmpf}"; then 29 | cat "${tmpf}" 30 | if grep -qEe '^NOCHANGE: ' "${tmpf}"; then 31 | exit 0 32 | fi 33 | echo "growpart failed" 34 | exit 1 35 | fi 36 | cat "${tmpf}" 37 | # Now, temporarily remount the sysroot writable in our mount namespace 38 | mount -o remount,rw /sysroot 39 | # And defer to systemd's growfs wrapper which handles dispatching on 40 | # the target filesystem type. 41 | /usr/lib/systemd/systemd-growfs /sysroot 42 | -------------------------------------------------------------------------------- /recipes/computer_vision/object_detection/Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | APP ?= object_detection_client 3 | PORT ?= 8501 4 | MODEL_NAME ?= facebook/detr-resnet-101 5 | 6 | include ../../common/Makefile.common 7 | 8 | .PHONY: functional-tests 9 | functional-tests: 10 | IMAGE_NAME=${IMAGE_NAME} REGISTRY=${REGISTRY} MODEL_NAME=${MODEL_NAME} pytest -vvv --driver=Chrome --driver-path=$(RECIPE_BINARIES_PATH)/chromedriver ${RELATIVE_TESTS_PATH}/functional 11 | 12 | RECIPE_BINARIES_PATH := $(shell realpath ../../common/bin) 13 | RELATIVE_MODELS_PATH := ../../../models 14 | RELATIVE_TESTS_PATH := ../tests 15 | 16 | -------------------------------------------------------------------------------- /recipes/computer_vision/object_detection/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: vision 4 | name: Object Detection 5 | description: Detect and classify objects in images. 6 | containers: 7 | - name: object-detection-server 8 | contextdir: ../../../model_servers/object_detection_python 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - pytorch 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8000 18 | image: quay.io/redhat-et/locallm-object-detection-server:latest 19 | - name: object-detection-client 20 | contextdir: ./app 21 | containerfile: Containerfile 22 | arch: 23 | - arm64 24 | - amd64 25 | ports: 26 | - 8501 27 | image: quay.io/redhat-et/locallm-object-detection-client:latest 28 | -------------------------------------------------------------------------------- /recipes/computer_vision/object_detection/app/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | WORKDIR /locallm 3 | COPY requirements.txt /locallm/requirements.txt 4 | RUN pip install --upgrade pip && \ 5 | pip install --no-cache-dir --upgrade -r requirements.txt 6 | COPY object_detection_client.py object_detection_client.py 7 | EXPOSE 8501 8 | ENTRYPOINT [ "streamlit", "run", "object_detection_client.py" ] 9 | -------------------------------------------------------------------------------- /recipes/computer_vision/object_detection/app/object_detection_client.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | import base64 3 | import requests 4 | from PIL import Image 5 | import os 6 | import io 7 | 8 | st.title("🕵️‍♀️ Object Detection") 9 | endpoint =os.getenv("MODEL_ENDPOINT", default = "http://0.0.0.0:8000") 10 | endpoint_bearer = os.getenv("MODEL_ENDPOINT_BEARER") 11 | headers = {"accept": "application/json", 12 | "Content-Type": "application/json"} 13 | if endpoint_bearer: 14 | headers["Authorization"] = f"Bearer {endpoint_bearer}" 15 | image = st.file_uploader("Upload Image") 16 | window = st.empty() 17 | 18 | if image: 19 | #Ensure image dimensions are appropriate 20 | img = Image.open(io.BytesIO(image.read())) 21 | scale_factor = (500 * 500)/(img.height * img.width) 22 | if scale_factor < 0.20: 23 | scale_factor = 0.20 24 | img = img.resize((int(img.width * scale_factor) , 25 | int(img.height * scale_factor))) 26 | window.image(img, use_column_width=True) 27 | # convert PIL image into bytes for post request 28 | bytes_io = io.BytesIO() 29 | if img.mode in ("RGBA", "P"): 30 | img = img.convert("RGB") 31 | img.save(bytes_io, "JPEG") 32 | img_bytes = bytes_io.getvalue() 33 | b64_image = base64.b64encode(img_bytes).decode('utf-8') 34 | data = {'image': b64_image} 35 | response = requests.post(f'{endpoint}/detection', headers=headers,json=data, verify=False) 36 | # parse response and display outputs 37 | response_json = response.json() 38 | image = response_json["image"] 39 | window.image(base64.b64decode(image), use_column_width=True) 40 | for box in response_json["boxes"]: 41 | st.markdown(box) 42 | -------------------------------------------------------------------------------- /recipes/computer_vision/object_detection/app/requirements.txt: -------------------------------------------------------------------------------- 1 | altair==5.3.0 2 | attrs==23.2.0 3 | blinker==1.7.0 4 | cachetools==5.3.3 5 | certifi==2024.2.2 6 | charset-normalizer==3.3.2 7 | click==8.1.8 8 | gitdb==4.0.12 9 | GitPython==3.1.44 10 | idna==3.7 11 | Jinja2==3.1.6 12 | jsonschema==4.21.1 13 | jsonschema-specifications==2023.12.1 14 | markdown-it-py==3.0.0 15 | MarkupSafe==2.1.5 16 | mdurl==0.1.2 17 | numpy==1.26.4 18 | packaging==24.0 19 | pandas==2.2.3 20 | pillow==10.3.0 21 | protobuf==4.25.3 22 | pyarrow==15.0.2 23 | pydeck==0.8.1b0 24 | Pygments==2.17.2 25 | python-dateutil==2.9.0.post0 26 | pytz==2024.1 27 | referencing==0.34.0 28 | requests==2.31.0 29 | rich==13.7.1 30 | rpds-py==0.18.1 31 | six==1.16.0 32 | smmap==5.0.2 33 | streamlit==1.33.0 34 | tenacity==8.2.3 35 | toml==0.10.2 36 | toolz==0.12.1 37 | tornado==6.4.2 38 | typing_extensions==4.11.0 39 | tzdata==2024.1 40 | urllib3==2.2.3 41 | -------------------------------------------------------------------------------- /recipes/computer_vision/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | 4 | 5 | @pytest.fixture 6 | def chrome_options(chrome_options): 7 | chrome_options.add_argument("--headless") 8 | return chrome_options 9 | -------------------------------------------------------------------------------- /recipes/computer_vision/tests/functional/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/recipes/computer_vision/tests/functional/__init__.py -------------------------------------------------------------------------------- /recipes/computer_vision/tests/functional/test_app.py: -------------------------------------------------------------------------------- 1 | import pytest_container 2 | from .conftest import CB 3 | import tenacity 4 | 5 | CONTAINER_IMAGES = [CB] 6 | 7 | def test_etc_os_release_present(auto_container: pytest_container.container.ContainerData): 8 | assert auto_container.connection.file("/etc/os-release").exists 9 | 10 | @tenacity.retry(stop=tenacity.stop_after_attempt(5), wait=tenacity.wait_exponential()) 11 | def test_alive(auto_container: pytest_container.container.ContainerData, host): 12 | host.run_expect([0],f"curl http://localhost:{auto_container.forwarded_ports[0].host_port}",).stdout.strip() 13 | 14 | def test_title(auto_container: pytest_container.container.ContainerData, selenium): 15 | selenium.get(f"http://localhost:{auto_container.forwarded_ports[0].host_port}") 16 | assert selenium.title == "Streamlit" 17 | 18 | -------------------------------------------------------------------------------- /recipes/computer_vision/tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/recipes/computer_vision/tests/integration/__init__.py -------------------------------------------------------------------------------- /recipes/computer_vision/tests/integration/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | 4 | 5 | @pytest.fixture() 6 | def url(): 7 | return os.environ["URL"] 8 | -------------------------------------------------------------------------------- /recipes/computer_vision/tests/integration/test_app.py: -------------------------------------------------------------------------------- 1 | def test_title(url,selenium): 2 | selenium.get(f"http://{url}:8501") 3 | assert selenium.title == "Streamlit" 4 | -------------------------------------------------------------------------------- /recipes/computer_vision/tests/requirements.txt: -------------------------------------------------------------------------------- 1 | pip==24.0 2 | pytest-container==0.4.0 3 | pytest-selenium==4.1.0 4 | pytest-testinfra==10.1.0 5 | pytest==8.1.1 6 | requests==2.31.0 7 | selenium==4.19.0 8 | tenacity==8.2.3 9 | -------------------------------------------------------------------------------- /recipes/multimodal/image_understanding/README.md: -------------------------------------------------------------------------------- 1 | # Image Analysis 2 | 3 | ### Build image 4 | ```bash 5 | cd image_understanding 6 | podman build -t image_understanding app 7 | ``` 8 | 9 | ### Run the Multimodal Model Server: 10 | 11 | #### 1. Get a Multimodal Model: 12 | 13 | For this example we will use the pre-quantized gguf version of LLava-v1.5. You can find the model files here: 14 | 15 | https://huggingface.co/mys/ggml_llava-v1.5-7b/tree/main 16 | 17 | The multimodal models require two gguf files. Please download the following two files to `models/`. 18 | 19 | * [ggml-model-q4_k.gguf](https://huggingface.co/mys/ggml_llava-v1.5-7b/tree/main) 20 | * [mmproj-model-f16.gguf](https://huggingface.co/mys/ggml_llava-v1.5-7b/tree/main) 21 | 22 | 23 | 24 | #### 2. Run Multi-Modal Model Server: 25 | Once you have the model files you can run the model server image locally. 26 | ```bash 27 | podman run -it -p 8001:8001 -v /locallm/models:/locallm/models:Z -e MODEL_PATH=models/ggml-model-q4_k.gguf -e CLIP_MODEL_PATH=models/mmproj-model-f16.gguf -e CHAT_FORMAT=llava-1-5 -e HOST=0.0.0.0 -e PORT=8001 playground 28 | ``` 29 | 30 | ### Run AI Application Image Locally 31 | 32 | ```bash 33 | podman run --rm -it -p 8501:8501 -e MODEL_ENDPOINT=http://10.88.0.1:8001 image_understanding 34 | ``` 35 | 36 | Interact with the application from your local browser at `localhost:8501`. You can upload an image file from your host machine and the app will provide a natural language description of the image. 37 | 38 | 39 | ![](/assets/image_analysis.png) 40 | -------------------------------------------------------------------------------- /recipes/multimodal/image_understanding/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | application: 2 | type: Multimodal 3 | name: image-understanding 4 | description: Analyze and describe the content of images. 5 | containers: 6 | - name: llamacpp-server 7 | contextdir: ../../../model_servers/llamacpp_python 8 | containerfile: ./base/Containerfile 9 | model-service: true 10 | backend: 11 | - llama-cpp 12 | arch: 13 | - arm64 14 | - amd64 15 | - name: image-understanding-inference-app 16 | contextdir: app 17 | containerfile: Containerfile 18 | arch: 19 | - arm64 20 | - amd64 21 | -------------------------------------------------------------------------------- /recipes/multimodal/image_understanding/app/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | WORKDIR /image_understanding 3 | COPY requirements.txt . 4 | RUN pip install --upgrade pip && \ 5 | pip install --no-cache-dir --upgrade -r requirements.txt 6 | COPY image_understanding.py . 7 | EXPOSE 8501 8 | ENTRYPOINT [ "streamlit", "run", "image_understanding.py" ] 9 | -------------------------------------------------------------------------------- /recipes/multimodal/image_understanding/app/image_understanding.py: -------------------------------------------------------------------------------- 1 | from openai import OpenAI 2 | import streamlit as st 3 | import base64 4 | import os 5 | 6 | model_service = os.getenv("MODEL_ENDPOINT", 7 | default="http://localhost:8001") 8 | 9 | st.title("📷 Image Analysis") 10 | image = st.file_uploader("Upload Image:",) 11 | top_container = st.container(border=True) 12 | if image is not None: 13 | b64_image = base64.b64encode(image.read()).decode("utf-8") 14 | client = OpenAI(base_url=f'{model_service}/v1', 15 | api_key="sk-xxx") 16 | with st.spinner("Analyzing Image..."): 17 | st.image(image) 18 | response = client.chat.completions.create( 19 | model="", 20 | stream=True, 21 | messages=[ 22 | { 23 | "role": "user", 24 | "content": [ 25 | { 26 | "type": "image_url", 27 | "image_url": { 28 | "url": f"data:image/png;base64,{b64_image}" 29 | }, 30 | }, 31 | {"type": "text", 32 | "text": "What does the image say"}, 33 | ], 34 | } 35 | ], 36 | ) 37 | top_container.chat_message("assistant").write_stream(response) 38 | -------------------------------------------------------------------------------- /recipes/multimodal/image_understanding/app/requirements.txt: -------------------------------------------------------------------------------- 1 | openai 2 | streamlit -------------------------------------------------------------------------------- /recipes/natural_language_processing/agents/Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | APP ?= react_agent 3 | PORT ?= 8501 4 | 5 | include ../../common/Makefile.common 6 | 7 | RECIPE_BINARIES_PATH := $(shell realpath ../../common/bin) 8 | RELATIVE_MODELS_PATH := ../../../models 9 | RELATIVE_TESTS_PATH := ../tests 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/agents/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: language 4 | name: ReAct_Agent_Streamlit 5 | description: ReAct framework implementation with Spotify API integration in a web frontend 6 | containers: 7 | - name: llamacpp-server 8 | contextdir: ../../../model_servers/llamacpp_python 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - llama-cpp 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8001 18 | image: quay.io/ai-lab/llamacpp_python:latest 19 | - name: streamlit-react-agent-app 20 | contextdir: app 21 | containerfile: Containerfile 22 | arch: 23 | - arm64 24 | - amd64 25 | ports: 26 | - 8501 27 | image: quay.io/ai-lab/react-agent:latest 28 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/agents/app/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | WORKDIR /agents 3 | COPY requirements.txt . 4 | RUN pip install --upgrade pip 5 | RUN pip install --no-cache-dir --upgrade -r /agents/requirements.txt 6 | COPY *.py . 7 | EXPOSE 8501 8 | ENTRYPOINT [ "streamlit", "run", "react-agent-app.py" ] 9 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/agents/app/requirements.txt: -------------------------------------------------------------------------------- 1 | streamlit>=1.24.0 2 | langchain-core>=0.1.0 3 | langchain-openai>=0.0.5 4 | python-dotenv>=0.19.0 5 | 6 | requests>=2.31.0 7 | 8 | typing-extensions>=4.5.0 9 | 10 | streamlit-chat>=0.1.1 11 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-java-quarkus/README.md: -------------------------------------------------------------------------------- 1 | # Java-based chatbot application - Quarkus 2 | 3 | This application implements a simple chatbot backed by Quarkus and its 4 | LangChain4j extension. The UI communicates with the backend application via 5 | web sockets and the backend uses the OpenAI API to talk to the model served 6 | by Podman AI Lab. 7 | 8 | Documentation for Quarkus+LangChain4j can be found at 9 | https://docs.quarkiverse.io/quarkus-langchain4j/dev/. 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-java-quarkus/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: language 4 | name: ChatBot_Java_Quarkus 5 | description: Chatbot sample based on Quarkus 6 | containers: 7 | - name: llamacpp-server 8 | contextdir: ../../../model_servers/llamacpp_python 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - llama-cpp 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8001 18 | image: quay.io/ai-lab/llamacpp_python:latest 19 | - name: quarkus-chat-app 20 | contextdir: app 21 | containerfile: Containerfile 22 | arch: 23 | - arm64 24 | - amd64 25 | ports: 26 | - 8080 -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-java-quarkus/app/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | wrapperVersion=3.3.2 18 | distributionType=only-script 19 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip 20 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-java-quarkus/app/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi8/openjdk-21:latest 2 | WORKDIR /app 3 | COPY --chown=185:0 --chmod=744 . . 4 | RUN mvn package 5 | EXPOSE 8080 6 | ENV JAVA_OPTS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" 7 | ENV JAVA_APP_JAR="/app/target/quarkus-app/quarkus-run.jar" 8 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-java-quarkus/app/src/main/java/io/quarkiverse/langchain4j/sample/chatbot/Bot.java: -------------------------------------------------------------------------------- 1 | package io.quarkiverse.langchain4j.sample.chatbot; 2 | 3 | import dev.langchain4j.service.SystemMessage; 4 | import dev.langchain4j.service.UserMessage; 5 | import io.quarkiverse.langchain4j.RegisterAiService; 6 | import io.smallrye.mutiny.Multi; 7 | import jakarta.enterprise.context.SessionScoped; 8 | 9 | @RegisterAiService 10 | @SessionScoped 11 | public interface Bot { 12 | 13 | Multi chat(@UserMessage String question); 14 | 15 | } 16 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-java-quarkus/app/src/main/java/io/quarkiverse/langchain4j/sample/chatbot/ChatBotWebSocket.java: -------------------------------------------------------------------------------- 1 | package io.quarkiverse.langchain4j.sample.chatbot; 2 | 3 | import io.quarkus.websockets.next.OnOpen; 4 | import io.quarkus.websockets.next.OnTextMessage; 5 | import io.quarkus.websockets.next.WebSocket; 6 | import io.smallrye.mutiny.Multi; 7 | 8 | @WebSocket(path = "/chatbot") 9 | public class ChatBotWebSocket { 10 | 11 | private final Bot bot; 12 | 13 | public ChatBotWebSocket(Bot bot) { 14 | this.bot = bot; 15 | } 16 | 17 | @OnTextMessage 18 | public Multi onMessage(String message) { 19 | return bot.chat(message).onFailure().recoverWithItem(t -> 20 | "There was an error in communicating with the model server"); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-java-quarkus/app/src/main/java/io/quarkiverse/langchain4j/sample/chatbot/ImportmapResource.java: -------------------------------------------------------------------------------- 1 | package io.quarkiverse.langchain4j.sample.chatbot; 2 | 3 | import jakarta.annotation.PostConstruct; 4 | import jakarta.enterprise.context.ApplicationScoped; 5 | import jakarta.ws.rs.GET; 6 | import jakarta.ws.rs.Path; 7 | import jakarta.ws.rs.Produces; 8 | 9 | import io.mvnpm.importmap.Aggregator; 10 | 11 | /** 12 | * Dynamically create the import map 13 | */ 14 | @ApplicationScoped 15 | @Path("/_importmap") 16 | public class ImportmapResource { 17 | private String importmap; 18 | 19 | // See https://github.com/WICG/import-maps/issues/235 20 | // This does not seem to be supported by browsers yet... 21 | @GET 22 | @Path("/dynamic.importmap") 23 | @Produces("application/importmap+json") 24 | public String importMap() { 25 | return this.importmap; 26 | } 27 | 28 | @GET 29 | @Path("/dynamic-importmap.js") 30 | @Produces("application/javascript") 31 | public String importMapJson() { 32 | return JAVASCRIPT_CODE.formatted(this.importmap); 33 | } 34 | 35 | @PostConstruct 36 | void init() { 37 | Aggregator aggregator = new Aggregator(); 38 | // Add our own mappings 39 | aggregator.addMapping("icons/", "/icons/"); 40 | aggregator.addMapping("components/", "/components/"); 41 | aggregator.addMapping("fonts/", "/fonts/"); 42 | this.importmap = aggregator.aggregateAsJson(); 43 | } 44 | 45 | private static final String JAVASCRIPT_CODE = """ 46 | const im = document.createElement('script'); 47 | im.type = 'importmap'; 48 | im.textContent = JSON.stringify(%s); 49 | document.currentScript.after(im); 50 | """; 51 | } 52 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-java-quarkus/app/src/main/resources/META-INF/resources/components/demo-title.js: -------------------------------------------------------------------------------- 1 | import {LitElement, html, css} from 'lit'; 2 | 3 | export class DemoTitle extends LitElement { 4 | 5 | static styles = css` 6 | h1 { 7 | font-family: "Red Hat Mono", monospace; 8 | font-size: 60px; 9 | font-style: normal; 10 | font-variant: normal; 11 | font-weight: 700; 12 | line-height: 26.4px; 13 | color: var(--main-highlight-text-color); 14 | } 15 | 16 | .title { 17 | text-align: center; 18 | padding: 1em; 19 | background: var(--main-bg-color); 20 | } 21 | 22 | .explanation { 23 | margin-left: auto; 24 | margin-right: auto; 25 | width: 50%; 26 | text-align: justify; 27 | font-size: 20px; 28 | } 29 | 30 | .explanation img { 31 | max-width: 60%; 32 | display: block; 33 | float:left; 34 | margin-right: 2em; 35 | margin-top: 1em; 36 | } 37 | ` 38 | 39 | render() { 40 | return html` 41 |
42 |

Chatbot

43 |
44 |
45 | This demo shows how to build a simple chatbot. 46 | Click the red robot button and start chatting. 47 |
48 | ` 49 | } 50 | 51 | } 52 | 53 | customElements.define('demo-title', DemoTitle); -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-java-quarkus/app/src/main/resources/META-INF/resources/images/chatbot-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/recipes/natural_language_processing/chatbot-java-quarkus/app/src/main/resources/META-INF/resources/images/chatbot-architecture.png -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-java-quarkus/app/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | quarkus.langchain4j.timeout=120s 2 | quarkus.langchain4j.openai.api-key=NOT-NECESSARY 3 | quarkus.langchain4j.openai.base-url=${MODEL_ENDPOINT}/v1/ -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-nodejs/Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | APP ?= chatbot-nodejs 3 | PORT ?= 8501 4 | 5 | include ../../common/Makefile.common 6 | 7 | RECIPE_BINARIES_PATH := $(shell realpath ../../common/bin) 8 | RELATIVE_MODELS_PATH := ../../../models 9 | RELATIVE_TESTS_PATH := ../tests 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-nodejs/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: language 4 | name: ChatBot_nodejs 5 | description: Chat with a model service in a web frontend. 6 | containers: 7 | - name: llamacpp-server 8 | contextdir: ../../../model_servers/llamacpp_python 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - llama-cpp 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8001 18 | image: quay.io/ai-lab/llamacpp_python:latest 19 | - name: nodejs-chat-app 20 | contextdir: app 21 | containerfile: Containerfile 22 | arch: 23 | - arm64 24 | - amd64 25 | ports: 26 | - 8501 27 | image: quay.io/ai-lab/chatbot-nodejs:latest 28 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-nodejs/app/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-nodejs/app/app/layout.tsx: -------------------------------------------------------------------------------- 1 | export const metadata = { 2 | title: 'Sample Node.js AI Chatbot', 3 | description: 'Sample Node.js AI Chatbot', 4 | } 5 | 6 | export default function RootLayout({ 7 | children, 8 | }: { 9 | children: React.ReactNode 10 | }) { 11 | return ( 12 | 13 | {children} 14 | 15 | ) 16 | } 17 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-nodejs/app/next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | module.exports = { 3 | output: "standalone", 4 | }; 5 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-nodejs/app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "app", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@langchain/community": "^0.2.18", 7 | "@langchain/core": "^0.2.14", 8 | "@langchain/openai": "^0.2.1", 9 | "@testing-library/jest-dom": "^5.17.0", 10 | "@testing-library/react": "^13.4.0", 11 | "@testing-library/user-event": "^13.5.0", 12 | "next": "^14.2.5", 13 | "react": "^18.3.1", 14 | "react-chatbotify": "^1.7.0", 15 | "react-dom": "^18.3.1", 16 | "react-scripts": "5.0.1", 17 | "socket.io": "^4.7.2", 18 | "socket.io-client": "^4.7.2", 19 | "web-vitals": "^2.1.4" 20 | }, 21 | "devDependencies": { 22 | "autoprefixer": "^10.4.20", 23 | "eslint": "^8", 24 | "eslint-config-next": "14.0.3", 25 | "postcss": "^8.4.39", 26 | "tailwindcss": "^3.4.16" 27 | }, 28 | "overrides": { 29 | "typescript": "^5.5.3", 30 | "@langchain/core": "^0.2.14" 31 | }, 32 | "scripts": { 33 | "dev": "next dev -p 8501", 34 | "build": "next build", 35 | "lint": "next lint" 36 | }, 37 | "eslintConfig": { 38 | "extends": [ 39 | "react-app", 40 | "react-app/jest" 41 | ] 42 | }, 43 | "browserslist": { 44 | "production": [ 45 | ">0.2%", 46 | "not dead", 47 | "not op_mini all" 48 | ], 49 | "development": [ 50 | "last 1 chrome version", 51 | "last 1 firefox version", 52 | "last 1 safari version" 53 | ] 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-nodejs/bootc/Containerfile.nocache: -------------------------------------------------------------------------------- 1 | # Example: an AI powered sample application is embedded as a systemd service 2 | # via Podman quadlet files in /usr/share/containers/systemd 3 | # 4 | # from recipes/natural_language_processing/chatbot-nodejs, run 5 | # 'make bootc' 6 | 7 | FROM quay.io/centos-bootc/centos-bootc:stream9 8 | ARG SSHPUBKEY 9 | 10 | # The --build-arg "SSHPUBKEY=$(cat ~/.ssh/id_rsa.pub)" option inserts your 11 | # public key into the image, allowing root access via ssh. 12 | RUN set -eu; mkdir -p /usr/ssh && \ 13 | echo 'AuthorizedKeysFile /usr/ssh/%u.keys .ssh/authorized_keys .ssh/authorized_keys2' >> /etc/ssh/sshd_config.d/30-auth-system.conf && \ 14 | echo ${SSHPUBKEY} > /usr/ssh/root.keys && chmod 0600 /usr/ssh/root.keys 15 | 16 | ARG RECIPE=chatbot-nodejs 17 | 18 | # Include growfs service 19 | COPY build/usr/lib /usr/lib 20 | COPY --chmod=0755 build/usr/libexec/bootc-generic-growpart /usr/libexec/bootc-generic-growpart 21 | 22 | # Add quadlet files to setup system to automatically run AI application on boot 23 | COPY build/${RECIPE}.image build/${RECIPE}.kube build/${RECIPE}.yaml /usr/share/containers/systemd 24 | 25 | # Added for running as an OCI Container to prevent Overlay on Overlay issues. 26 | VOLUME /var/lib/containers 27 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-nodejs/quadlet/README.md: -------------------------------------------------------------------------------- 1 | ### Run chatbot as a systemd service 2 | 3 | ```bash 4 | (cd ../;make quadlet) 5 | sudo cp ../build/chatbot-nodejs.yaml ../build/chatbot-nodejs.kube ../build/chatbot-nodejs.image /usr/share/containers/systemd/ 6 | sudo /usr/libexec/podman/quadlet --dryrun #optional 7 | sudo systemctl daemon-reload 8 | sudo systemctl start chatbot-nodejs 9 | ``` 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-nodejs/quadlet/chatbot-nodejs.image: -------------------------------------------------------------------------------- 1 | ../../../common/quadlet/app.image -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-nodejs/quadlet/chatbot-nodejs.kube: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes YAML file used to do chatbot inferencing 3 | Documentation=man:podman-generate-systemd(1) 4 | Wants=network-online.target 5 | After=network-online.target 6 | RequiresMountsFor=%t/containers 7 | 8 | [Kube] 9 | # Point to the yaml file in the same directory 10 | Yaml=chatbot-nodejs.yaml 11 | 12 | [Service] 13 | Restart=always 14 | 15 | [Install] 16 | WantedBy=default.target 17 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-nodejs/quadlet/chatbot-nodejs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | app: chatbot-nodejs 6 | name: chatbot-nodejs 7 | spec: 8 | initContainers: 9 | - name: model-file 10 | image: MODEL_IMAGE 11 | command: ['/usr/bin/install', "/model/model.file", "/shared/"] 12 | volumeMounts: 13 | - name: model-file 14 | mountPath: /shared 15 | containers: 16 | - env: 17 | - name: MODEL_ENDPOINT 18 | value: http://0.0.0.0:8001 19 | image: APP_IMAGE 20 | name: chatbot-nodejs-inference 21 | ports: 22 | - containerPort: 8501 23 | hostPort: 8501 24 | securityContext: 25 | runAsNonRoot: true 26 | - env: 27 | - name: HOST 28 | value: 0.0.0.0 29 | - name: PORT 30 | value: 8001 31 | - name: MODEL_PATH 32 | value: /model/model.file 33 | image: SERVER_IMAGE 34 | name: chatbot-model-service 35 | ports: 36 | - containerPort: 8001 37 | hostPort: 8001 38 | securityContext: 39 | runAsNonRoot: true 40 | volumeMounts: 41 | - name: model-file 42 | mountPath: /model 43 | volumes: 44 | - name: model-file 45 | emptyDir: {} 46 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-pydantic-ai/Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | APP ?= chatbot-pydantic-ai 3 | PORT ?= 8501 4 | 5 | include ../../common/Makefile.common 6 | 7 | RECIPE_BINARIES_PATH := $(shell realpath ../../common/bin) 8 | RELATIVE_MODELS_PATH := ../../../models 9 | RELATIVE_TESTS_PATH := ../tests -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-pydantic-ai/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: language 4 | name: chatbot-pydantic-ai 5 | description: Chat with a model service in a web frontend with history using pydantic ai framework. 6 | containers: 7 | - name: llamacpp-server 8 | contextdir: ../../../model_servers/llamacpp_python 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - llama-cpp 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8001 18 | image: quay.io/ai-lab/llamacpp_python:latest 19 | - name: streamlit-chat-app 20 | contextdir: app 21 | containerfile: Containerfile 22 | arch: 23 | - arm64 24 | - amd64 25 | ports: 26 | - 8501 27 | image: quay.io/ai-lab/chatbot-pydantic-ai:latest 28 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-pydantic-ai/app/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | WORKDIR /chatbot-pydantic-ai 3 | # Ensure the default user has the correct permissions on the working directory 4 | RUN chown -R 1001:0 /chatbot-pydantic-ai 5 | COPY requirements.txt . 6 | RUN pip install --upgrade pip 7 | RUN pip install --no-cache-dir --upgrade -r /chatbot-pydantic-ai/requirements.txt 8 | COPY chatbot-pydantic-ai.py . 9 | EXPOSE 8501 10 | ENTRYPOINT [ "streamlit", "run", "chatbot-pydantic-ai.py" ] -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-pydantic-ai/app/requirements.txt: -------------------------------------------------------------------------------- 1 | streamlit==1.41.1 2 | pydantic-ai==0.0.55 -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-pydantic-ai/bootc/Containerfile.nocache: -------------------------------------------------------------------------------- 1 | # Example: an AI powered sample application is embedded as a systemd service 2 | # via Podman quadlet files in /usr/share/containers/systemd 3 | # 4 | # from recipes/natural_language_processing/chatbot, run 5 | # 'make bootc' 6 | 7 | FROM quay.io/centos-bootc/centos-bootc:stream9 8 | ARG SSHPUBKEY 9 | 10 | # The --build-arg "SSHPUBKEY=$(cat ~/.ssh/id_rsa.pub)" option inserts your 11 | # public key into the image, allowing root access via ssh. 12 | RUN set -eu; mkdir -p /usr/ssh && \ 13 | echo 'AuthorizedKeysFile /usr/ssh/%u.keys .ssh/authorized_keys .ssh/authorized_keys2' >> /etc/ssh/sshd_config.d/30-auth-system.conf && \ 14 | echo ${SSHPUBKEY} > /usr/ssh/root.keys && chmod 0600 /usr/ssh/root.keys 15 | 16 | ARG RECIPE=chatbot-pydantic-ai 17 | 18 | # Include growfs service 19 | COPY build/usr/lib /usr/lib 20 | COPY --chmod=0755 build/usr/libexec/bootc-generic-growpart /usr/libexec/bootc-generic-growpart 21 | 22 | # Add quadlet files to setup system to automatically run AI application on boot 23 | COPY build/${RECIPE}.image build/${RECIPE}.kube build/${RECIPE}.yaml /usr/share/containers/systemd 24 | 25 | # Added for running as an OCI Container to prevent Overlay on Overlay issues. 26 | VOLUME /var/lib/containers 27 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-pydantic-ai/provision/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Test Environment Provisioning 3 | hosts: test_environments 4 | remote_user: fedora 5 | become: true 6 | gather_facts: false 7 | 8 | tasks: 9 | 10 | - name: Wait until the instance is ready 11 | ansible.builtin.wait_for_connection: 12 | delay: 10 13 | timeout: 60 14 | 15 | - name: Gather facts for first time 16 | ansible.builtin.setup: 17 | 18 | - name: Required Packages 19 | ansible.builtin.package: 20 | name: "{{ item }}" 21 | state: present 22 | with_items: 23 | - podman 24 | - python3-libdnf5 25 | 26 | - name: Models host directory 27 | ansible.builtin.file: 28 | path: locallm/models 29 | state: directory 30 | 31 | - name: Download Model 32 | ansible.builtin.get_url: 33 | url: https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_S.gguf 34 | dest: locallm/models 35 | 36 | - name: Run Model 37 | containers.podman.podman_container: 38 | name: llamacpp_python 39 | image: ghcr.io/containers/llamacpp_python:latest 40 | state: started 41 | interactive: true 42 | tty: true 43 | detach: true 44 | ports: 45 | - 8001:8001 46 | volume: 47 | - ./locallm/models:/locallm/models:ro,Z 48 | env: 49 | MODEL_PATH: models/llama-2-7b-chat.Q5_K_S.gguf 50 | HOST: 0.0.0.0 51 | PORT: 8001 52 | 53 | - name: Run Application 54 | containers.podman.podman_container: 55 | name: chatbot 56 | image: ghcr.io/containers/chatbot-pydantic-ai:latest 57 | state: started 58 | interactive: true 59 | tty: true 60 | ports: 61 | - 8501:8501 62 | env: 63 | MODEL_ENDPOINT: http://10.88.0.1:8001 64 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-pydantic-ai/provision/requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | collections: 3 | - name: containers.podman 4 | version: 1.13.0 5 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-pydantic-ai/quadlet/chatbot-pydantic-ai.image: -------------------------------------------------------------------------------- 1 | [Install] 2 | WantedBy=APP.service 3 | 4 | [Service] 5 | TimeoutStartSec=infinity 6 | 7 | [Image] 8 | Image=APP_IMAGE 9 | Image=MODEL_IMAGE 10 | Image=SERVER_IMAGE -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-pydantic-ai/quadlet/chatbot-pydantic-ai.kube: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes YAML file used to do chatbot pydantic ai inferencing 3 | Documentation=man:podman-generate-systemd(1) 4 | Wants=network-online.target 5 | After=network-online.target 6 | RequiresMountsFor=%t/containers 7 | 8 | [Kube] 9 | # Point to the yaml file in the same directory 10 | Yaml=chatbot-pydantic-ai.yaml 11 | 12 | [Service] 13 | Restart=always 14 | 15 | [Install] 16 | WantedBy=default.target 17 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot-pydantic-ai/quadlet/chatbot-pydantic-ai.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | app: chatbot-pydantic-ai 6 | name: chatbot-pydantic-ai 7 | spec: 8 | initContainers: 9 | - name: model-file 10 | image: MODEL_IMAGE 11 | command: ['/usr/bin/install', "/model/model.file", "/shared/"] 12 | volumeMounts: 13 | - name: model-file 14 | mountPath: /shared 15 | containers: 16 | - env: 17 | - name: MODEL_ENDPOINT 18 | value: http://0.0.0.0:8001 19 | image: APP_IMAGE 20 | name: chatbot-pydantic-ai-inference 21 | ports: 22 | - containerPort: 8501 23 | hostPort: 8501 24 | securityContext: 25 | runAsNonRoot: true 26 | - env: 27 | - name: HOST 28 | value: 0.0.0.0 29 | - name: PORT 30 | value: 8001 31 | - name: MODEL_PATH 32 | value: /model/model.file 33 | image: SERVER_IMAGE 34 | name: chatbot-pydantic-ai-model-service 35 | ports: 36 | - containerPort: 8001 37 | hostPort: 8001 38 | securityContext: 39 | runAsNonRoot: true 40 | volumeMounts: 41 | - name: model-file 42 | mountPath: /model 43 | volumes: 44 | - name: model-file 45 | emptyDir: {} 46 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot/Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | APP ?= chatbot 3 | PORT ?= 8501 4 | 5 | include ../../common/Makefile.common 6 | 7 | RECIPE_BINARIES_PATH := $(shell realpath ../../common/bin) 8 | RELATIVE_MODELS_PATH := ../../../models 9 | RELATIVE_TESTS_PATH := ../tests 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: language 4 | name: ChatBot_Streamlit 5 | description: Chat with a model service in a web frontend. 6 | containers: 7 | - name: llamacpp-server 8 | contextdir: ../../../model_servers/llamacpp_python 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - llama-cpp 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8001 18 | image: quay.io/ai-lab/llamacpp_python:latest 19 | - name: streamlit-chat-app 20 | contextdir: app 21 | containerfile: Containerfile 22 | arch: 23 | - arm64 24 | - amd64 25 | ports: 26 | - 8501 27 | image: quay.io/ai-lab/chatbot:latest 28 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot/app/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | WORKDIR /chat 3 | COPY requirements.txt . 4 | RUN pip install --upgrade pip 5 | RUN pip install --no-cache-dir --upgrade -r /chat/requirements.txt 6 | COPY chatbot_ui.py . 7 | EXPOSE 8501 8 | ENTRYPOINT [ "streamlit", "run", "chatbot_ui.py" ] 9 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot/app/requirements.txt: -------------------------------------------------------------------------------- 1 | langchain==0.2.3 2 | langchain-openai==0.1.7 3 | langchain-community==0.2.4 4 | streamlit==1.34.0 -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot/bootc/Containerfile.nocache: -------------------------------------------------------------------------------- 1 | # Example: an AI powered sample application is embedded as a systemd service 2 | # via Podman quadlet files in /usr/share/containers/systemd 3 | # 4 | # from recipes/natural_language_processing/chatbot, run 5 | # 'make bootc' 6 | 7 | FROM quay.io/centos-bootc/centos-bootc:stream9 8 | ARG SSHPUBKEY 9 | 10 | # The --build-arg "SSHPUBKEY=$(cat ~/.ssh/id_rsa.pub)" option inserts your 11 | # public key into the image, allowing root access via ssh. 12 | RUN set -eu; mkdir -p /usr/ssh && \ 13 | echo 'AuthorizedKeysFile /usr/ssh/%u.keys .ssh/authorized_keys .ssh/authorized_keys2' >> /etc/ssh/sshd_config.d/30-auth-system.conf && \ 14 | echo ${SSHPUBKEY} > /usr/ssh/root.keys && chmod 0600 /usr/ssh/root.keys 15 | 16 | ARG RECIPE=chatbot 17 | 18 | # Include growfs service 19 | COPY build/usr/lib /usr/lib 20 | COPY --chmod=0755 build/usr/libexec/bootc-generic-growpart /usr/libexec/bootc-generic-growpart 21 | 22 | # Add quadlet files to setup system to automatically run AI application on boot 23 | COPY build/${RECIPE}.image build/${RECIPE}.kube build/${RECIPE}.yaml /usr/share/containers/systemd 24 | 25 | # Added for running as an OCI Container to prevent Overlay on Overlay issues. 26 | VOLUME /var/lib/containers 27 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot/provision/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Test Environment Provisioning 3 | hosts: test_environments 4 | remote_user: fedora 5 | become: true 6 | gather_facts: false 7 | 8 | tasks: 9 | 10 | - name: Wait until the instance is ready 11 | ansible.builtin.wait_for_connection: 12 | delay: 10 13 | timeout: 60 14 | 15 | - name: Gather facts for first time 16 | ansible.builtin.setup: 17 | 18 | - name: Required Packages 19 | ansible.builtin.package: 20 | name: "{{ item }}" 21 | state: present 22 | with_items: 23 | - podman 24 | - python3-libdnf5 25 | 26 | - name: Models host directory 27 | ansible.builtin.file: 28 | path: locallm/models 29 | state: directory 30 | 31 | - name: Download Model 32 | ansible.builtin.get_url: 33 | url: https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_S.gguf 34 | dest: locallm/models 35 | 36 | - name: Run Model 37 | containers.podman.podman_container: 38 | name: llamacpp_python 39 | image: ghcr.io/containers/llamacpp_python:latest 40 | state: started 41 | interactive: true 42 | tty: true 43 | detach: true 44 | ports: 45 | - 8001:8001 46 | volume: 47 | - ./locallm/models:/locallm/models:ro,Z 48 | env: 49 | MODEL_PATH: models/llama-2-7b-chat.Q5_K_S.gguf 50 | HOST: 0.0.0.0 51 | PORT: 8001 52 | 53 | - name: Run Application 54 | containers.podman.podman_container: 55 | name: chatbot 56 | image: ghcr.io/containers/chatbot:latest 57 | state: started 58 | interactive: true 59 | tty: true 60 | ports: 61 | - 8501:8501 62 | env: 63 | MODEL_ENDPOINT: http://10.88.0.1:8001 64 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot/provision/requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | collections: 3 | - name: containers.podman 4 | version: 1.13.0 5 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot/quadlet/README.md: -------------------------------------------------------------------------------- 1 | ### Run chatbot as a systemd service 2 | 3 | ```bash 4 | (cd ../;make quadlet) 5 | sudo cp ../build/chatbot.yaml ../build/chatbot.kube ../build/chatbot.image /usr/share/containers/systemd/ 6 | sudo /usr/libexec/podman/quadlet --dryrun #optional 7 | sudo systemctl daemon-reload 8 | sudo systemctl start chatbot 9 | ``` 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot/quadlet/chatbot.image: -------------------------------------------------------------------------------- 1 | ../../../common/quadlet/app.image -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot/quadlet/chatbot.kube: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes YAML file used to do chatbot inferencing 3 | Documentation=man:podman-generate-systemd(1) 4 | Wants=network-online.target 5 | After=network-online.target 6 | RequiresMountsFor=%t/containers 7 | 8 | [Kube] 9 | # Point to the yaml file in the same directory 10 | Yaml=chatbot.yaml 11 | 12 | [Service] 13 | Restart=always 14 | 15 | [Install] 16 | WantedBy=default.target 17 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/chatbot/quadlet/chatbot.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | app: chatbot 6 | name: chatbot 7 | spec: 8 | initContainers: 9 | - name: model-file 10 | image: MODEL_IMAGE 11 | command: ['/usr/bin/install', "/model/model.file", "/shared/"] 12 | volumeMounts: 13 | - name: model-file 14 | mountPath: /shared 15 | containers: 16 | - env: 17 | - name: MODEL_ENDPOINT 18 | value: http://0.0.0.0:8001 19 | image: APP_IMAGE 20 | name: chatbot-inference 21 | ports: 22 | - containerPort: 8501 23 | hostPort: 8501 24 | securityContext: 25 | runAsNonRoot: true 26 | - env: 27 | - name: HOST 28 | value: 0.0.0.0 29 | - name: PORT 30 | value: 8001 31 | - name: MODEL_PATH 32 | value: /model/model.file 33 | image: SERVER_IMAGE 34 | name: chatbot-model-service 35 | ports: 36 | - containerPort: 8001 37 | hostPort: 8001 38 | securityContext: 39 | runAsNonRoot: true 40 | volumeMounts: 41 | - name: model-file 42 | mountPath: /model 43 | volumes: 44 | - name: model-file 45 | emptyDir: {} 46 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/codegen/Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | APP ?= codegen 3 | PORT ?= 8501 4 | 5 | include ../../common/Makefile.common 6 | 7 | RECIPE_BINARIES_PATH := $(shell realpath ../../common/bin) 8 | RELATIVE_MODELS_PATH := ../../../models 9 | RELATIVE_TESTS_PATH := ../tests 10 | MODEL_IMAGE := quay.io/ai-lab/mistral-7b-code-16k-qlora:latest 11 | MODEL_NAME := mistral-7b-code-16k-qlora.Q4_K_M.gguf 12 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/codegen/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: language 4 | name: codegen-demo 5 | description: Generate code in countless programming languages. 6 | containers: 7 | - name: llamacpp-server 8 | contextdir: ../../../model_servers/llamacpp_python 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - llama-cpp 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8001 18 | image: quay.io/ai-lab/llamacpp_python:latest 19 | - name: codegen-app 20 | contextdir: app 21 | containerfile: Containerfile 22 | arch: 23 | - arm64 24 | - amd64 25 | ports: 26 | - 8501 27 | image: quay.io/ai-lab/codegen:latest 28 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/codegen/app/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | WORKDIR /codegen 3 | COPY requirements.txt . 4 | RUN pip install --upgrade pip 5 | RUN pip install --no-cache-dir --upgrade -r /codegen/requirements.txt 6 | COPY codegen-app.py . 7 | EXPOSE 8501 8 | ENTRYPOINT ["streamlit", "run", "codegen-app.py"] 9 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/codegen/app/requirements.txt: -------------------------------------------------------------------------------- 1 | langchain==0.1.20 2 | langchain-openai==0.1.7 3 | streamlit==1.34.0 4 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/codegen/bootc/Containerfile.nocache: -------------------------------------------------------------------------------- 1 | # Example: an AI powered sample application is embedded as a systemd service 2 | # via Podman quadlet files in /usr/share/containers/systemd 3 | # 4 | # from recipes/natural_language_processing/codegen, run 5 | # 'make bootc' 6 | 7 | FROM quay.io/centos-bootc/centos-bootc:stream9 8 | ARG SSHPUBKEY 9 | 10 | # The --build-arg "SSHPUBKEY=$(cat ~/.ssh/id_rsa.pub)" option inserts your 11 | # public key into the image, allowing root access via ssh. 12 | RUN set -eu; mkdir -p /usr/ssh && \ 13 | echo 'AuthorizedKeysFile /usr/ssh/%u.keys .ssh/authorized_keys .ssh/authorized_keys2' >> /etc/ssh/sshd_config.d/30-auth-system.conf && \ 14 | echo ${SSHPUBKEY} > /usr/ssh/root.keys && chmod 0600 /usr/ssh/root.keys 15 | 16 | ARG RECIPE=codegen 17 | 18 | # Add quadlet files to setup system to automatically run AI application on boot 19 | COPY build/${RECIPE}.image build/${RECIPE}.kube build/${RECIPE}.yaml /usr/share/containers/systemd 20 | 21 | # Added for running as an OCI Container to prevent Overlay on Overlay issues. 22 | VOLUME /var/lib/containers 23 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/codegen/provision/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Test Environment Provisioning 3 | hosts: test_environments 4 | remote_user: fedora 5 | become: true 6 | gather_facts: false 7 | 8 | tasks: 9 | 10 | - name: Wait until the instance is ready 11 | ansible.builtin.wait_for_connection: 12 | delay: 10 13 | timeout: 60 14 | 15 | - name: Gather facts for first time 16 | ansible.builtin.setup: 17 | 18 | - name: Required Packages 19 | ansible.builtin.package: 20 | name: podman 21 | state: present 22 | 23 | - name: Models host directory 24 | ansible.builtin.file: 25 | path: locallm/models 26 | state: directory 27 | 28 | - name: Download Model 29 | ansible.builtin.get_url: 30 | url: https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_S.gguf 31 | dest: locallm/models 32 | 33 | - name: Run Model 34 | containers.podman.podman_container: 35 | name: llamacpp_python 36 | image: ghcr.io/containers/llamacpp_python:latest 37 | state: started 38 | interactive: true 39 | tty: true 40 | detach: true 41 | ports: 42 | - 8001:8001 43 | volume: 44 | - ./locallm/models:/locallm/models:ro,Z 45 | env: 46 | MODEL_PATH: models/llama-2-7b-chat.Q5_K_S.gguf 47 | HOST: 0.0.0.0 48 | PORT: 8001 49 | 50 | - name: Run Application 51 | containers.podman.podman_container: 52 | name: codegen 53 | image: ghcr.io/containers/codegen:latest 54 | state: started 55 | interactive: true 56 | tty: true 57 | ports: 58 | - 8501:8501 59 | env: 60 | MODEL_SERVICE_ENDPOINT: http://10.88.0.1:8001/v1 61 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/codegen/provision/requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | collections: 3 | - name: containers.podman 4 | version: 1.13.0 5 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/codegen/quadlet/README.md: -------------------------------------------------------------------------------- 1 | ### Run codegen as a systemd service 2 | 3 | ```bash 4 | (cd ../;make quadlet) 5 | sudo cp ../build/codegen.yaml ../build/codegen.kube /usr/share/containers/systemd/codegen.kube ../build/codegen.image /usr/share/containers/systemd/ 6 | sudo /usr/libexec/podman/quadlet --dryrun #optional 7 | sudo systemctl daemon-reload 8 | sudo systemctl start codegen 9 | ``` 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/codegen/quadlet/codegen.image: -------------------------------------------------------------------------------- 1 | ../../../common/quadlet/app.image -------------------------------------------------------------------------------- /recipes/natural_language_processing/codegen/quadlet/codegen.kube: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Python script to run against downloaded LLM 3 | Documentation=man:podman-generate-systemd(1) 4 | Wants=network-online.target 5 | After=network-online.target 6 | RequiresMountsFor=%t/containers 7 | 8 | [Kube] 9 | # Point to the yaml file in the same directory 10 | Yaml=codegen.yaml 11 | 12 | [Service] 13 | Restart=always 14 | 15 | [Install] 16 | WantedBy=default.target 17 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/codegen/quadlet/codegen.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | app: codegen 6 | name: codegen 7 | spec: 8 | initContainers: 9 | - name: model-file 10 | image: MODEL_IMAGE 11 | command: ['/usr/bin/install', "/model/model.file", "/shared/"] 12 | volumeMounts: 13 | - name: model-file 14 | mountPath: /shared 15 | containers: 16 | - env: 17 | - name: MODEL_ENDPOINT 18 | value: http://0.0.0.0:8001 19 | image: APP_IMAGE 20 | name: codegen-inference 21 | ports: 22 | - containerPort: 8501 23 | hostPort: 8501 24 | securityContext: 25 | runAsNonRoot: true 26 | - env: 27 | - name: HOST 28 | value: 0.0.0.0 29 | - name: PORT 30 | value: 8001 31 | - name: MODEL_PATH 32 | value: /model/model.file 33 | image: SERVER_IMAGE 34 | name: codegen-model-service 35 | ports: 36 | - containerPort: 8001 37 | hostPort: 8001 38 | securityContext: 39 | runAsNonRoot: true 40 | volumeMounts: 41 | - name: model-file 42 | mountPath: /model 43 | volumes: 44 | - name: model-file 45 | emptyDir: {} 46 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | APP ?= function-calling-nodejs 3 | PORT ?= 8501 4 | 5 | include ../../common/Makefile.common 6 | 7 | RECIPE_BINARIES_PATH := $(shell realpath ../../common/bin) 8 | RELATIVE_MODELS_PATH := ../../../models 9 | RELATIVE_TESTS_PATH := ../tests 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: language 4 | name: FunctionCalling_nodejs 5 | description: Call a function to get weather data. 6 | containers: 7 | - name: llamacpp-server 8 | contextdir: ../../../model_servers/llamacpp_python 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - llama-cpp 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8001 18 | image: quay.io/ai-lab/llamacpp_python:latest 19 | - name: nodejs-function-calling-app 20 | contextdir: app 21 | containerfile: Containerfile 22 | arch: 23 | - arm64 24 | - amd64 25 | ports: 26 | - 8501 27 | image: quay.io/ai-lab/function-calling-nodejs:latest 28 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/app/.dockerignore: -------------------------------------------------------------------------------- 1 | ./node_modules 2 | 3 | Dockerfile 4 | 5 | .dockerignore 6 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/app/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/app/Containerfile: -------------------------------------------------------------------------------- 1 | # Install the app dependencies in a full UBI Node docker image 2 | FROM registry.access.redhat.com/ubi9/nodejs-20:9.5-1746604787 3 | 4 | # Copy package.json 5 | COPY --chown=default:root --chmod=0775 package*.json ./ 6 | COPY --chown=default:root --chmod=0775 ./ ./ 7 | 8 | # Install npm packages 9 | RUN npm install 10 | 11 | ENV NODE_ENV production 12 | 13 | USER 1001 14 | EXPOSE 8501 15 | 16 | ENV PORT 8501 17 | 18 | CMD ["sh", "-c", "HOSTNAME=0.0.0.0 node ."] 19 | 20 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/app/ai/tools/weather.mjs: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | import { tool } from '@langchain/core/tools'; 3 | 4 | const weatherSchema = z.object({ 5 | latitude: z.number().describe('The latitude of a place'), 6 | longitude: z.number().describe('The longitude of a place') 7 | }); 8 | 9 | const weather = tool( 10 | async function ({ latitude, longitude }) { 11 | const response = await fetch(`https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}&hourly=temperature_2m`); 12 | const json = await response.json(); 13 | return json; 14 | }, 15 | { 16 | name: 'weather', 17 | description: 'Get the current weather in a given latitude and longitude.', 18 | schema: weatherSchema 19 | } 20 | ); 21 | 22 | export default weather; 23 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "function-calling-nodejs-just-html", 3 | "version": "1.0.0", 4 | "main": "server.mjs", 5 | "scripts": { 6 | "test": "echo \"Error: no test specified\" && exit 1" 7 | }, 8 | "keywords": [], 9 | "author": "", 10 | "license": "ISC", 11 | "description": "", 12 | "dependencies": { 13 | "@fastify/static": "^8.1.1", 14 | "@langchain/core": "^0.3.41", 15 | "@langchain/langgraph": "^0.2.49", 16 | "@langchain/openai": "^0.4.4", 17 | "fastify": "^5.2.1", 18 | "fastify-plugin": "^5.0.1", 19 | "zod": "^3.24.2" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/app/routes/temperatures.mjs: -------------------------------------------------------------------------------- 1 | import { askQuestion } from '../ai/weather-prompt.mjs'; 2 | 3 | async function temperatureRoutes (fastify, options) { 4 | fastify.post('/api/temperatures', async (request, reply) => { 5 | const city = request.body.city; 6 | 7 | // Call the AI stuff 8 | const response = await askQuestion(city); 9 | 10 | return { 11 | result: response 12 | } 13 | }); 14 | } 15 | 16 | export default temperatureRoutes; -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/app/server.mjs: -------------------------------------------------------------------------------- 1 | import Fastify from 'fastify'; 2 | import fastifyStatic from '@fastify/static'; 3 | import path from 'node:path'; 4 | import { fileURLToPath } from 'url'; 5 | import fs from 'node:fs'; 6 | const __filename = fileURLToPath(import.meta.url); 7 | const __dirname = path.dirname(__filename); 8 | 9 | import temperatureRoute from './routes/temperatures.mjs'; 10 | 11 | // Setup Logging 12 | const fastify = Fastify({ 13 | logger: true 14 | }); 15 | 16 | // WebUI related setup and serving 17 | const webuiLocation = './public'; 18 | 19 | fastify.register(fastifyStatic, { 20 | wildcard: false, 21 | root: path.join(__dirname, webuiLocation) 22 | }); 23 | 24 | fastify.get('/*', (req, res) => { 25 | res.send(fs.createReadStream(path.join(__dirname, webuiLocation, 'index.html'))); 26 | }); 27 | 28 | fastify.register(temperatureRoute); 29 | 30 | /** 31 | * Run the server! 32 | */ 33 | const start = async () => { 34 | try { 35 | await fastify.listen({ port: process.env.PORT || 8005, host: process.env.HOSTNAME }) 36 | } catch (err) { 37 | fastify.log.error(err) 38 | process.exit(1) 39 | } 40 | }; 41 | start(); -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/bootc/Containerfile.nocache: -------------------------------------------------------------------------------- 1 | # Example: an AI powered sample application is embedded as a systemd service 2 | # via Podman quadlet files in /usr/share/containers/systemd 3 | # 4 | # from recipes/natural_language_processing/function-calling-nodejs, run 5 | # 'make bootc' 6 | 7 | FROM quay.io/centos-bootc/centos-bootc:stream9 8 | ARG SSHPUBKEY 9 | 10 | # The --build-arg "SSHPUBKEY=$(cat ~/.ssh/id_rsa.pub)" option inserts your 11 | # public key into the image, allowing root access via ssh. 12 | RUN set -eu; mkdir -p /usr/ssh && \ 13 | echo 'AuthorizedKeysFile /usr/ssh/%u.keys .ssh/authorized_keys .ssh/authorized_keys2' >> /etc/ssh/sshd_config.d/30-auth-system.conf && \ 14 | echo ${SSHPUBKEY} > /usr/ssh/root.keys && chmod 0600 /usr/ssh/root.keys 15 | 16 | ARG RECIPE=function-calling-nodejs 17 | 18 | # Include growfs service 19 | COPY build/usr/lib /usr/lib 20 | COPY --chmod=0755 build/usr/libexec/bootc-generic-growpart /usr/libexec/bootc-generic-growpart 21 | 22 | # Add quadlet files to setup system to automatically run AI application on boot 23 | COPY build/${RECIPE}.image build/${RECIPE}.kube build/${RECIPE}.yaml /usr/share/containers/systemd 24 | 25 | # Added for running as an OCI Container to prevent Overlay on Overlay issues. 26 | VOLUME /var/lib/containers 27 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/quadlet/README.md: -------------------------------------------------------------------------------- 1 | ### Run chatbot as a systemd service 2 | 3 | ```bash 4 | (cd ../;make quadlet) 5 | sudo cp ../build/function-callling-nodejs.yaml ../build/function-callling-nodejs.kube ../build/function-callling-nodejs.image /usr/share/containers/systemd/ 6 | sudo /usr/libexec/podman/quadlet --dryrun #optional 7 | sudo systemctl daemon-reload 8 | sudo systemctl start function-callling-nodejs 9 | ``` 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/quadlet/function-calling-nodejs.image: -------------------------------------------------------------------------------- 1 | ../../../common/quadlet/app.image -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/quadlet/function-calling-nodejs.kube: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes YAML file used to do function-calling inferencing 3 | Documentation=man:podman-generate-systemd(1) 4 | Wants=network-online.target 5 | After=network-online.target 6 | RequiresMountsFor=%t/containers 7 | 8 | [Kube] 9 | # Point to the yaml file in the same directory 10 | Yaml=function-calling-nodejs.yaml 11 | 12 | [Service] 13 | Restart=always 14 | 15 | [Install] 16 | WantedBy=default.target 17 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function-calling-nodejs/quadlet/function-calling-nodejs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | app: function-calling-nodejs 6 | name: function-calling-nodejs 7 | spec: 8 | initContainers: 9 | - name: model-file 10 | image: MODEL_IMAGE 11 | command: ['/usr/bin/install', "/model/model.file", "/shared/"] 12 | volumeMounts: 13 | - name: model-file 14 | mountPath: /shared 15 | containers: 16 | - env: 17 | - name: MODEL_ENDPOINT 18 | value: http://0.0.0.0:8001 19 | image: APP_IMAGE 20 | name: function-calling-nodejs-inference 21 | ports: 22 | - containerPort: 8501 23 | hostPort: 8501 24 | securityContext: 25 | runAsNonRoot: true 26 | - env: 27 | - name: HOST 28 | value: 0.0.0.0 29 | - name: PORT 30 | value: 8001 31 | - name: MODEL_PATH 32 | value: /model/model.file 33 | image: SERVER_IMAGE 34 | name: chatbot-model-service 35 | ports: 36 | - containerPort: 8001 37 | hostPort: 8001 38 | securityContext: 39 | runAsNonRoot: true 40 | volumeMounts: 41 | - name: model-file 42 | mountPath: /model 43 | volumes: 44 | - name: model-file 45 | emptyDir: {} 46 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function_calling/Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | APP ?= function_calling 3 | PORT ?= 8501 4 | 5 | include ../../common/Makefile.common 6 | 7 | RECIPE_BINARIES_PATH := $(shell realpath ../../common/bin) 8 | RELATIVE_MODELS_PATH := ../../../models 9 | RELATIVE_TESTS_PATH := ../tests 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function_calling/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: language 4 | name: Function_Calling_Streamlit 5 | description: Function calling a remote service with a model service in a web frontend. 6 | containers: 7 | - name: llamacpp-server 8 | contextdir: ../../../model_servers/llamacpp_python 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - llama-cpp 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8001 18 | image: quay.io/ai-lab/llamacpp_python:latest 19 | - name: streamlit-function-calling-app 20 | contextdir: app 21 | containerfile: Containerfile 22 | arch: 23 | - arm64 24 | - amd64 25 | ports: 26 | - 8501 27 | image: quay.io/ai-lab/function-calling:latest 28 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function_calling/app/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | WORKDIR /function-call 3 | COPY requirements.txt . 4 | RUN pip install --upgrade pip 5 | RUN pip install --no-cache-dir --upgrade -r /function-call/requirements.txt 6 | COPY *.py . 7 | EXPOSE 8501 8 | ENTRYPOINT [ "streamlit", "run", "app.py" ] 9 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function_calling/app/requirements.txt: -------------------------------------------------------------------------------- 1 | langchain==0.2.3 2 | langchain-openai==0.1.7 3 | langchain-community==0.2.4 4 | streamlit==1.34.0 -------------------------------------------------------------------------------- /recipes/natural_language_processing/function_calling/bootc/Containerfile.nocache: -------------------------------------------------------------------------------- 1 | # Example: an AI powered sample application is embedded as a systemd service 2 | # via Podman quadlet files in /usr/share/containers/systemd 3 | # 4 | # from recipes/natural_language_processing/chatbot, run 5 | # 'make bootc' 6 | 7 | FROM quay.io/centos-bootc/centos-bootc:stream9 8 | ARG SSHPUBKEY 9 | 10 | # The --build-arg "SSHPUBKEY=$(cat ~/.ssh/id_rsa.pub)" option inserts your 11 | # public key into the image, allowing root access via ssh. 12 | RUN set -eu; mkdir -p /usr/ssh && \ 13 | echo 'AuthorizedKeysFile /usr/ssh/%u.keys .ssh/authorized_keys .ssh/authorized_keys2' >> /etc/ssh/sshd_config.d/30-auth-system.conf && \ 14 | echo ${SSHPUBKEY} > /usr/ssh/root.keys && chmod 0600 /usr/ssh/root.keys 15 | 16 | ARG RECIPE=function_calling 17 | 18 | # Include growfs service 19 | COPY build/usr/lib /usr/lib 20 | COPY --chmod=0755 build/usr/libexec/bootc-generic-growpart /usr/libexec/bootc-generic-growpart 21 | 22 | # Add quadlet files to setup system to automatically run AI application on boot 23 | COPY build/${RECIPE}.image build/${RECIPE}.kube build/${RECIPE}.yaml /usr/share/containers/systemd 24 | 25 | # Added for running as an OCI Container to prevent Overlay on Overlay issues. 26 | VOLUME /var/lib/containers 27 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function_calling/provision/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Test Environment Provisioning 3 | hosts: test_environments 4 | remote_user: fedora 5 | become: true 6 | gather_facts: false 7 | 8 | tasks: 9 | 10 | - name: Wait until the instance is ready 11 | ansible.builtin.wait_for_connection: 12 | delay: 10 13 | timeout: 60 14 | 15 | - name: Gather facts for first time 16 | ansible.builtin.setup: 17 | 18 | - name: Required Packages 19 | ansible.builtin.package: 20 | name: "{{ item }}" 21 | state: present 22 | with_items: 23 | - podman 24 | - python3-libdnf5 25 | 26 | - name: Models host directory 27 | ansible.builtin.file: 28 | path: locallm/models 29 | state: directory 30 | 31 | - name: Download Model 32 | ansible.builtin.get_url: 33 | url: https://huggingface.co/MaziyarPanahi/Mistral-7B-Instruct-v0.3-GGUF/resolve/main/Mistral-7B-Instruct-v0.3.Q4_K_M.gguf 34 | dest: locallm/models 35 | 36 | - name: Run Model 37 | containers.podman.podman_container: 38 | name: llamacpp_python 39 | image: ghcr.io/containers/llamacpp_python:latest 40 | state: started 41 | interactive: true 42 | tty: true 43 | detach: true 44 | ports: 45 | - 8001:8001 46 | volume: 47 | - ./locallm/models:/locallm/models:ro,Z 48 | env: 49 | MODEL_PATH: models/Mistral-7B-Instruct-v0.3.Q4_K_M.gguf 50 | HOST: 0.0.0.0 51 | PORT: 8001 52 | 53 | - name: Run Application 54 | containers.podman.podman_container: 55 | name: function_calling 56 | image: ghcr.io/containers/function_calling:latest 57 | state: started 58 | interactive: true 59 | tty: true 60 | ports: 61 | - 8501:8501 62 | env: 63 | MODEL_ENDPOINT: http://10.88.0.1:8001 64 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function_calling/provision/requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | collections: 3 | - name: containers.podman 4 | version: 1.13.0 5 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function_calling/quadlet/README.md: -------------------------------------------------------------------------------- 1 | ### Run function calling as a systemd service 2 | 3 | ```bash 4 | (cd ../;make quadlet) 5 | sudo cp ../build/function_calling.yaml ../build/function_calling.kube ../build/function_calling.image /usr/share/containers/systemd/ 6 | sudo /usr/libexec/podman/quadlet --dryrun #optional 7 | sudo systemctl daemon-reload 8 | sudo systemctl start function_calling 9 | ``` 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function_calling/quadlet/function_calling.kube: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes YAML file used to do function calling inferencing 3 | Documentation=man:podman-generate-systemd(1) 4 | Wants=network-online.target 5 | After=network-online.target 6 | RequiresMountsFor=%t/containers 7 | 8 | [Kube] 9 | # Point to the yaml file in the same directory 10 | Yaml=function_calling.yaml 11 | 12 | [Service] 13 | Restart=always 14 | 15 | [Install] 16 | WantedBy=default.target 17 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/function_calling/quadlet/function_calling.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | app: function_calling 6 | name: function_calling 7 | spec: 8 | initContainers: 9 | - name: model-file 10 | image: MODEL_IMAGE 11 | command: ['/usr/bin/install', "/model/model.file", "/shared/"] 12 | volumeMounts: 13 | - name: model-file 14 | mountPath: /shared 15 | containers: 16 | - env: 17 | - name: MODEL_ENDPOINT 18 | value: http://0.0.0.0:8001 19 | image: APP_IMAGE 20 | name: function_calling-inference 21 | ports: 22 | - containerPort: 8501 23 | hostPort: 8501 24 | securityContext: 25 | runAsNonRoot: true 26 | - env: 27 | - name: HOST 28 | value: 0.0.0.0 29 | - name: PORT 30 | value: 8001 31 | - name: MODEL_PATH 32 | value: /model/model.file 33 | image: SERVER_IMAGE 34 | name: function_calling-model-service 35 | ports: 36 | - containerPort: 8001 37 | hostPort: 8001 38 | securityContext: 39 | runAsNonRoot: true 40 | volumeMounts: 41 | - name: model-file 42 | mountPath: /model 43 | volumes: 44 | - name: model-file 45 | emptyDir: {} 46 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/graph-rag/Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | APP ?= graph-rag 3 | PORT ?= 8501 4 | # MODEL_IMAGE ?= huggingface://TheBloke/Mistral-7B-Instruct-v0.2-GGUF/mistral-7b-instruct-v0.2.Q4_K_M.gguf 5 | 6 | include ../../common/Makefile.common 7 | 8 | RECIPE_BINARIES_PATH := $(shell realpath ../../common/bin) 9 | RELATIVE_MODELS_PATH := ../../../models 10 | RELATIVE_TESTS_PATH := ../tests -------------------------------------------------------------------------------- /recipes/natural_language_processing/graph-rag/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: language 4 | name: Graphrag_Streamlit 5 | description: Chat with a model service in a web frontend. 6 | containers: 7 | - name: llamacpp-server 8 | contextdir: ../../../model_servers/llamacpp_python 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - llama-cpp 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8001 18 | image: quay.io/ai-lab/llamacpp_python:latest 19 | - name: streamlit-chat-app 20 | contextdir: app 21 | containerfile: Containerfile 22 | arch: 23 | - arm64 24 | - amd64 25 | ports: 26 | - 8501 27 | image: quay.io/ai-lab/graph-rag:latest 28 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/graph-rag/app/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | 3 | # Set the working directory 4 | WORKDIR /graph-rag 5 | # Ensure the default user has the correct permissions on the working directory 6 | RUN chown -R 1001:0 /graph-rag 7 | 8 | # Copy the application files 9 | COPY requirements.txt . 10 | COPY rag_app.py . 11 | 12 | RUN pip install --upgrade pip && \ 13 | pip install --no-cache-dir --upgrade -r /graph-rag/requirements.txt 14 | 15 | # Expose the port for the application 16 | EXPOSE 8501 17 | 18 | # Set the entrypoint to run the application 19 | ENTRYPOINT [ "streamlit", "run", "rag_app.py" ] 20 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/graph-rag/app/requirements.txt: -------------------------------------------------------------------------------- 1 | streamlit==1.45.1 2 | langchain-graph-retriever==0.8.0 3 | langchain-huggingface==0.2.0 4 | langchain-openai==0.3.17 5 | transformers==4.52.4 6 | torch==2.7.0 7 | PyMuPDF==1.25.5 -------------------------------------------------------------------------------- /recipes/natural_language_processing/graph-rag/bootc/Containerfile.nocache: -------------------------------------------------------------------------------- 1 | # Example: an AI powered sample application is embedded as a systemd service 2 | # via Podman quadlet files in /usr/share/containers/systemd 3 | # 4 | # from recipes/natural_language_processing/chatbot, run 5 | # 'make bootc' 6 | 7 | FROM quay.io/centos-bootc/centos-bootc:stream9 8 | ARG SSHPUBKEY 9 | 10 | # The --build-arg "SSHPUBKEY=$(cat ~/.ssh/id_rsa.pub)" option inserts your 11 | # public key into the image, allowing root access via ssh. 12 | RUN set -eu; mkdir -p /usr/ssh && \ 13 | echo 'AuthorizedKeysFile /usr/ssh/%u.keys .ssh/authorized_keys .ssh/authorized_keys2' >> /etc/ssh/sshd_config.d/30-auth-system.conf && \ 14 | echo ${SSHPUBKEY} > /usr/ssh/root.keys && chmod 0600 /usr/ssh/root.keys 15 | 16 | ARG RECIPE=graph-rag 17 | 18 | # Include growfs service 19 | COPY build/usr/lib /usr/lib 20 | COPY --chmod=0755 build/usr/libexec/bootc-generic-growpart /usr/libexec/bootc-generic-growpart 21 | 22 | # Add quadlet files to setup system to automatically run AI application on boot 23 | COPY build/${RECIPE}.image build/${RECIPE}.kube build/${RECIPE}.yaml /usr/share/containers/systemd 24 | 25 | # Added for running as an OCI Container to prevent Overlay on Overlay issues. 26 | VOLUME /var/lib/containers 27 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/graph-rag/provision/playbook.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Test Environment Provisioning 3 | hosts: test_environments 4 | remote_user: fedora 5 | become: true 6 | gather_facts: false 7 | 8 | tasks: 9 | 10 | - name: Wait until the instance is ready 11 | ansible.builtin.wait_for_connection: 12 | delay: 10 13 | timeout: 60 14 | 15 | - name: Gather facts for first time 16 | ansible.builtin.setup: 17 | 18 | - name: Required Packages 19 | ansible.builtin.package: 20 | name: "{{ item }}" 21 | state: present 22 | with_items: 23 | - podman 24 | - python3-libdnf5 25 | 26 | - name: Models host directory 27 | ansible.builtin.file: 28 | path: locallm/models 29 | state: directory 30 | 31 | - name: Download Model 32 | ansible.builtin.get_url: 33 | url: https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_S.gguf 34 | dest: locallm/models 35 | 36 | - name: Run Model 37 | containers.podman.podman_container: 38 | name: llamacpp_python 39 | image: ghcr.io/containers/llamacpp_python:latest 40 | state: started 41 | interactive: true 42 | tty: true 43 | detach: true 44 | ports: 45 | - 8001:8001 46 | volume: 47 | - ./locallm/models:/locallm/models:ro,Z 48 | env: 49 | MODEL_PATH: models/llama-2-7b-chat.Q5_K_S.gguf 50 | HOST: 0.0.0.0 51 | PORT: 8001 52 | 53 | - name: Run Application 54 | containers.podman.podman_container: 55 | name: graph-rag 56 | image: ghcr.io/containers/graph-rag:latest 57 | state: started 58 | interactive: true 59 | tty: true 60 | ports: 61 | - 8501:8501 62 | env: 63 | MODEL_ENDPOINT: http://10.88.0.1:8001 64 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/graph-rag/provision/requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | collections: 3 | - name: containers.podman 4 | version: 1.13.0 5 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/graph-rag/quadlet/README.md: -------------------------------------------------------------------------------- 1 | ### Run chatbot as a systemd service 2 | 3 | ```bash 4 | (cd ../;make quadlet) 5 | sudo cp ../build/chatbot.yaml ../build/chatbot.kube ../build/chatbot.image /usr/share/containers/systemd/ 6 | sudo /usr/libexec/podman/quadlet --dryrun #optional 7 | sudo systemctl daemon-reload 8 | sudo systemctl start chatbot 9 | ``` 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/graph-rag/quadlet/graph-rag.image: -------------------------------------------------------------------------------- 1 | ../../../common/quadlet/app.image -------------------------------------------------------------------------------- /recipes/natural_language_processing/graph-rag/quadlet/graph-rag.kube: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes YAML file used to do chatbot inferencing 3 | Documentation=man:podman-generate-systemd(1) 4 | Wants=network-online.target 5 | After=network-online.target 6 | RequiresMountsFor=%t/containers 7 | 8 | [Kube] 9 | # Point to the yaml file in the same directory 10 | Yaml=graph-rag.yaml 11 | 12 | [Service] 13 | Restart=always 14 | 15 | [Install] 16 | WantedBy=default.target 17 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/graph-rag/quadlet/graph-rag.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | app: graph-rag 6 | name: graph-rag 7 | spec: 8 | initContainers: 9 | - name: model-file 10 | image: MODEL_IMAGE 11 | command: ['/usr/bin/install', "/model/model.file", "/shared/"] 12 | volumeMounts: 13 | - name: model-file 14 | mountPath: /shared 15 | containers: 16 | - env: 17 | - name: MODEL_ENDPOINT 18 | value: http://0.0.0.0:8001 19 | image: APP_IMAGE 20 | name: graph-rag-inference 21 | ports: 22 | - containerPort: 8501 23 | hostPort: 8501 24 | securityContext: 25 | runAsNonRoot: true 26 | - env: 27 | - name: HOST 28 | value: 0.0.0.0 29 | - name: PORT 30 | value: 8001 31 | - name: MODEL_PATH 32 | value: /model/model.file 33 | image: SERVER_IMAGE 34 | name: graph-rag-model-service 35 | ports: 36 | - containerPort: 8001 37 | hostPort: 8001 38 | securityContext: 39 | runAsNonRoot: true 40 | volumeMounts: 41 | - name: model-file 42 | mountPath: /model 43 | volumes: 44 | - name: model-file 45 | emptyDir: {} 46 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag-nodejs/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: language 4 | name: nodejs-rag-demo 5 | description: A Node.js RAG chat bot using local documents. 6 | containers: 7 | - name: llamacpp-server 8 | contextdir: ../../../model_servers/llamacpp_python 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - llama-cpp 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8001 18 | image: quay.io/ai-lab/llamacpp_python:latest 19 | - name: chromadb-server 20 | contextdir: ../../../vector_dbs/chromadb 21 | containerfile: Containerfile 22 | vectordb: true 23 | arch: 24 | - arm64 25 | - amd64 26 | ports: 27 | - 8000 28 | image: quay.io/ai-lab/chromadb:latest 29 | - name: nodejs-rag-inference-app 30 | contextdir: app 31 | containerfile: Containerfile 32 | arch: 33 | - arm64 34 | - amd64 35 | ports: 36 | - 8501 37 | image: quay.io/ai-lab/rag-nodejs:latest 38 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag-nodejs/app/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag-nodejs/app/app/layout.tsx: -------------------------------------------------------------------------------- 1 | export const metadata = { 2 | title: 'Sample Node.js RAG example ', 3 | description: 'Sample Node.js RAG example', 4 | } 5 | 6 | export default function RootLayout({ 7 | children, 8 | }: { 9 | children: React.ReactNode 10 | }) { 11 | return ( 12 | 13 | {children} 14 | 15 | ) 16 | } 17 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag-nodejs/app/next-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | /// 3 | /// 4 | 5 | // NOTE: This file should not be edited 6 | // see https://nextjs.org/docs/app/building-your-application/configuring/typescript for more information. 7 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag-nodejs/app/next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | module.exports = { 3 | output: "standalone", 4 | 5 | // Indicate that these packages should not be bundled by webpack 6 | experimental: { 7 | serverComponentsExternalPackages: ['sharp', 'onnxruntime-node', 'chromadb', '@xenova/transformers' ], 8 | }, 9 | }; 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag-nodejs/app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "app", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@langchain/community": "^0.3.10", 7 | "@langchain/core": "^0.2.14", 8 | "@langchain/openai": "^0.2.1", 9 | "@testing-library/jest-dom": "^5.17.0", 10 | "@testing-library/react": "^13.4.0", 11 | "@testing-library/user-event": "^13.5.0", 12 | "@xenova/transformers": "^2.17.2", 13 | "chromadb": "^1.10.0", 14 | "install": "^0.13.0", 15 | "next": "^14.2.5", 16 | "npm": "^10.9.1", 17 | "pdf-parse": "^1.1.1", 18 | "primereact": "^10.8.5", 19 | "react": "^18.3.1", 20 | "react-chatbotify": "^1.7.0", 21 | "react-dom": "^18.3.1", 22 | "react-scripts": "5.0.1", 23 | "socket.io": "^4.7.2", 24 | "socket.io-client": "^4.7.2", 25 | "web-vitals": "^2.1.4" 26 | }, 27 | "devDependencies": { 28 | "autoprefixer": "^10.4.20", 29 | "eslint": "^8", 30 | "eslint-config-next": "14.0.3", 31 | "postcss": "^8", 32 | "tailwindcss": "^3.3.0" 33 | }, 34 | "overrides": { 35 | "typescript": "^5.5.3", 36 | "@langchain/core": "^0.2.14" 37 | }, 38 | "scripts": { 39 | "dev": "next dev -p 8501", 40 | "build": "next build", 41 | "lint": "next lint" 42 | }, 43 | "eslintConfig": { 44 | "extends": [ 45 | "react-app", 46 | "react-app/jest" 47 | ] 48 | }, 49 | "browserslist": { 50 | "production": [ 51 | ">0.2%", 52 | "not dead", 53 | "not op_mini all" 54 | ], 55 | "development": [ 56 | "last 1 chrome version", 57 | "last 1 firefox version", 58 | "last 1 safari version" 59 | ] 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag-nodejs/app/pages/api/delete/index.js: -------------------------------------------------------------------------------- 1 | import { ChromaClient } from 'chromadb'; 2 | 3 | const vectorDBHost = process.env.VECTORDB_HOST || '0.0.0.0'; 4 | const vectorDBPort = process.env.VECTORDB_PORT || 8000; 5 | const vectorDBName = process.env.VECTORDB_NAME || 'nodejs_test_collection'; 6 | 7 | const url = `http://${vectorDBHost}:${vectorDBPort}`; 8 | const client = new ChromaClient({path: url}); 9 | 10 | ///////////////////////////////////////////////////// 11 | // delete and recreate the collection in Chroma when 12 | // requested by the front end 13 | const HANDLER = async (req, res) => { 14 | try { 15 | const collection = await client.getOrCreateCollection({name: vectorDBName}); 16 | const result = await collection.get({include: []}); 17 | if (result && result.ids && (result.ids.length > 0)) { 18 | await collection.delete({ids: result.ids}); 19 | } 20 | res.statusCode = 200; 21 | res.statusMessage = 'Success'; 22 | res.end("Deleted succesfully, count is:" + await collection.count()); 23 | } catch (error) { 24 | res.statusCode = 500; 25 | res.statusMessage = 'Deletion failed'; 26 | res.end(error.toString()); 27 | } 28 | }; 29 | 30 | export default HANDLER; 31 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag-nodejs/app/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": [ 4 | "dom", 5 | "dom.iterable", 6 | "esnext" 7 | ], 8 | "allowJs": true, 9 | "skipLibCheck": true, 10 | "strict": false, 11 | "noEmit": true, 12 | "incremental": true, 13 | "module": "esnext", 14 | "esModuleInterop": true, 15 | "moduleResolution": "node", 16 | "resolveJsonModule": true, 17 | "isolatedModules": true, 18 | "jsx": "preserve", 19 | "plugins": [ 20 | { 21 | "name": "next" 22 | } 23 | ], 24 | "strictNullChecks": true 25 | }, 26 | "include": [ 27 | "next-env.d.ts", 28 | ".next/types/**/*.ts", 29 | "**/*.ts", 30 | "**/*.tsx" 31 | ], 32 | "exclude": [ 33 | "node_modules" 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag-nodejs/bootc/Containerfile.nocache: -------------------------------------------------------------------------------- 1 | # Example: an AI powered sample application is embedded as a systemd service 2 | # via Podman quadlet files in /usr/share/containers/systemd 3 | # 4 | # from recipes/natural_language_processing/rag-nodejs, run 5 | # 'make bootc' 6 | 7 | FROM quay.io/centos-bootc/centos-bootc:stream9 8 | ARG SSHPUBKEY 9 | 10 | # The --build-arg "SSHPUBKEY=$(cat ~/.ssh/id_rsa.pub)" option inserts your 11 | # public key into the image, allowing root access via ssh. 12 | RUN set -eu; mkdir -p /usr/ssh && \ 13 | echo 'AuthorizedKeysFile /usr/ssh/%u.keys .ssh/authorized_keys .ssh/authorized_keys2' >> /etc/ssh/sshd_config.d/30-auth-system.conf && \ 14 | echo ${SSHPUBKEY} > /usr/ssh/root.keys && chmod 0600 /usr/ssh/root.keys 15 | 16 | ARG RECIPE=rag-nodejs 17 | 18 | # Add quadlet files to setup system to automatically run AI application on boot 19 | COPY build/${RECIPE}.image build/${RECIPE}.kube build/${RECIPE}.yaml /usr/share/containers/systemd 20 | 21 | # Added for running as an OCI Container to prevent Overlay on Overlay issues. 22 | VOLUME /var/lib/containers 23 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag-nodejs/quadlet/README.md: -------------------------------------------------------------------------------- 1 | ### Run rag-nodejs as a systemd service 2 | 3 | ```bash 4 | (cd ../;make quadlet) 5 | sudo cp ../build/rag-nodejs.yaml ../build/rag-nodejs.kube ../build/rag-nodejs.image /usr/share/containers/systemd/ 6 | sudo /usr/libexec/podman/quadlet --dryrun #optional 7 | sudo systemctl daemon-reload 8 | sudo systemctl start rag-nodejs 9 | ``` 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag-nodejs/quadlet/rag-nodejs.image: -------------------------------------------------------------------------------- 1 | ../../../common/quadlet/app.image -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag-nodejs/quadlet/rag-nodejs.kube: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes YAML file used to do rag inferencing 3 | Documentation=man:podman-generate-systemd(1) 4 | Wants=network-online.target 5 | After=network-online.target 6 | RequiresMountsFor=%t/containers 7 | 8 | [Kube] 9 | # Point to the yaml file in the same directory 10 | Yaml=rag-nodejs.yaml 11 | 12 | [Service] 13 | Restart=always 14 | 15 | [Install] 16 | WantedBy=default.target 17 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag-nodejs/quadlet/rag-nodejs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | app: rag-nodejs 6 | name: rag-nodejs 7 | spec: 8 | initContainers: 9 | - name: model-file 10 | image: MODEL_IMAGE 11 | command: ['/usr/bin/install', "/model/model.file", "/shared/"] 12 | volumeMounts: 13 | - name: model-file 14 | mountPath: /shared 15 | containers: 16 | - env: 17 | - name: MODEL_ENDPOINT 18 | value: http://0.0.0.0:8001 19 | image: APP_IMAGE 20 | name: rag-nodejs-inference 21 | ports: 22 | - containerPort: 8501 23 | hostPort: 8501 24 | securityContext: 25 | runAsNonRoot: true 26 | - env: 27 | - name: CHROMADB_ENDPOINT 28 | value: http://0.0.0.0:8000/v1 29 | image: ghcr.io/chroma-core/chroma:0.5.23 30 | name: rag-chromadb 31 | ports: 32 | - containerPort: 8000 33 | hostPort: 8000 34 | securityContext: 35 | runAsNonRoot: true 36 | - env: 37 | - name: HOST 38 | value: 0.0.0.0 39 | - name: PORT 40 | value: 8001 41 | - name: MODEL_PATH 42 | value: /model/model.file 43 | image: SERVER_IMAGE 44 | name: rag-model-service 45 | ports: 46 | - containerPort: 8001 47 | hostPort: 8001 48 | securityContext: 49 | runAsNonRoot: true 50 | volumeMounts: 51 | - name: model-file 52 | mountPath: /model 53 | volumes: 54 | - name: model-file 55 | emptyDir: {} 56 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: language 4 | name: rag-demo 5 | description: A RAG chat bot using local documents. 6 | containers: 7 | - name: llamacpp-server 8 | contextdir: ../../../model_servers/llamacpp_python 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - llama-cpp 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8001 18 | image: quay.io/ai-lab/llamacpp_python:latest 19 | - name: chromadb-server 20 | contextdir: ../../../vector_dbs/chromadb 21 | containerfile: Containerfile 22 | vectordb: true 23 | arch: 24 | - arm64 25 | - amd64 26 | ports: 27 | - 8000 28 | image: quay.io/ai-lab/chromadb:latest 29 | - name: rag-inference-app 30 | contextdir: app 31 | containerfile: Containerfile 32 | arch: 33 | - arm64 34 | - amd64 35 | ports: 36 | - 8501 37 | image: quay.io/ai-lab/rag:latest 38 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag/app/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | ### Update sqlite for chroma 3 | USER root 4 | RUN dnf remove sqlite3 -y 5 | RUN wget https://www.sqlite.org/2023/sqlite-autoconf-3410200.tar.gz 6 | RUN tar -xvzf sqlite-autoconf-3410200.tar.gz 7 | WORKDIR sqlite-autoconf-3410200 8 | RUN ./configure 9 | RUN make 10 | RUN make install 11 | RUN mv /usr/local/bin/sqlite3 /usr/bin/sqlite3 12 | ENV LD_LIBRARY_PATH="/usr/local/lib" 13 | #### 14 | WORKDIR /rag 15 | COPY requirements.txt . 16 | RUN pip install --upgrade pip 17 | RUN pip install --no-cache-dir --upgrade -r /rag/requirements.txt 18 | COPY rag_app.py . 19 | COPY manage_vectordb.py . 20 | EXPOSE 8501 21 | ENV HF_HUB_CACHE=/rag/models/ 22 | RUN mkdir -p /rag/models/ 23 | RUN chgrp -R 0 /rag/models/ && chmod -R g=u /rag/models/ 24 | ENTRYPOINT [ "streamlit", "run" ,"rag_app.py" ] 25 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag/app/requirements.txt: -------------------------------------------------------------------------------- 1 | langchain-openai==0.1.7 2 | langchain==0.1.20 3 | chromadb==0.5.23 4 | sentence-transformers==2.7.0 5 | streamlit==1.34.0 6 | pypdf==4.2.0 7 | pymilvus==2.4.1 8 | 9 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag/bootc/Containerfile.nocache: -------------------------------------------------------------------------------- 1 | # Example: an AI powered sample application is embedded as a systemd service 2 | # via Podman quadlet files in /usr/share/containers/systemd 3 | # 4 | # from recipes/natural_language_processing/rag, run 5 | # 'make bootc' 6 | 7 | FROM quay.io/centos-bootc/centos-bootc:stream9 8 | ARG SSHPUBKEY 9 | 10 | # The --build-arg "SSHPUBKEY=$(cat ~/.ssh/id_rsa.pub)" option inserts your 11 | # public key into the image, allowing root access via ssh. 12 | RUN set -eu; mkdir -p /usr/ssh && \ 13 | echo 'AuthorizedKeysFile /usr/ssh/%u.keys .ssh/authorized_keys .ssh/authorized_keys2' >> /etc/ssh/sshd_config.d/30-auth-system.conf && \ 14 | echo ${SSHPUBKEY} > /usr/ssh/root.keys && chmod 0600 /usr/ssh/root.keys 15 | 16 | ARG RECIPE=rag 17 | 18 | # Add quadlet files to setup system to automatically run AI application on boot 19 | COPY build/${RECIPE}.image build/${RECIPE}.kube build/${RECIPE}.yaml /usr/share/containers/systemd 20 | 21 | # Added for running as an OCI Container to prevent Overlay on Overlay issues. 22 | VOLUME /var/lib/containers 23 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag/provision/requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | collections: 3 | - name: containers.podman 4 | version: 1.13.0 5 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag/quadlet/README.md: -------------------------------------------------------------------------------- 1 | ### Run rag as a systemd service 2 | 3 | ```bash 4 | (cd ../;make quadlet) 5 | sudo cp ../build/rag.yaml ../build/rag.kube ../build/rag.image /usr/share/containers/systemd/ 6 | sudo /usr/libexec/podman/quadlet --dryrun #optional 7 | sudo systemctl daemon-reload 8 | sudo systemctl start rag 9 | ``` 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag/quadlet/rag.image: -------------------------------------------------------------------------------- 1 | ../../../common/quadlet/app.image -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag/quadlet/rag.kube: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes YAML file used to do rag inferencing 3 | Documentation=man:podman-generate-systemd(1) 4 | Wants=network-online.target 5 | After=network-online.target 6 | RequiresMountsFor=%t/containers 7 | 8 | [Kube] 9 | # Point to the yaml file in the same directory 10 | Yaml=rag.yaml 11 | 12 | [Service] 13 | Restart=always 14 | 15 | [Install] 16 | WantedBy=default.target 17 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/rag/quadlet/rag.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | app: rag 6 | name: rag 7 | spec: 8 | initContainers: 9 | - name: model-file 10 | image: MODEL_IMAGE 11 | command: ['/usr/bin/install', "/model/model.file", "/shared/"] 12 | volumeMounts: 13 | - name: model-file 14 | mountPath: /shared 15 | containers: 16 | - env: 17 | - name: MODEL_ENDPOINT 18 | value: http://0.0.0.0:8001 19 | image: APP_IMAGE 20 | name: rag-inference 21 | ports: 22 | - containerPort: 8501 23 | hostPort: 8501 24 | securityContext: 25 | runAsNonRoot: true 26 | - env: 27 | - name: CHROMADB_ENDPOINT 28 | value: http://0.0.0.0:800O/v1 29 | image: CHROMADB_IMAGE 30 | name: rag-chromadb 31 | ports: 32 | - containerPort: 8000 33 | hostPort: 8000 34 | securityContext: 35 | runAsNonRoot: true 36 | - env: 37 | - name: HOST 38 | value: 0.0.0.0 39 | - name: PORT 40 | value: 8001 41 | - name: MODEL_PATH 42 | value: /model/model.file 43 | image: SERVER_IMAGE 44 | name: rag-model-service 45 | ports: 46 | - containerPort: 8001 47 | hostPort: 8001 48 | securityContext: 49 | runAsNonRoot: true 50 | volumeMounts: 51 | - name: model-file 52 | mountPath: /model 53 | volumes: 54 | - name: model-file 55 | emptyDir: {} 56 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/summarizer/Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | APP ?= summarizer 3 | PORT ?= 8501 4 | 5 | include ../../common/Makefile.common 6 | 7 | RECIPE_BINARIES_PATH := $(shell realpath ../../common/bin) 8 | RELATIVE_MODELS_PATH := ../../../models 9 | RELATIVE_TESTS_PATH := ../tests 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/summarizer/ai-lab.yaml: -------------------------------------------------------------------------------- 1 | version: v1.0 2 | application: 3 | type: language 4 | name: Summarizer_App 5 | description: Summarize text files in a web frontend. 6 | containers: 7 | - name: llamacpp-server 8 | contextdir: ../../../model_servers/llamacpp_python 9 | containerfile: ./base/Containerfile 10 | model-service: true 11 | backend: 12 | - llama-cpp 13 | arch: 14 | - arm64 15 | - amd64 16 | ports: 17 | - 8001 18 | image: quay.io/ai-lab/llamacpp_python:latest 19 | - name: streamlit-summary-app 20 | contextdir: app 21 | containerfile: Containerfile 22 | arch: 23 | - arm64 24 | - amd64 25 | ports: 26 | - 8501 27 | image: quay.io/ai-lab/summarizer:latest 28 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/summarizer/app/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316 2 | WORKDIR /summarizer 3 | COPY requirements.txt . 4 | RUN pip install --upgrade pip 5 | RUN pip install --no-cache-dir --upgrade -r /summarizer/requirements.txt 6 | COPY summarizer.py . 7 | EXPOSE 8501 8 | ENTRYPOINT [ "streamlit", "run", "summarizer.py" ] 9 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/summarizer/app/requirements.txt: -------------------------------------------------------------------------------- 1 | langchain==0.1.20 2 | langchain-openai==0.1.7 3 | streamlit==1.34.0 4 | PyMuPDF==1.24.11 5 | rouge_score==0.1.2 6 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/summarizer/bootc/Containerfile.nocache: -------------------------------------------------------------------------------- 1 | # Example: an AI powered sample application is embedded as a systemd service 2 | # via Podman quadlet files in /usr/share/containers/systemd 3 | # 4 | # from recipes/natural_language_processing/summarizer, run 5 | # 'make bootc' 6 | 7 | FROM quay.io/centos-bootc/centos-bootc:stream9 8 | ARG SSHPUBKEY 9 | 10 | # The --build-arg "SSHPUBKEY=$(cat ~/.ssh/id_rsa.pub)" option inserts your 11 | # public key into the image, allowing root access via ssh. 12 | RUN set -eu; mkdir -p /usr/ssh && \ 13 | echo 'AuthorizedKeysFile /usr/ssh/%u.keys .ssh/authorized_keys .ssh/authorized_keys2' >> /etc/ssh/sshd_config.d/30-auth-system.conf && \ 14 | echo ${SSHPUBKEY} > /usr/ssh/root.keys && chmod 0600 /usr/ssh/root.keys 15 | 16 | ARG RECIPE=summarizer 17 | 18 | # Add quadlet files to setup system to automatically run AI application on boot 19 | COPY build/${RECIPE}.image build/${RECIPE}.kube build/${RECIPE}.yaml /usr/share/containers/systemd 20 | 21 | # Added for running as an OCI Container to prevent Overlay on Overlay issues. 22 | VOLUME /var/lib/containers 23 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/summarizer/provision/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Test Environment Provisioning 3 | hosts: test_environments 4 | remote_user: fedora 5 | become: true 6 | gather_facts: false 7 | 8 | tasks: 9 | 10 | - name: Wait until the instance is ready 11 | ansible.builtin.wait_for_connection: 12 | delay: 10 13 | timeout: 60 14 | 15 | - name: Gather facts for first time 16 | ansible.builtin.setup: 17 | 18 | - name: Required Packages 19 | ansible.builtin.package: 20 | name: podman 21 | state: present 22 | 23 | - name: Models host directory 24 | ansible.builtin.file: 25 | path: locallm/models 26 | state: directory 27 | 28 | - name: Download Model 29 | ansible.builtin.get_url: 30 | url: https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_S.gguf 31 | dest: locallm/models 32 | 33 | - name: Run Model 34 | containers.podman.podman_container: 35 | name: llamacpp_python 36 | image: ghcr.io/containers/llamacpp_python:latest 37 | state: started 38 | interactive: true 39 | tty: true 40 | detach: true 41 | ports: 42 | - 8001:8001 43 | volume: 44 | - ./locallm/models:/locallm/models:ro,Z 45 | env: 46 | MODEL_PATH: models/llama-2-7b-chat.Q5_K_S.gguf 47 | HOST: 0.0.0.0 48 | PORT: 8001 49 | 50 | - name: Run Application 51 | containers.podman.podman_container: 52 | name: summarizer 53 | image: ghcr.io/containers/summarizer:latest 54 | state: started 55 | interactive: true 56 | tty: true 57 | ports: 58 | - 8501:8501 59 | env: 60 | MODEL_SERVICE_ENDPOINT: http://10.88.0.1:8001/v1 61 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/summarizer/provision/requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | collections: 3 | - name: containers.podman 4 | version: 1.13.0 5 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/summarizer/quadlet/README.md: -------------------------------------------------------------------------------- 1 | ### Run summarizer as a systemd service 2 | 3 | ```bash 4 | (cd ../;make quadlet) 5 | sudo cp ../build/summarizer.yaml ../build/summarizer.kube ../build/summarizer.image /usr/share/containers/systemd/ 6 | sudo /usr/libexec/podman/quadlet --dryrun #optional 7 | sudo systemctl daemon-reload 8 | sudo systemctl start summarizer 9 | ``` 10 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/summarizer/quadlet/summarizer.image: -------------------------------------------------------------------------------- 1 | ../../../common/quadlet/app.image -------------------------------------------------------------------------------- /recipes/natural_language_processing/summarizer/quadlet/summarizer.kube: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Python script to run against downloaded LLM 3 | Documentation=man:podman-generate-systemd(1) 4 | Wants=network-online.target 5 | After=network-online.target 6 | RequiresMountsFor=%t/containers 7 | 8 | [Kube] 9 | # Point to the yaml file in the same directory 10 | Yaml=summarizer.yaml 11 | 12 | [Service] 13 | Restart=always 14 | 15 | [Install] 16 | WantedBy=default.target 17 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/summarizer/quadlet/summarizer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | labels: 5 | app: summarizer 6 | name: summarizer 7 | spec: 8 | initContainers: 9 | - name: model-file 10 | image: MODEL_IMAGE 11 | command: ['/usr/bin/install', "/model/model.file", "/shared/"] 12 | volumeMounts: 13 | - name: model-file 14 | mountPath: /shared 15 | containers: 16 | - env: 17 | - name: MODEL_ENDPOINT 18 | value: http://0.0.0.0:8001 19 | image: APP_IMAGE 20 | name: summarizer-inference 21 | ports: 22 | - containerPort: 8501 23 | hostPort: 8501 24 | securityContext: 25 | runAsNonRoot: true 26 | - env: 27 | - name: HOST 28 | value: 0.0.0.0 29 | - name: PORT 30 | value: 8001 31 | - name: MODEL_PATH 32 | value: /model/model.file 33 | image: SERVER_IMAGE 34 | name: summarizer-model-service 35 | ports: 36 | - containerPort: 8001 37 | hostPort: 8001 38 | securityContext: 39 | runAsNonRoot: true 40 | volumeMounts: 41 | - name: model-file 42 | mountPath: /model 43 | volumes: 44 | - name: model-file 45 | emptyDir: {} 46 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | 4 | 5 | @pytest.fixture 6 | def chrome_options(chrome_options): 7 | chrome_options.add_argument("--headless") 8 | return chrome_options 9 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/tests/functional/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/recipes/natural_language_processing/tests/functional/__init__.py -------------------------------------------------------------------------------- /recipes/natural_language_processing/tests/functional/test_app.py: -------------------------------------------------------------------------------- 1 | import pytest_container 2 | from .conftest import CB 3 | import tenacity 4 | 5 | CONTAINER_IMAGES = [CB] 6 | 7 | def test_etc_os_release_present(auto_container: pytest_container.container.ContainerData): 8 | assert auto_container.connection.file("/etc/os-release").exists 9 | 10 | @tenacity.retry(stop=tenacity.stop_after_attempt(5), wait=tenacity.wait_exponential()) 11 | def test_alive(auto_container: pytest_container.container.ContainerData, host): 12 | host.run_expect([0],f"curl http://localhost:{auto_container.forwarded_ports[0].host_port}",).stdout.strip() 13 | 14 | def test_title(auto_container: pytest_container.container.ContainerData, selenium): 15 | selenium.get(f"http://localhost:{auto_container.forwarded_ports[0].host_port}") 16 | assert selenium.title == "Streamlit" 17 | 18 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/recipes/natural_language_processing/tests/integration/__init__.py -------------------------------------------------------------------------------- /recipes/natural_language_processing/tests/integration/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | 4 | 5 | @pytest.fixture() 6 | def url(): 7 | return os.environ["URL"] 8 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/tests/integration/test_app.py: -------------------------------------------------------------------------------- 1 | def test_title(url,selenium): 2 | selenium.get(f"http://{url}:8501") 3 | assert selenium.title == "Streamlit" 4 | -------------------------------------------------------------------------------- /recipes/natural_language_processing/tests/requirements.txt: -------------------------------------------------------------------------------- 1 | pip==24.0 2 | pytest-container==0.4.0 3 | pytest-selenium==4.1.0 4 | pytest-testinfra==10.1.0 5 | pytest==8.1.1 6 | requests==2.31.0 7 | selenium==4.19.0 8 | tenacity==8.2.3 9 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "github>platform-engineering-org/.github" 5 | ], 6 | "packageRules": [ 7 | { 8 | "matchPackageNames": ["langchain"], 9 | "allowedVersions": "<0.2.3" 10 | }, 11 | { 12 | "matchPackageNames": ["langchain-openai"], 13 | "allowedVersions": "<=0.1.7" 14 | }, 15 | { 16 | "matchPackageNames": ["langchain-community"], 17 | "allowedVersions": "<=0.2.4" 18 | }, 19 | { 20 | "matchPackageNames": ["pymilvus"], 21 | "allowedVersions": "<=2.4.1" 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | pip==24.0 2 | pytest-container==0.4.3 3 | pytest-selenium==4.1.0 4 | pytest-testinfra==10.1.1 5 | pytest==8.2.2 6 | requests==2.31.0 7 | selenium==4.20.0 8 | tenacity==8.2.3 9 | -------------------------------------------------------------------------------- /training/amd-bootc/Makefile: -------------------------------------------------------------------------------- 1 | HARDWARE ?= amd 2 | IMAGE_NAME ?= $(HARDWARE)-bootc 3 | 4 | include ../common/Makefile.common 5 | 6 | default: bootc 7 | 8 | .PHONY: bootc 9 | bootc: prepare-files 10 | "${CONTAINER_TOOL}" build \ 11 | $(ARCH:%=--platform linux/%) \ 12 | $(BUILD_ARG_FILE:%=--build-arg-file=%) \ 13 | $(EXTRA_RPM_PACKAGES:%=--build-arg EXTRA_RPM_PACKAGES=%) \ 14 | $(DRIVER_TOOLKIT_IMAGE:%=--build-arg DRIVER_TOOLKIT_IMAGE=%) \ 15 | $(FROM:%=--build-arg BASEIMAGE=%) \ 16 | $(INSTRUCTLAB_IMAGE:%=--build-arg INSTRUCTLAB_IMAGE=%) \ 17 | $(SOURCE_DATE_EPOCH:%=--timestamp=%) \ 18 | $(VENDOR:%=--build-arg VENDOR=%) \ 19 | $(if $(SSH_PUBKEY),--build-arg SSHPUBKEY='$(SSH_PUBKEY)') \ 20 | --cap-add SYS_ADMIN \ 21 | --file Containerfile \ 22 | --security-opt label=disable \ 23 | --tag "${BOOTC_IMAGE}" \ 24 | -v ${OUTDIR}:/run/.input:ro \ 25 | ${CONTAINER_TOOL_EXTRA_ARGS} . 26 | -------------------------------------------------------------------------------- /training/amd-bootc/containers-storage.conf: -------------------------------------------------------------------------------- 1 | [storage] 2 | driver = "overlay" 3 | 4 | [storage.options] 5 | size = "" 6 | remap-uids = "" 7 | remap-gids = "" 8 | ignore_chown_errors = "" 9 | remap-user = "" 10 | remap-group = "" 11 | skip_mount_home = "" 12 | mount_program = "/usr/bin/fuse-overlayfs" 13 | mountopt = "" 14 | additionalimagestores = [ "/usr/lib/containers/storage",] 15 | 16 | [storage.options.overlay] 17 | force_mask = "shared" 18 | -------------------------------------------------------------------------------- /training/amd-bootc/repos.d/amdgpu.repo: -------------------------------------------------------------------------------- 1 | [amdgpu] 2 | name=amdgpu 3 | baseurl=https://repo.radeon.com/amdgpu/6.2/el/9.4/main/x86_64/ 4 | enabled=1 5 | priority=50 6 | gpgcheck=1 7 | gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key 8 | -------------------------------------------------------------------------------- /training/amd-bootc/repos.d/rocm.repo: -------------------------------------------------------------------------------- 1 | [ROCm-6.2] 2 | name=ROCm6.2 3 | baseurl=https://repo.radeon.com/rocm/el9/6.2/main 4 | enabled=1 5 | priority=50 6 | gpgcheck=1 7 | gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key 8 | -------------------------------------------------------------------------------- /training/cloud/Containerfile: -------------------------------------------------------------------------------- 1 | ARG BASEIMAGE=quay.io/ai-labs/bootc-nvidia:latest 2 | FROM ${BASEIMAGE} 3 | 4 | ARG CLOUD 5 | 6 | COPY $CLOUD/cloud-setup.sh /tmp 7 | RUN /tmp/cloud-setup.sh && rm -f /tmp/cloud-setup.sh 8 | COPY $CLOUD/files/ / 9 | -------------------------------------------------------------------------------- /training/cloud/Makefile: -------------------------------------------------------------------------------- 1 | CLOUD ?= 2 | VERSION ?= 1.1 3 | HARDWARE ?= nvidia 4 | REGISTRY ?= quay.io 5 | REGISTRY_ORG ?= ai-lab 6 | IMAGE_NAME ?= bootc-${HARDWARE}-rhel9-${CLOUD} 7 | IMAGE_TAG ?= ${VERSION} 8 | CONTAINER_TOOL ?= podman 9 | CONTAINER_TOOL_EXTRA_ARGS ?= 10 | 11 | BOOTC_IMAGE_CLOUD ?= ${REGISTRY}/${REGISTRY_ORG}/${IMAGE_NAME}:${IMAGE_TAG} 12 | 13 | default: help 14 | 15 | -include $(CLOUD)/Makefile.env 16 | 17 | cloud-image: ## Create bootc image for a cloud, using stable RHEL AI as base 18 | "${CONTAINER_TOOL}" build \ 19 | $(BASEIMAGE:%=--build-arg BASEIMAGE=%) \ 20 | $(CLOUD:%=--build-arg CLOUD=%) \ 21 | ${CONTAINER_TOOL_EXTRA_ARGS} \ 22 | --tag ${BOOTC_IMAGE_CLOUD} \ 23 | --file Containerfile \ 24 | . 25 | 26 | cloud-disk: ## Create disk image for a cloud, using the image built with cloud-image target 27 | make -f ../common/Makefile.common bootc-image-builder \ 28 | BOOTC_IMAGE=${BOOTC_IMAGE_CLOUD} \ 29 | DISK_TYPE=${DISK_TYPE} \ 30 | IMAGE_BUILDER_CONFIG=$(abspath $(CLOUD))/config.toml 31 | 32 | help: ## Shows this message. 33 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(shell echo "$(MAKEFILE_LIST) " | tac -s' ') | perl -pe 's/^.*Makefile.*?://g' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 34 | 35 | -------------------------------------------------------------------------------- /training/cloud/aws/Makefile.env: -------------------------------------------------------------------------------- 1 | DISK_TYPE=ami 2 | -------------------------------------------------------------------------------- /training/cloud/aws/README.md: -------------------------------------------------------------------------------- 1 | # Amazon Web Services modifications for RHEL AI 2 | Trying to mimic as much as possible the [changes on RHEL for AWS](https://github.com/osbuild/images/blob/main/pkg/distro/rhel/rhel9/ami.go) 3 | 4 | ## Changes 5 | 6 | - Extra kernel parameters 7 | 8 | ``` 9 | console=ttyS0,115200n8 net.ifnames=0 nvme_core.io_timeout=4294967295 10 | ``` 11 | 12 | - Timezone: UTC 13 | - Chrony configuration: 14 | - Change server 15 | - LeapsecTz 16 | - Locale: en_US.UTF-8 17 | - Keymap: us 18 | - X11 layout: us 19 | 20 | - Getty configuration 21 | - NautoVTs false 22 | 23 | - Packages 24 | - @core metapackage 25 | - authselect-compat 26 | - langpacks-en 27 | - tuned 28 | 29 | - Services 30 | - nm-cloud-setup.service 31 | - nm-cloud-setup.timer 32 | - tuned 33 | -------------------------------------------------------------------------------- /training/cloud/aws/cloud-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | mv /etc/selinux /etc/selinux.tmp 6 | dnf install -y --nobest \ 7 | cloud-init \ 8 | langpacks-en \ 9 | tuned 10 | mv /etc/selinux.tmp /etc/selinux 11 | 12 | # Chrony configuration 13 | sed -i \ 14 | -e '/^pool /c\server 169.254.169.123 prefer iburst minpoll 4 maxpoll 4' \ 15 | -e '/^leapsectz /d' \ 16 | /etc/chrony.conf 17 | -------------------------------------------------------------------------------- /training/cloud/aws/config.toml: -------------------------------------------------------------------------------- 1 | [customizations.kernel] 2 | name = "customizations-for-aws" 3 | append = "console=ttyS0,115200n8 net.ifnames=0 nvme_core.io_timeout=4294967295" 4 | 5 | -------------------------------------------------------------------------------- /training/cloud/aws/files/etc/X11/xorg.conf.d/00-keyboard.conf: -------------------------------------------------------------------------------- 1 | # Do not edit manually, use localectl(1). 2 | Section "InputClass" 3 | Identifier "system-keyboard" 4 | MatchIsKeyboard "on" 5 | Option "XkbLayout" "us" 6 | EndSection 7 | -------------------------------------------------------------------------------- /training/cloud/aws/files/etc/locale.conf: -------------------------------------------------------------------------------- 1 | LANG=en_US.UTF-8 2 | -------------------------------------------------------------------------------- /training/cloud/aws/files/etc/localtime: -------------------------------------------------------------------------------- 1 | ../usr/share/zoneinfo/UTC -------------------------------------------------------------------------------- /training/cloud/aws/files/etc/systemd/system/NetworkManager.service.wants/nm-cloud-setup.service: -------------------------------------------------------------------------------- 1 | /usr/lib/systemd/system/nm-cloud-setup.service -------------------------------------------------------------------------------- /training/cloud/aws/files/etc/systemd/system/sshd-keygen@.service.d/disable-sshd-keygen-if-cloud-init-active.conf: -------------------------------------------------------------------------------- 1 | # In some cloud-init enabled images the sshd-keygen template service may race 2 | # with cloud-init during boot causing issues with host key generation. This 3 | # drop-in config adds a condition to sshd-keygen@.service if it exists and 4 | # prevents the sshd-keygen units from running *if* cloud-init is going to run. 5 | # 6 | [Unit] 7 | ConditionPathExists=!/run/systemd/generator.early/multi-user.target.wants/cloud-init.target 8 | -------------------------------------------------------------------------------- /training/cloud/aws/files/etc/systemd/system/timers.target.wants/nm-cloud-setup.timer: -------------------------------------------------------------------------------- 1 | /usr/lib/systemd/system/nm-cloud-setup.timer -------------------------------------------------------------------------------- /training/cloud/aws/files/etc/vconsole.conf: -------------------------------------------------------------------------------- 1 | KEYMAP=us 2 | -------------------------------------------------------------------------------- /training/cloud/aws/files/usr/lib/bootc/install/05-cloud-kargs.toml: -------------------------------------------------------------------------------- 1 | [install] 2 | kargs = ["console=tty0", "console=ttyS0,115200n8", "net.ifnames=0", "nvme_core.io_timeout=4294967295"] 3 | -------------------------------------------------------------------------------- /training/cloud/aws/files/usr/lib/systemd/logind.conf.d/00-getty-fixes.conf: -------------------------------------------------------------------------------- 1 | [Login] 2 | NAutoVTs=0 3 | 4 | -------------------------------------------------------------------------------- /training/cloud/aws/files/usr/lib/systemd/system/nm-cloud-setup.service.d/10-rh-enable-for-ec2.conf: -------------------------------------------------------------------------------- 1 | [Service] 2 | Environment="NM_CLOUD_SETUP_EC2=yes" 3 | -------------------------------------------------------------------------------- /training/cloud/azure/Makefile.env: -------------------------------------------------------------------------------- 1 | DISK_TYPE=raw 2 | -------------------------------------------------------------------------------- /training/cloud/azure/README.md: -------------------------------------------------------------------------------- 1 | # Azure for RHEL AI 2 | Trying to mimic as much as possible the [changes on RHEL for Azure](https://github.com/osbuild/images/blob/main/pkg/distro/rhel/rhel9/azure.go) 3 | 4 | # Summary 5 | - Extra kernel parameters 6 | 7 | Even if in the link [Kernel Parameters on RHEL for Azure](https://github.com/osbuild/images/blob/a4ae81dc3eed3e86c359635e3135fc8a07f411dd/pkg/distro/rhel/rhel9/azure.go#L454) we see other changes, when running a RHEL instance in Azure, the extra kernel parameters are others, so we will take those as our reference 8 | ``` 9 | loglevel=3 console=tty1 console=ttyS0,115200n8 earlyprintk=ttyS0,115200 net.ifnames=0 cloud-init=disabled 10 | ``` 11 | 12 | Note that we also disable cloud-init via kernel parameter 13 | 14 | - Timezone: UTC 15 | - Locale: en_US.UTF-8 16 | - Keymap: us 17 | - X11 layout: us 18 | 19 | - sshd config 20 | - ClientAliveInterval: 180 21 | 22 | - Packages 23 | - hyperv-daemons 24 | - langpacks-en 25 | - NetworkManager-cloud-setup 26 | - nvme-cli 27 | - patch 28 | - rng-tools 29 | - uuid 30 | - WALinuxAgent 31 | 32 | - Services 33 | - nm-cloud-setup.service 34 | - nm-cloud-setup.timer 35 | - waagent 36 | 37 | - Systemd 38 | - nm-cloud-setup.service: `Environment=NM_CLOUD_SETUP_AZURE=yes` 39 | 40 | - Kernel Modules 41 | - blacklist amdgpu 42 | - blacklist intel_cstate 43 | - blacklist floppy 44 | - blacklist nouveau 45 | - blacklist lbm-nouveau 46 | - blacklist skx_edac 47 | 48 | - Cloud Init 49 | - 10-azure-kvp.cfg 50 | - 91-azure_datasource.cfg 51 | 52 | - PwQuality 53 | - /etc/security/pwquality.conf 54 | 55 | - WaAgentConfig 56 | - RDFormat false 57 | - RDEnableSwap false 58 | 59 | - udev rules 60 | - /etc/udev/rules.d/68-azure-sriov-nm-unmanaged.rules 61 | -------------------------------------------------------------------------------- /training/cloud/azure/cloud-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | mv /etc/selinux /etc/selinux.tmp 6 | dnf install -y --nobest \ 7 | cloud-init \ 8 | hyperv-daemons \ 9 | langpacks-en \ 10 | NetworkManager-cloud-setup \ 11 | nvme-cli \ 12 | patch \ 13 | rng-tools \ 14 | uuid \ 15 | WALinuxAgent 16 | mv /etc/selinux.tmp /etc/selinux 17 | 18 | # sshd configuration 19 | cat << EOF >> /etc/ssh/sshd_config 20 | ClientAliveInterval 180 21 | EOF 22 | 23 | # pwquality configuration 24 | cat << EOF >> /etc/security/pwquality.conf 25 | minlen = 6 26 | dcredit = 0 27 | ucredit = 0 28 | lcredit = 0 29 | ocredit = 0 30 | minclass = 3 31 | EOF 32 | 33 | # WAAgent configuration 34 | sed -i \ 35 | -e '/^ResourceDisk.Format=y/c\ResourceDisk.Format=n' \ 36 | -e '/^ResourceDisk.EnableSwap=y/c\ResourceDisk.EnableSwap=n' \ 37 | -e '/^Provisioning.RegenerateSshHostKeyPair=y/c\Provisioning.RegenerateSshHostKeyPair=n' \ 38 | /etc/waagent.conf 39 | -------------------------------------------------------------------------------- /training/cloud/azure/config.toml: -------------------------------------------------------------------------------- 1 | [customizations.kernel] 2 | name = "customizations-for-azure" 3 | # This is suggested by https://github.com/osbuild/images/blob/a4ae81dc3eed3e86c359635e3135fc8a07f411dd/pkg/distro/rhel/rhel9/azure.go#L454 4 | # append = "ro loglevel=3 console=tty1 console=ttyS0 earlyprintk=ttyS0 rootdelay=300" 5 | # However, starting a RHEL instance in azure shows this one, and I'll be using it 6 | append = "loglevel=3 console=tty1 console=ttyS0,115200n8 earlyprintk=ttyS0,115200 net.ifnames=0" 7 | -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/X11/xorg.conf.d/00-keyboard.conf: -------------------------------------------------------------------------------- 1 | # Do not edit manually, use localectl(1). 2 | Section "InputClass" 3 | Identifier "system-keyboard" 4 | MatchIsKeyboard "on" 5 | Option "XkbLayout" "us" 6 | EndSection 7 | -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/cloud/cloud.cfg.d/10-azure-kvp.cfg: -------------------------------------------------------------------------------- 1 | # This configuration file is used to enable logging to Hyper-V kvp 2 | reporting: 3 | logging: 4 | type: log 5 | telemetry: 6 | type: hyperv 7 | -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/cloud/cloud.cfg.d/91-azure_datasource.cfg: -------------------------------------------------------------------------------- 1 | datasource_list: [ Azure ] 2 | datasource: 3 | Azure: 4 | apply_network_config: False 5 | -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/locale.conf: -------------------------------------------------------------------------------- 1 | LANG=en_US.UTF-8 2 | -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/localtime: -------------------------------------------------------------------------------- 1 | ../usr/share/zoneinfo/UTC -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/modprobe.d/blacklist-floppy.conf: -------------------------------------------------------------------------------- 1 | blacklist floppy 2 | -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/modprobe.d/blacklist-intel-cstate.conf: -------------------------------------------------------------------------------- 1 | blacklist intel_cstate 2 | -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/modprobe.d/blacklist-nouveau.conf: -------------------------------------------------------------------------------- 1 | blacklist nouveau 2 | blacklist lbm-nouveau 3 | -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/modprobe.d/blacklist-skylake-edac.conf: -------------------------------------------------------------------------------- 1 | blacklist skx_edac 2 | -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/systemd/system/NetworkManager.service.wants/nm-cloud-setup.service: -------------------------------------------------------------------------------- 1 | /usr/lib/systemd/system/nm-cloud-setup.service -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/systemd/system/multi-user.target.wants/waagent.service: -------------------------------------------------------------------------------- 1 | /usr/lib/systemd/system/waagent.service -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/systemd/system/timers.target.wants/nm-cloud-setup.timer: -------------------------------------------------------------------------------- 1 | /usr/lib/systemd/system/nm-cloud-setup.timer -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/udev/rules.d/68-azure-sriov-nm-unmanaged.rules: -------------------------------------------------------------------------------- 1 | # Accelerated Networking on Azure exposes a new SRIOV interface to the VM. 2 | # This interface is transparently bonded to the synthetic interface, 3 | # so NetworkManager should just ignore any SRIOV interfaces. 4 | SUBSYSTEM=="net", DRIVERS=="hv_pci", ACTION=="add", ENV{NM_UNMANAGED}="1" 5 | -------------------------------------------------------------------------------- /training/cloud/azure/files/etc/vconsole.conf: -------------------------------------------------------------------------------- 1 | KEYMAP=us 2 | -------------------------------------------------------------------------------- /training/cloud/azure/files/usr/lib/bootc/install/05-cloud-kargs.toml: -------------------------------------------------------------------------------- 1 | [install] 2 | # This is suggested by https://github.com/osbuild/images/blob/a4ae81dc3eed3e86c359635e3135fc8a07f411dd/pkg/distro/rhel/rhel9/azure.go#L454 3 | # kargs = ["ro", "loglevel=3", "console=tty1", "console=ttyS0", "earlyprintk=ttyS0", "rootdelay=300"] 4 | # However, starting a RHEL instance in azure shows this one, and I'll be using it 5 | kargs = ["loglevel=3", "console=tty1", "console=ttyS0,115200n8", "earlyprintk=ttyS0,115200", "net.ifnames=0"] 6 | -------------------------------------------------------------------------------- /training/cloud/azure/files/usr/lib/systemd/system/nm-cloud-setup.service.d/10-rh-enable-for-azure.conf: -------------------------------------------------------------------------------- 1 | [Service] 2 | Environment="NM_CLOUD_SETUP_AZURE=yes" 3 | -------------------------------------------------------------------------------- /training/cloud/gcp/Makefile.env: -------------------------------------------------------------------------------- 1 | DISK_TYPE=raw 2 | -------------------------------------------------------------------------------- /training/cloud/gcp/README.md: -------------------------------------------------------------------------------- 1 | # Google Cloud Platform modifications for RHEL AI 2 | Trying to mimic as much as possible the [changes on RHEL for GCP](https://github.com/osbuild/images/blob/main/pkg/distro/rhel/rhel9/gce.go) 3 | 4 | ## Changes 5 | 6 | - Extra kernel parameters 7 | 8 | ``` 9 | net.ifnames=0 biosdevname=0 scsi_mod.use_blk_mq=Y console=ttyS0,38400n8d 10 | ``` 11 | 12 | - Timezone: UTC 13 | - Chrony configuration: 14 | - Change server 15 | - Locale: en_US.UTF-8 16 | - Keymap: us 17 | - X11 layout: us 18 | 19 | - sshd config 20 | - PasswordAuthentication: false 21 | - ClientAliveInterval: 420 22 | - PermitRootLogin: No 23 | 24 | - Modules 25 | - blacklist floppy 26 | 27 | - GCPGuestAgentConfig 28 | - SetBotoConfig: false 29 | 30 | - Packages 31 | - langpacks-en 32 | - acpid 33 | - rng-tools 34 | - vim 35 | - google-compute-engine 36 | - google-osconfig-agent 37 | - gce-disk-expand 38 | - timedatex 39 | - tuned 40 | 41 | - Remove Packages 42 | - irqbalance 43 | - microcode_ctl 44 | -------------------------------------------------------------------------------- /training/cloud/gcp/cloud-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | eval $(grep VERSION_ID /etc/os-release) 6 | tee /etc/yum.repos.d/google-cloud.repo << EOF 7 | [google-compute-engine] 8 | name=Google Compute Engine 9 | baseurl=https://packages.cloud.google.com/yum/repos/google-compute-engine-el${VERSION_ID/.*}-x86_64-stable 10 | enabled=1 11 | gpgcheck=1 12 | repo_gpgcheck=0 13 | gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg 14 | https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg 15 | EOF 16 | 17 | mv /etc/selinux /etc/selinux.tmp 18 | dnf install -y --nobest \ 19 | acpid \ 20 | cloud-init \ 21 | google-compute-engine \ 22 | google-osconfig-agent \ 23 | langpacks-en \ 24 | rng-tools \ 25 | timedatex \ 26 | tuned \ 27 | vim 28 | mv /etc/selinux.tmp /etc/selinux 29 | 30 | # The current version of google-cloud-ops-agent is impacted by a CVE: https://access.redhat.com/security/cve/CVE-2024-41110 31 | # It will be disable for the meantime 32 | # 33 | # # Install Google Ops Agent 34 | # curl -sSo /tmp/add-google-cloud-ops-agent-repo.sh https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh 35 | # bash /tmp/add-google-cloud-ops-agent-repo.sh --also-install --remove-repo 36 | # rm /tmp/add-google-cloud-ops-agent-repo.sh 37 | 38 | # rpm-state is needed to remove microcode_ctl 39 | mkdir -p /var/lib/rpm-state 40 | dnf remove -y \ 41 | irqbalance \ 42 | microcode_ctl 43 | 44 | rm -f /etc/yum.repos.d/google-cloud.repo 45 | 46 | # Chrony configuration 47 | sed -i \ 48 | -e '/^pool /c\server metadata.google.internal iburst' \ 49 | /etc/chrony.conf 50 | 51 | # sshd configuration 52 | cat << EOF >> /etc/ssh/sshd_config 53 | PermitRootLogin no 54 | PasswordAuthentication no 55 | ClientAliveInterval 420 56 | EOF 57 | -------------------------------------------------------------------------------- /training/cloud/gcp/config.toml: -------------------------------------------------------------------------------- 1 | [customizations.kernel] 2 | name = "customizations-for-gcp" 3 | append = "net.ifnames=0 biosdevname=0 scsi_mod.use_blk_mq=Y console=ttyS0,38400n8d cloud-init=disabled" 4 | 5 | -------------------------------------------------------------------------------- /training/cloud/gcp/files/etc/X11/xorg.conf.d/00-keyboard.conf: -------------------------------------------------------------------------------- 1 | # Do not edit manually, use localectl(1). 2 | Section "InputClass" 3 | Identifier "system-keyboard" 4 | MatchIsKeyboard "on" 5 | Option "XkbLayout" "us" 6 | EndSection 7 | -------------------------------------------------------------------------------- /training/cloud/gcp/files/etc/default/instance_configs.cfg: -------------------------------------------------------------------------------- 1 | # Disable boto plugin setup. 2 | [InstanceSetup] 3 | set_boto_config = false 4 | -------------------------------------------------------------------------------- /training/cloud/gcp/files/etc/locale.conf: -------------------------------------------------------------------------------- 1 | LANG=en_US.UTF-8 2 | -------------------------------------------------------------------------------- /training/cloud/gcp/files/etc/localtime: -------------------------------------------------------------------------------- 1 | ../usr/share/zoneinfo/UTC -------------------------------------------------------------------------------- /training/cloud/gcp/files/etc/modprobe.d/blacklist-floppy.conf: -------------------------------------------------------------------------------- 1 | blacklist floppy 2 | -------------------------------------------------------------------------------- /training/cloud/gcp/files/etc/vconsole.conf: -------------------------------------------------------------------------------- 1 | KEYMAP=us 2 | -------------------------------------------------------------------------------- /training/cloud/gcp/files/usr/lib/bootc/install/05-cloud-kargs.toml: -------------------------------------------------------------------------------- 1 | [install] 2 | kargs = ["net.ifnames=0", "biosdevname=0", "scsi_mod.use_blk_mq=Y", "console=ttyS0,38400n8d", "cloud-init=disabled"] 3 | -------------------------------------------------------------------------------- /training/cloud/ibm/Makefile.env: -------------------------------------------------------------------------------- 1 | DISK_TYPE=qcow2 2 | -------------------------------------------------------------------------------- /training/cloud/ibm/README.md: -------------------------------------------------------------------------------- 1 | # IBM Cloud modifications for RHEL AI 2 | 3 | ## Changes 4 | 5 | - Timezone: UTC 6 | - Chrony configuration: 7 | - Change server 8 | - LeapsecTz 9 | - Locale: en_US.UTF-8 10 | - Keymap: us 11 | - X11 layout: us 12 | 13 | - Packages 14 | - langpacks-en 15 | -------------------------------------------------------------------------------- /training/cloud/ibm/cloud-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | mv /etc/selinux /etc/selinux.tmp 6 | dnf install -y --nobest \ 7 | cloud-init \ 8 | langpacks-en 9 | mv /etc/selinux.tmp /etc/selinux 10 | -------------------------------------------------------------------------------- /training/cloud/ibm/config.toml: -------------------------------------------------------------------------------- 1 | [customizations.kernel] 2 | 3 | -------------------------------------------------------------------------------- /training/cloud/ibm/files/etc/X11/xorg.conf.d/00-keyboard.conf: -------------------------------------------------------------------------------- 1 | # Do not edit manually, use localectl(1). 2 | Section "InputClass" 3 | Identifier "system-keyboard" 4 | MatchIsKeyboard "on" 5 | Option "XkbLayout" "us" 6 | EndSection 7 | -------------------------------------------------------------------------------- /training/cloud/ibm/files/etc/locale.conf: -------------------------------------------------------------------------------- 1 | LANG=en_US.UTF-8 2 | -------------------------------------------------------------------------------- /training/cloud/ibm/files/etc/localtime: -------------------------------------------------------------------------------- 1 | ../usr/share/zoneinfo/UTC -------------------------------------------------------------------------------- /training/cloud/ibm/files/etc/vconsole.conf: -------------------------------------------------------------------------------- 1 | KEYMAP=us 2 | -------------------------------------------------------------------------------- /training/common/usr/lib/systemd/system/basic.target.wants/upgrade-informer.service: -------------------------------------------------------------------------------- 1 | ../upgrade-informer.service -------------------------------------------------------------------------------- /training/common/usr/lib/systemd/system/timers.target.wants/upgrade-informer.timer: -------------------------------------------------------------------------------- 1 | ../upgrade-informer.timer -------------------------------------------------------------------------------- /training/common/usr/lib/systemd/system/upgrade-informer.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Check for available RHEL AI upgrade 3 | ConditionPathExists=/run/ostree-booted 4 | After=network-online.target 5 | StartLimitIntervalSec=400 6 | StartLimitBurst=3 7 | 8 | [Service] 9 | Type=oneshot 10 | ExecStart=/usr/libexec/upgrade-informer 11 | Restart=on-failure 12 | RestartSec=90 13 | -------------------------------------------------------------------------------- /training/common/usr/lib/systemd/system/upgrade-informer.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Runs upgrade informer periodically 3 | ConditionPathExists=/run/ostree-booted 4 | 5 | [Timer] 6 | OnBootSec=1h 7 | OnUnitInactiveSec=1day 8 | RandomizedDelaySec=2h 9 | 10 | [Install] 11 | WantedBy=timers.target 12 | -------------------------------------------------------------------------------- /training/common/usr/libexec/upgrade-informer: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Run the command and capture its output 4 | output=$(bootc upgrade --check | sed -e 1q) 5 | message_file="/etc/motd.d/upgrade-message" 6 | bootc_auth="/etc/ostree/auth.json" 7 | 8 | if [[ $output == Update\ available* ]]; then 9 | if [[ ! -f $message_file ]]; then 10 | echo "New version was found" 11 | bootc_image=$(awk '{print $4}' <<< "$output") 12 | # If auth file exists we should use it 13 | auth_params="" 14 | if [[ -f $bootc_auth ]]; then 15 | auth_params="--authfile $bootc_auth" 16 | fi 17 | 18 | # Get image version 19 | # shellcheck disable=SC2086 20 | image_version_id=$(skopeo inspect --format json $auth_params "$bootc_image" | jq -r '.Labels | .["image_version_id"] // empty') 21 | 22 | # If upgrade available, write the output to the file 23 | cat > $message_file << EOF 24 | 25 | ** Attention! ** 26 | ** A new $image_version_id version is available ** 27 | ** In order to apply it run: bootc upgrade --apply 28 | ** Please note that the system will reboot after the upgrade ** 29 | 30 | EOF 31 | fi 32 | else 33 | echo "No upgrade was found" 34 | rm $message_file 2> /dev/null 35 | fi 36 | 37 | echo "Finished running upgrade informer" 38 | -------------------------------------------------------------------------------- /training/deepspeed/Containerfile: -------------------------------------------------------------------------------- 1 | # Containerfile for running deepspeed training 2 | FROM nvcr.io/nvidia/cuda:12.1.1-cudnn8-devel-ubi9 3 | 4 | ARG VENDOR='' 5 | LABEL vendor=${VENDOR} 6 | LABEL org.opencontainers.image.vendor=${VENDOR} 7 | 8 | RUN dnf install -y python python-devel git 9 | RUN python -m ensurepip --upgrade 10 | RUN pip3 install torch==2.1.2 --index-url https://download.pytorch.org/whl/cu121 11 | RUN pip3 install packaging wheel 12 | RUN pip3 install flash-attn==2.5.7 13 | RUN pip3 install deepspeed==0.14.2 14 | RUN pip3 install transformers==4.40.1 15 | RUN pip3 install ipdb jupyterlab gpustat matplotlib hydra-core datasets rich numba 16 | RUN git clone https://github.com/instructlab/training.git 17 | RUN mkdir -p /ilab-data/training_output 18 | 19 | WORKDIR /training 20 | 21 | CMD ["/bin/bash"] 22 | -------------------------------------------------------------------------------- /training/deepspeed/Makefile: -------------------------------------------------------------------------------- 1 | IMAGE_NAME ?= deepspeed-trainer 2 | CONTAINER_TOOL ?= podman 3 | SOURCE_DATE_EPOCH ?= $(shell git log -1 --pretty=%ct) 4 | BUILD_ARG_FILE ?= 5 | 6 | default: image 7 | 8 | .PHONY: image 9 | image: 10 | @mkdir -p ../build 11 | rm -rf ../build/deepspeed-trainer 12 | "${CONTAINER_TOOL}" build \ 13 | $(ARCH:%=--platform linux/%) \ 14 | $(BUILD_ARG_FILE:%=--build-arg-file=%) \ 15 | $(SOURCE_DATE_EPOCH:%=--timestamp=%) \ 16 | $(VENDOR:%=--build-arg VENDOR=%) \ 17 | --file Containerfile \ 18 | --squash-all \ 19 | --tag oci:../build/deepspeed-trainer 20 | -------------------------------------------------------------------------------- /training/ilab-wrapper/ilab-qlora: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Template values replaced by container build 4 | TRAIN_DEVICE="__REPLACE_TRAIN_DEVICE__" 5 | CONTAINER_DEVICE="__REPLACE_CONTAINER_DEVICE__" 6 | CONTAINER_NAME="__REPLACE_CONTAINER_NAME__" 7 | 8 | # HF caching uses relative symlink structures, so keep cache relative to 9 | # the central working directory 10 | CONTAINER_CACHE="/instructlab/cache" 11 | HOST_CACHE="$(pwd)/cache" 12 | WORKDIR="$(pwd)" 13 | 14 | has_argument() { 15 | match=$1 16 | shift 17 | for arg in "$@"; do 18 | if [[ "$arg" == *"$match"* ]]; then 19 | return 0 20 | fi 21 | done 22 | return 1 23 | } 24 | 25 | mkdir -p "${HOST_CACHE}" 26 | PODMAN_COMMAND=("podman" "run" "--rm" "-it" "--device" "${CONTAINER_DEVICE}" \ 27 | "--security-opt" "label=disable" "--net" "host" \ 28 | "-v" "${WORKDIR}:/instructlab" "--entrypoint" "" \ 29 | "-e" "HF_HOME=${CONTAINER_CACHE}" \ 30 | "${CONTAINER_NAME}") 31 | if [[ "$1" = "init" ]]; then 32 | if ! has_argument "--repository" "$@"; then 33 | shift 34 | "${PODMAN_COMMAND[@]}" ilab init \ 35 | --repository https://github.com/instructlab/taxonomy.git "$@" 36 | exit $? 37 | fi 38 | elif [[ "$1" = "train" ]]; then 39 | if ! has_argument "--device" "$@"; then 40 | shift 41 | "${PODMAN_COMMAND[@]}" ilab train --device ${TRAIN_DEVICE} "$@" 42 | exit $? 43 | fi 44 | fi 45 | 46 | "${PODMAN_COMMAND[@]}" ilab "$@" 47 | 48 | -------------------------------------------------------------------------------- /training/intel-bootc/Makefile: -------------------------------------------------------------------------------- 1 | IMAGE_NAME ?= intel-bootc 2 | 3 | include ../common/Makefile.common 4 | 5 | default: bootc 6 | 7 | .PHONY: bootc 8 | bootc: prepare-files 9 | ${CONTAINER_TOOL} build \ 10 | $(ARCH:%=--platform linux/%) \ 11 | $(BUILD_ARG_FILE:%=--build-arg-file=%) \ 12 | $(DRIVER_TOOLKIT_IMAGE:%=--build-arg DRIVER_TOOLKIT_IMAGE=%) \ 13 | $(DRIVER_VERSION:%=--build-arg DRIVER_VERSION=%) \ 14 | $(EXTRA_RPM_PACKAGES:%=--build-arg EXTRA_RPM_PACKAGES=%) \ 15 | $(FROM:%=--build-arg BASEIMAGE=%) \ 16 | $(INSTRUCTLAB_IMAGE:%=--build-arg INSTRUCTLAB_IMAGE=%) \ 17 | $(SOURCE_DATE_EPOCH:%=--timestamp=%) \ 18 | $(VENDOR:%=--build-arg VENDOR=%) \ 19 | $(if $(SSH_PUBKEY),--build-arg SSHPUBKEY='$(SSH_PUBKEY)') \ 20 | --cap-add SYS_ADMIN \ 21 | --file Containerfile \ 22 | --security-opt label=disable \ 23 | --tag "${BOOTC_IMAGE}" \ 24 | -v ${OUTDIR}:/run/.input:ro \ 25 | ${CONTAINER_TOOL_EXTRA_ARGS} . 26 | -------------------------------------------------------------------------------- /training/intel-bootc/duplicated/common/usr/lib/systemd/system/basic.target.wants/upgrade-informer.service: -------------------------------------------------------------------------------- 1 | ../upgrade-informer.service -------------------------------------------------------------------------------- /training/intel-bootc/duplicated/common/usr/lib/systemd/system/timers.target.wants/upgrade-informer.timer: -------------------------------------------------------------------------------- 1 | ../upgrade-informer.timer -------------------------------------------------------------------------------- /training/intel-bootc/duplicated/common/usr/lib/systemd/system/upgrade-informer.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Check for available RHEL AI upgrade 3 | ConditionPathExists=/run/ostree-booted 4 | After=network-online.target 5 | StartLimitIntervalSec=400 6 | StartLimitBurst=3 7 | 8 | [Service] 9 | Type=oneshot 10 | ExecStart=/usr/libexec/upgrade-informer 11 | Restart=on-failure 12 | RestartSec=90 13 | -------------------------------------------------------------------------------- /training/intel-bootc/duplicated/common/usr/lib/systemd/system/upgrade-informer.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Runs upgrade informer periodically 3 | ConditionPathExists=/run/ostree-booted 4 | 5 | [Timer] 6 | OnBootSec=1h 7 | OnUnitInactiveSec=1day 8 | RandomizedDelaySec=2h 9 | 10 | [Install] 11 | WantedBy=timers.target 12 | -------------------------------------------------------------------------------- /training/intel-bootc/duplicated/common/usr/libexec/upgrade-informer: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Run the command and capture its output 4 | output=$(bootc upgrade --check | sed -e 1q) 5 | message_file="/etc/motd.d/upgrade-message" 6 | bootc_auth="/etc/ostree/auth.json" 7 | 8 | if [[ $output == Update\ available* ]]; then 9 | if [[ ! -f $message_file ]]; then 10 | echo "New version was found" 11 | bootc_image=$(awk '{print $4}' <<< "$output") 12 | # If auth file exists we should use it 13 | auth_params="" 14 | if [[ -f $bootc_auth ]]; then 15 | auth_params="--authfile $bootc_auth" 16 | fi 17 | 18 | # Get image version 19 | # shellcheck disable=SC2086 20 | image_version_id=$(skopeo inspect --format json $auth_params "$bootc_image" | jq -r '.Labels | .["image_version_id"] // empty') 21 | 22 | # If upgrade available, write the output to the file 23 | cat > $message_file << EOF 24 | 25 | ** Attention! ** 26 | ** A new $image_version_id version is available ** 27 | ** In order to apply it run: bootc upgrade --apply 28 | ** Please note that the system will reboot after the upgrade ** 29 | 30 | EOF 31 | fi 32 | else 33 | echo "No upgrade was found" 34 | rm $message_file 2> /dev/null 35 | fi 36 | 37 | echo "Finished running upgrade informer" 38 | -------------------------------------------------------------------------------- /training/intel-bootc/scripts/os_dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | REPOS_REPO="redhat/rhel-ai/wheels/builder.git" 3 | 4 | centos_habana_repos() { 5 | echo "[habanalabs]" > /etc/yum.repos.d/habanalabs.repo && \ 6 | echo "name=Habana RH9 Linux repo" >> /etc/yum.repos.d/habanalabs.repo && \ 7 | echo "baseurl=https://${ARTIFACTORY_URL}/artifactory/rhel/9/9.4" >> /etc/yum.repos.d/habanalabs.repo && \ 8 | echo "gpgkey=https://${ARTIFACTORY_URL}/artifactory/api/v2/repositories/rhel/keyPairs/primary/public" >> /etc/yum.repos.d/habanalabs.repo && \ 9 | echo "gpgcheck=1" >> /etc/yum.repos.d/habanalabs.repo && \ 10 | update-crypto-policies --set DEFAULT:SHA1 11 | } 12 | centos_epel_crb() { 13 | #EPEL only needed in CentOS for libsox-devel 14 | dnf config-manager --set-enabled crb && \ 15 | dnf install -y https://dl.fedoraproject.org/pub/epel/epel{,-next}-release-latest-9.noarch.rpm 16 | } 17 | OS=$(grep -w ID /etc/os-release) 18 | 19 | echo "OS line is $OS" 20 | if [[ "$OS" == *"rhel"* ]]; then \ 21 | mkdir -p /tmp/git && cd /tmp/git && \ 22 | GIT_TOKEN=$(cat /run/secrets/extra-secrets-intel-bootc/BUILDERS_TOKEN) && \ 23 | git clone https://dummy_user:${GIT_TOKEN}@gitlab.com/${REPOS_REPO} && \ 24 | cd builder/repos && \ 25 | cp redhat.repo rhelai.repo habanalabs.repo /etc/yum.repos.d/ && \ 26 | cp RPM-GPG-KEY-HABANALABS /etc/pki/rpm-gpg/ && \ 27 | dnf config-manager --enable habanalabs && \ 28 | dnf config-manager --enable rhelai-1.2-stage && \ 29 | rm -rf /tmp/git; 30 | elif [[ "$OS" == *"centos"* ]]; then \ 31 | centos_habana_repos && centos_epel_crb; \ 32 | else 33 | echo "Only RHEL and CentOS supported." 34 | fi 35 | 36 | -------------------------------------------------------------------------------- /training/model/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/ubi 2 | 3 | ARG VENDOR='' 4 | LABEL vendor=${VENDOR} 5 | LABEL org.opencontainers.image.vendor=${VENDOR} 6 | 7 | RUN dnf install -y python3-pip && python3 -m pip install huggingface_hub[cli] 8 | COPY entrypoint.sh /entrypoint.sh 9 | WORKDIR /download 10 | ENTRYPOINT ["bash", "/entrypoint.sh"] 11 | -------------------------------------------------------------------------------- /training/model/Makefile: -------------------------------------------------------------------------------- 1 | include ../common/Makefile.common 2 | 3 | TARGET_MODELS_IMAGE ?= $(BOOTC_MODELS_IMAGE) 4 | FROM_BOOTC_IMAGE ?= $(BOOTC_IMAGE) 5 | 6 | GRANITE_MODEL_REPO ?= instructlab/granite-7b-lab 7 | 8 | HF_TOKEN ?= $(shell echo $$HF_TOKEN) 9 | 10 | BUILD_MODELS_PATH := $(shell realpath ..)/build/models 11 | COMMON_PATH := $(shell realpath ../common) 12 | 13 | MODEL_REPO ?= 14 | 15 | default: download 16 | 17 | .PHONY: image 18 | image: 19 | "${CONTAINER_TOOL}" build \ 20 | --file Containerfile \ 21 | --tag $(REGISTRY)/$(REGISTRY_ORG)/model-downloader:latest \ 22 | ${CONTAINER_TOOL_EXTRA_ARGS} . 23 | 24 | .PHONY: download-all 25 | download-all: image 26 | $(MAKE) MODEL_REPO=$(GRANITE_MODEL_REPO) download-model 27 | 28 | .PHONY: download-model 29 | download-model: 30 | mkdir -p $(BUILD_MODELS_PATH) 31 | podman run \ 32 | -e HF_TOKEN=$(HF_TOKEN) \ 33 | -v $(BUILD_MODELS_PATH):/download:z \ 34 | --pull=never \ 35 | -e MODEL_REPO=$(MODEL_REPO) \ 36 | -t $(REGISTRY)/$(REGISTRY_ORG)/model-downloader:latest 37 | 38 | .PHONY: generate-model-cfile 39 | generate-model-cfile: download-all 40 | echo "FROM ${FROM_BOOTC_IMAGE}" > ${MODELS_CONTAINERFILE} 41 | echo "RUN rsync -ah --progress --exclude '.hug*' --exclude '*.safetensors' /run/.input/models /usr/share" >> ${MODELS_CONTAINERFILE} 42 | "${CONTAINER_TOOL}" run \ 43 | -v .:/work:z \ 44 | -v ${OUTDIR}:/run/.input:ro \ 45 | --pull=never \ 46 | --entrypoint python3 \ 47 | $(REGISTRY)/$(REGISTRY_ORG)/model-downloader:latest \ 48 | /work/generate-model-cfile.py /run/.input/models >> ${MODELS_CONTAINERFILE} 49 | 50 | .PHONY: bootc-models 51 | bootc-models: generate-model-cfile 52 | "${CONTAINER_TOOL}" build \ 53 | $(ARCH:%=--platform linux/%) \ 54 | --file ${MODELS_CONTAINERFILE} \ 55 | --security-opt label=disable \ 56 | --tag "${TARGET_MODELS_IMAGE}" \ 57 | -v ${OUTDIR}:/run/.input:ro 58 | -------------------------------------------------------------------------------- /training/model/entrypoint.sh: -------------------------------------------------------------------------------- 1 | set -x 2 | if [ -z "$HF_TOKEN" ]; then 3 | echo "Error. Please set your \$HF_TOKEN in env. Required to pull mixtral." 4 | exit 1 5 | fi 6 | 7 | huggingface-cli download --exclude "*.pt" --local-dir "/download/${MODEL_REPO}" "${MODEL_REPO}" 8 | -------------------------------------------------------------------------------- /training/model/generate-model-cfile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import sys 5 | 6 | def isHuggingDir(elements): 7 | for n in s: 8 | if n.startswith(".hug"): 9 | return True 10 | return False 11 | 12 | def printNonEmpty(*args): 13 | if len(args[0]) > 0: 14 | print(*args) 15 | 16 | dir = sys.argv[1] 17 | cwd = os.getcwd() 18 | os.chdir(dir) 19 | c = 0 20 | result="" 21 | for root, dirs, files in os.walk("."): 22 | s = root.split(os.path.sep) 23 | s.pop(0) 24 | if isHuggingDir(s): 25 | continue 26 | for file in files: 27 | if not file.endswith(".safetensors"): 28 | continue 29 | path = os.path.join("/run", ".input", "models", *s, file) 30 | partial = os.path.join("/usr", "share", "models", *s, file) 31 | cmd = f"rsync -ah --progress {path} {partial}" 32 | if c % 4 != 0: 33 | result = f"{result} \\\n\t && {cmd}" 34 | else: 35 | printNonEmpty(result) 36 | result = f"RUN {cmd}" 37 | c += 1 38 | printNonEmpty(result) 39 | os.chdir(cwd) 40 | -------------------------------------------------------------------------------- /training/nvidia-bootc/Makefile: -------------------------------------------------------------------------------- 1 | HARDWARE ?= nvidia 2 | IMAGE_NAME ?= $(HARDWARE)-bootc 3 | 4 | CUDA_VERSION ?= 5 | OS_VERSION_MAJOR ?= 6 | include ../common/Makefile.common 7 | 8 | default: bootc 9 | 10 | .PHONY: bootc 11 | bootc: driver-toolkit check-sshkey prepare-files 12 | "${CONTAINER_TOOL}" build \ 13 | $(ARCH:%=--platform linux/%) \ 14 | $(BUILD_ARG_FILE:%=--build-arg-file=%) \ 15 | $(CUDA_VERSION:%=--build-arg CUDA_VERSION=%) \ 16 | $(DRIVER_TOOLKIT_IMAGE:%=--build-arg DRIVER_TOOLKIT_IMAGE=%) \ 17 | $(DRIVER_VERSION:%=--build-arg DRIVER_VERSION=%) \ 18 | $(DRIVER_VERSION:%=--label driver-version=%) \ 19 | $(IMAGE_VERSION_ID:%=--build-arg IMAGE_VERSION_ID=%) \ 20 | $(EXTRA_RPM_PACKAGES:%=--build-arg EXTRA_RPM_PACKAGES=%) \ 21 | $(FROM:%=--build-arg BASEIMAGE=%) \ 22 | $(INSTRUCTLAB_IMAGE:%=--build-arg INSTRUCTLAB_IMAGE=%) \ 23 | $(KERNEL_VERSION:%=--build-arg KERNEL_VERSION=%) \ 24 | $(OS_VERSION_MAJOR:%=--build-arg OS_VERSION_MAJOR=%) \ 25 | $(SOURCE_DATE_EPOCH:%=--timestamp=%) \ 26 | $(VENDOR:%=--build-arg VENDOR=%) \ 27 | $(if $(SSH_PUBKEY),--build-arg SSHPUBKEY='$(SSH_PUBKEY)') \ 28 | --cap-add SYS_ADMIN \ 29 | --file Containerfile \ 30 | --security-opt label=disable \ 31 | --tag "${BOOTC_IMAGE}" \ 32 | -v ${OUTDIR}:/run/.input:ro \ 33 | ${CONTAINER_TOOL_EXTRA_ARGS} . 34 | -------------------------------------------------------------------------------- /training/nvidia-bootc/containers-storage.conf: -------------------------------------------------------------------------------- 1 | [storage] 2 | driver = "overlay" 3 | 4 | [storage.options] 5 | size = "" 6 | remap-uids = "" 7 | remap-gids = "" 8 | ignore_chown_errors = "" 9 | remap-user = "" 10 | remap-group = "" 11 | skip_mount_home = "" 12 | mount_program = "/usr/bin/fuse-overlayfs" 13 | mountopt = "" 14 | additionalimagestores = [ "/usr/lib/containers/storage",] 15 | 16 | [storage.options.overlay] 17 | force_mask = "shared" 18 | -------------------------------------------------------------------------------- /training/nvidia-bootc/duplicated/common/usr/lib/systemd/system/basic.target.wants/upgrade-informer.service: -------------------------------------------------------------------------------- 1 | ../upgrade-informer.service -------------------------------------------------------------------------------- /training/nvidia-bootc/duplicated/common/usr/lib/systemd/system/timers.target.wants/upgrade-informer.timer: -------------------------------------------------------------------------------- 1 | ../upgrade-informer.timer -------------------------------------------------------------------------------- /training/nvidia-bootc/duplicated/common/usr/lib/systemd/system/upgrade-informer.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Check for available RHEL AI upgrade 3 | ConditionPathExists=/run/ostree-booted 4 | After=network-online.target 5 | StartLimitIntervalSec=400 6 | StartLimitBurst=3 7 | 8 | [Service] 9 | Type=oneshot 10 | ExecStart=/usr/libexec/upgrade-informer 11 | Restart=on-failure 12 | RestartSec=90 13 | -------------------------------------------------------------------------------- /training/nvidia-bootc/duplicated/common/usr/lib/systemd/system/upgrade-informer.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Runs upgrade informer periodically 3 | ConditionPathExists=/run/ostree-booted 4 | 5 | [Timer] 6 | OnBootSec=1h 7 | OnUnitInactiveSec=1day 8 | RandomizedDelaySec=2h 9 | 10 | [Install] 11 | WantedBy=timers.target 12 | -------------------------------------------------------------------------------- /training/nvidia-bootc/duplicated/common/usr/libexec/upgrade-informer: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Run the command and capture its output 4 | output=$(bootc upgrade --check | sed -e 1q) 5 | message_file="/etc/motd.d/upgrade-message" 6 | bootc_auth="/etc/ostree/auth.json" 7 | 8 | if [[ $output == Update\ available* ]]; then 9 | if [[ ! -f $message_file ]]; then 10 | echo "New version was found" 11 | bootc_image=$(awk '{print $4}' <<< "$output") 12 | # If auth file exists we should use it 13 | auth_params="" 14 | if [[ -f $bootc_auth ]]; then 15 | auth_params="--authfile $bootc_auth" 16 | fi 17 | 18 | # Get image version 19 | # shellcheck disable=SC2086 20 | image_version_id=$(skopeo inspect --format json $auth_params "$bootc_image" | jq -r '.Labels | .["image_version_id"] // empty') 21 | 22 | # If upgrade available, write the output to the file 23 | cat > $message_file << EOF 24 | 25 | ** Attention! ** 26 | ** A new $image_version_id version is available ** 27 | ** In order to apply it run: bootc upgrade --apply 28 | ** Please note that the system will reboot after the upgrade ** 29 | 30 | EOF 31 | fi 32 | else 33 | echo "No upgrade was found" 34 | rm $message_file 2> /dev/null 35 | fi 36 | 37 | echo "Finished running upgrade informer" 38 | -------------------------------------------------------------------------------- /training/nvidia-bootc/nvidia-toolkit-firstboot.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | # For more information see https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/cdi-support.html 3 | # It looks like the podman/CDI integration wants a pre-generated list of hardware 4 | Description=Generate /etc/cdi/nvidia.yaml 5 | 6 | [Service] 7 | Type=oneshot 8 | ExecStart=nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml 9 | RemainAfterExit=yes 10 | 11 | [Install] 12 | # TODO: Ensure we have a target that is like "container setup" 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /training/nvidia-bootc/x509-configuration.ini: -------------------------------------------------------------------------------- 1 | [ req ] 2 | default_bits = 4096 3 | distinguished_name = req_distinguished_name 4 | prompt = no 5 | string_mask = utf8only 6 | x509_extensions = myexts 7 | [ req_distinguished_name ] 8 | O = Project Magma 9 | CN = Project Magma 10 | emailAddress = magma@acme.com 11 | [ myexts ] 12 | basicConstraints=critical,CA:FALSE 13 | keyUsage=digitalSignature 14 | subjectKeyIdentifier=hash 15 | authorityKeyIdentifier=keyid 16 | -------------------------------------------------------------------------------- /training/tests/ansible.cfg: -------------------------------------------------------------------------------- 1 | [ssh_connection] 2 | ssh_common_args = -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ; 3 | [defaults] 4 | host_key_checking = False -------------------------------------------------------------------------------- /training/tests/e2e-tests/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Test Environment Provisioning 3 | hosts: test_environments 4 | remote_user: root 5 | become: true 6 | gather_facts: false 7 | 8 | tasks: 9 | 10 | - name: Wait until the instance is ready 11 | ansible.builtin.wait_for_connection: 12 | delay: 15 13 | timeout: 180 14 | 15 | - name: Gather facts for first time 16 | ansible.builtin.setup: 17 | 18 | - name: ilab init 19 | shell: ilab init 20 | environment: 21 | HF_TOKEN: "{{ HF_TOKEN }}" 22 | 23 | - name: ilab pull the models for debug currently 24 | shell: ilab download --repository mistralai/Mixtral-8x7B-Instruct-v0.1 25 | environment: 26 | HF_TOKEN: "{{ HF_TOKEN }}" 27 | 28 | - name: ilab pull the models for debug currently 29 | shell: ilab download --repository ibm/granite-7b-base 30 | environment: 31 | HF_TOKEN: "{{ HF_TOKEN }}" 32 | 33 | - name: Get test script 34 | ansible.builtin.get_url: 35 | url: https://raw.githubusercontent.com/instructlab/instructlab/main/scripts/basic-workflow-tests.sh 36 | dest: /tmp/basic-workflow-tests.sh 37 | mode: 755 38 | environment: 39 | HF_TOKEN: "{{ HF_TOKEN }}" 40 | 41 | # Allow for debugging with tmate 42 | # - name: Wait for 15 minutes 43 | # pause: 44 | # minutes: 15 45 | 46 | - name: Run tests 47 | ansible.builtin.shell: /tmp/basic-workflow-tests.sh 48 | register: out 49 | 50 | - name: Test Results - stdout 51 | debug: 52 | msg: "{{out.stdout_lines}}" 53 | 54 | - name: Test Results - stderr 55 | debug: 56 | msg: "{{out.stderr_lines}}" 57 | -------------------------------------------------------------------------------- /training/tests/provision/requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | collections: 3 | - name: containers.podman 4 | version: 1.13.0 5 | -------------------------------------------------------------------------------- /training/tests/provision/templates/Containerfile.j2: -------------------------------------------------------------------------------- 1 | FROM quay.io/ai-lab/{{ image_name }}:latest 2 | ARG sshpubkey 3 | 4 | ARG VENDOR='' 5 | LABEL vendor=${VENDOR} 6 | LABEL org.opencontainers.image.vendor=${VENDOR} 7 | 8 | RUN set -eu && mkdir /usr/etc-system && \ 9 | echo 'AuthorizedKeysFile /usr/etc-system/%u.keys' > /etc/ssh/sshd_config.d/30-auth-system.conf && \ 10 | echo $sshpubkey > /usr/etc-system/root.keys && \ 11 | chmod 0600 /usr/etc-system/root.keys 12 | 13 | RUN dnf install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm 14 | -------------------------------------------------------------------------------- /training/vllm/Containerfile: -------------------------------------------------------------------------------- 1 | FROM quay.io/wxpe/tgis-vllm:release.4e3ff78 2 | 3 | ARG VENDOR='' 4 | LABEL vendor=${VENDOR} 5 | LABEL org.opencontainers.image.vendor=${VENDOR} 6 | 7 | USER root 8 | RUN ln -s /usr/lib64/libcuda.so.1 /usr/lib64/libcuda.so 9 | COPY mixtral.jinja . 10 | -------------------------------------------------------------------------------- /training/vllm/Makefile: -------------------------------------------------------------------------------- 1 | CONTAINER_TOOL ?= podman 2 | SOURCE_DATE_EPOCH ?= $(shell git log -1 --pretty=%ct) 3 | BUILD_ARG_FILE ?= 4 | 5 | default: image 6 | 7 | .PHONY: image 8 | image: 9 | @mkdir -p ../build 10 | rm -rf ../build/vllm 11 | "${CONTAINER_TOOL}" build \ 12 | $(ARCH:%=--platform linux/%) \ 13 | $(BUILD_ARG_FILE:%=--build-arg-file=%) \ 14 | $(SOURCE_DATE_EPOCH:%=--timestamp=%) \ 15 | $(VENDOR:%=--build-arg VENDOR=%) \ 16 | --file Containerfile \ 17 | --squash-all \ 18 | --tag oci:../build/vllm \ 19 | ${CONTAINER_TOOL_EXTRA_ARGS} . 20 | -------------------------------------------------------------------------------- /training/vllm/mixtral.jinja: -------------------------------------------------------------------------------- 1 | {% set bos_token = "" %} 2 | 3 | {% set eos_token = "" %} 4 | 5 | {{ bos_token }} 6 | {% for message in messages %} 7 | {% if message['role'] == 'user' %} 8 | {{ '[INST] ' + message['content'] + ' [/INST]' }} 9 | {% elif message['role'] == 'assistant' %} 10 | {{ message['content'] + eos_token}} 11 | {% endif %} 12 | {% endfor %} 13 | -------------------------------------------------------------------------------- /vector_dbs/README.md: -------------------------------------------------------------------------------- 1 | # Directory to store vector_dbs files 2 | This directory has make files and container files for open source vector databases. The built container images are used by recipes like `rag` to provide required database functions. 3 | 4 | ## Chroma 5 | [Chroma](https://www.trychroma.com/) is an AI-native open-source embedding database. 6 | Chroma makes it easy to build LLM apps by making knowledge, facts, and skills 7 | pluggable for LLMs. 8 | 9 | ## Milvus 10 | [Milvus](https://milvus.io/) is an open-source vector database built to power embedding similarity search and AI applications. It is highly scalable and offers many production ready features for search. 11 | -------------------------------------------------------------------------------- /vector_dbs/chromadb/Containerfile: -------------------------------------------------------------------------------- 1 | # DO NOT UPVERSION UNTIL APP HAS BEEN UPDATED AS WELL 2 | # AS IT IS THE LAST VERSION THAT WORKS WITH THE RAG RECIPE 3 | FROM docker.io/chromadb/chroma:0.5.23 4 | -------------------------------------------------------------------------------- /vector_dbs/chromadb/Makefile: -------------------------------------------------------------------------------- 1 | CONTAINER_TOOL ?= podman 2 | APP ?= chromadb 3 | APPIMAGE ?= quay.io/ai-lab/${APP}:latest 4 | 5 | .PHONY: build 6 | build: 7 | "${CONTAINER_TOOL}" build -f Containerfile -t ${APPIMAGE} . 8 | -------------------------------------------------------------------------------- /vector_dbs/milvus/Containerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/milvusdb/milvus:master-20240426-bed6363f 2 | ADD embedEtcd.yaml /milvus/configs/embedEtcd.yaml 3 | -------------------------------------------------------------------------------- /vector_dbs/milvus/Makefile: -------------------------------------------------------------------------------- 1 | CONTAINER_TOOL ?= podman 2 | REGISTRY ?= quay.io 3 | REGISTRY_ORG ?= ai-lab 4 | COMPONENT = vector_dbs 5 | 6 | IMAGE ?= $(REGISTRY)/$(REGISTRY_ORG)/$(COMPONENT)/milvus:latest 7 | 8 | ARCH ?= $(shell uname -m) 9 | PLATFORM ?= linux/$(ARCH) 10 | 11 | gRCP_PORT := 19530 12 | REST_PORT := 9091 13 | CLIENT_PORT := 2379 14 | 15 | LIB_MILVUS_DIR_MOUNTPATH := $(shell pwd)/volumes/milvus 16 | 17 | .PHONY: build 18 | build: 19 | "${CONTAINER_TOOL}" build --platform $(PLATFORM) -f Containerfile -t ${IMAGE} . 20 | 21 | .PHONY: run 22 | run: 23 | podman run -it \ 24 | --name milvus-standalone \ 25 | --security-opt seccomp:unconfined \ 26 | -e ETCD_USE_EMBED=true \ 27 | -e ETCD_CONFIG_PATH=/milvus/configs/embedEtcd.yaml \ 28 | -e COMMON_STORAGETYPE=local \ 29 | -v $(LIB_MILVUS_DIR_MOUNTPATH):/var/lib/milvus \ 30 | -p $(gRCP_PORT):$(gRCP_PORT) \ 31 | -p $(REST_PORT):$(REST_PORT) \ 32 | -p $(CLIENT_PORT):$(CLIENT_PORT) \ 33 | --health-cmd="curl -f http://localhost:$(REST_PORT)/healthz" \ 34 | --health-interval=30s \ 35 | --health-start-period=90s \ 36 | --health-timeout=20s \ 37 | --health-retries=3 \ 38 | $(IMAGE) \ 39 | milvus run standalone 1> /dev/null 40 | 41 | .PHONY: stop 42 | stop: 43 | -podman stop milvus-standalone 44 | 45 | .PHONY: delete 46 | delete: 47 | -podman rm milvus-standalone -f 48 | 49 | .PHONY: podman-clean 50 | podman-clean: 51 | @container_ids=$$(podman ps --format "{{.ID}} {{.Image}}" | awk '$$2 == "$(IMAGE)" {print $$1}'); \ 52 | echo "removing all containers with IMAGE=$(IMAGE)"; \ 53 | for id in $$container_ids; do \ 54 | echo "Removing container: $$id,"; \ 55 | podman rm -f $$id; \ 56 | done 57 | -------------------------------------------------------------------------------- /vector_dbs/milvus/embedEtcd.yaml: -------------------------------------------------------------------------------- 1 | listen-client-urls: http://0.0.0.0:2379 2 | advertise-client-urls: http://0.0.0.0:2379 3 | quota-backend-bytes: 4294967296 4 | auto-compaction-mode: revision 5 | auto-compaction-retention: '1000' 6 | -------------------------------------------------------------------------------- /vector_dbs/milvus/volumes/milvus/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/containers/ai-lab-recipes/1bace0e7c5b8aa50d723e80315eeb6a8cdee86c0/vector_dbs/milvus/volumes/milvus/.gitkeep --------------------------------------------------------------------------------