├── app ├── start-dev.sh ├── .env.test ├── backend │ ├── public │ │ └── .folder_keep │ ├── startdev.sh │ ├── .env.example │ ├── Pipfile │ ├── default_prompt.txt │ ├── helpers.py │ ├── classes.py │ └── milvus_retriever_with_score_threshold.py ├── .env.development ├── frontend │ ├── .prettierignore │ ├── __mocks__ │ │ ├── styleMock.js │ │ └── fileMock.js │ ├── startdev.sh │ ├── .env.example │ ├── .prettierrc │ ├── .gitignore │ ├── src │ │ ├── app │ │ │ ├── config.tsx │ │ │ ├── utils │ │ │ │ └── useDocumentTitle.ts │ │ │ ├── index.tsx │ │ │ ├── components │ │ │ │ ├── UserContext │ │ │ │ │ └── UserContext.tsx │ │ │ │ ├── NotFound │ │ │ │ │ └── NotFound.tsx │ │ │ │ └── Chat │ │ │ │ │ └── classes.tsx │ │ │ ├── assets │ │ │ │ └── bgimages │ │ │ │ │ ├── github-mark-white.svg │ │ │ │ │ ├── github-mark.svg │ │ │ │ │ ├── star.svg │ │ │ │ │ ├── star-white.svg │ │ │ │ │ ├── default-user.svg │ │ │ │ │ └── orb.svg │ │ │ ├── routes.tsx │ │ │ └── app.test.tsx │ │ ├── typings.d.ts │ │ ├── index.html │ │ ├── i18n │ │ │ └── config.ts │ │ └── index.tsx │ ├── .editorconfig │ ├── dr-surge.js │ ├── scripts │ │ └── dotenv.js │ ├── webpack.dev.js │ ├── stylePaths.js │ ├── tsconfig.json │ ├── babel.config.js │ ├── LICENSE │ ├── webpack.prod.js │ ├── jest.config.js │ ├── public │ │ └── locales │ │ │ ├── cn │ │ │ └── translation.json │ │ │ ├── jp │ │ │ └── translation.json │ │ │ ├── en │ │ │ └── translation.json │ │ │ ├── de │ │ │ └── translation.json │ │ │ ├── es │ │ │ └── translation.json │ │ │ └── fr │ │ │ └── translation.json │ └── .eslintrc.js ├── .env.production ├── .dockerignore ├── Containerfile ├── deployment │ ├── service.yaml │ ├── route.yaml │ └── deployment.yaml ├── README.md └── package.json ├── img └── ui.png ├── products-documentation-ingestion ├── kfp_config.ini ├── requirements.txt ├── Containerfile ├── Pipfile ├── component_metadata │ └── doc_ingest.yaml ├── classes.py ├── doc_processing_docling_server.py ├── requirements_frozen.txt ├── md_splitter.py ├── doc_ingestion_pipeline.py ├── collections_loader.py └── README.md ├── tekton ├── workspace.yaml ├── trigger-rollout.yaml ├── rbac.yaml ├── doc-process-status-task.yaml ├── execute-doc-ingestion-task.yaml ├── README.md ├── trigger.yaml └── pipeline.yaml ├── rh-doc-splits-generation ├── Pipfile ├── requirements.txt └── rh-doc-splits-generation.ipynb ├── collections ├── edge_management.json ├── red_hat_insights.json ├── red_hat_device_edge.json ├── red_hat_customer_portal.json ├── red_hat_developer_tools.json ├── hybrid_committed_spend.json ├── red_hat_security_data_api.json ├── red_hat_openshift_lightspeed.json ├── red_hat_hardware_certification.json ├── red_hat_hybrid_cloud_console.json ├── red_hat_software_certification.json ├── migration_toolkit_for_containers.json ├── red_hat_application_foundations.json ├── red_hat_openshift_api_management.json ├── red_hat_openshift_service_on_aws.json ├── red_hat_openshift_ai_cloud_service.json ├── red_hat_product_life_cycle_data_api.json ├── red_hat_openshift_virtualization_engine.json ├── red_hat_openstack_services_on_openshift.json ├── red_hat_certified_cloud_and_service_provider_certification.json ├── red_hat_ansible_lightspeed_with_ibm_watsonx_code_assistant.json ├── red_hat_build_of_cryostat.json ├── red_hat_directory_server.json ├── red_hat_certificate_system.json ├── red_hat_enterprise_linux_for_sap_solutions.json ├── red_hat_advanced_developer_suite.json ├── open_liberty.json ├── red_hat_ansible_inside.json ├── red_hat_build_of_keycloak.json ├── red_hat_software_collections.json ├── red_hat_service_interconnect.json ├── red_hat_build_of_apache_camel.json ├── red_hat_build_of_optaplanner.json ├── red_hat_connectivity_link.json ├── migration_toolkit_for_runtimes.json ├── red_hat_jboss_data_virtualization.json ├── red_hat_openshift_virtualization.json ├── red_hat_enterprise_linux_for_real_time.json ├── red_hat_openshift_service_mesh.json ├── red_hat_amq.json ├── red_hat_build_of_openjdk.json ├── red_hat_developer_toolset.json ├── red_hat_amq_broker.json ├── red_hat_enterprise_linux_ai.json ├── builds_for_red_hat_openshift.json ├── red_hat_build_of_node.js.json ├── red_hat_virtualization.json ├── red_hat_update_infrastructure.json ├── red_hat_jboss_core_services.json ├── red_hat_ansible_automation_platform.json ├── red_hat_trusted_application_pipeline.json ├── red_hat_trusted_artifact_signer.json ├── red_hat_gluster_storage.json ├── red_hat_build_of_debezium.json ├── red_hat_build_of_eclipse_vert.x.json ├── red_hat_support_for_spring_boot.json ├── red_hat_build_of_apicurio_registry.json ├── red_hat_hyperconverged_infrastructure_for_virtualization.json ├── red_hat_single_sign-on.json ├── red_hat_build_of_microshift.json ├── migration_toolkit_for_virtualization.json ├── red_hat_enterprise_linux.json ├── red_hat_trusted_profile_analyzer.json ├── red_hat_jboss_enterprise_application_platform.json ├── red_hat_ceph_storage.json ├── workload_availability_for_red_hat_openshift.json ├── red_hat_build_of_quarkus.json ├── red_hat_openstack_platform.json ├── red_hat_developer_hub.json ├── red_hat_openshift_data_foundation.json ├── red_hat_openshift_serverless.json ├── streams_for_apache_kafka.json ├── red_hat_openshift_gitops.json ├── red_hat_openshift_pipelines.json ├── red_hat_openshift_ai_self-managed.json ├── red_hat_advanced_cluster_security_for_kubernetes.json ├── red_hat_advanced_cluster_management_for_kubernetes.json ├── red_hat_quay.json ├── red_hat_jboss_web_server.json └── red_hat_decision_manager.json ├── .gitignore └── LICENSE /app/start-dev.sh: -------------------------------------------------------------------------------- 1 | npm run dev -------------------------------------------------------------------------------- /app/.env.test: -------------------------------------------------------------------------------- 1 | APP_ENV=test 2 | -------------------------------------------------------------------------------- /app/backend/public/.folder_keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/.env.development: -------------------------------------------------------------------------------- 1 | APP_ENV=development 2 | -------------------------------------------------------------------------------- /app/frontend/.prettierignore: -------------------------------------------------------------------------------- 1 | package.json 2 | -------------------------------------------------------------------------------- /app/frontend/__mocks__/styleMock.js: -------------------------------------------------------------------------------- 1 | module.exports = {}; 2 | -------------------------------------------------------------------------------- /app/frontend/startdev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | npm run start:dev 3 | -------------------------------------------------------------------------------- /app/frontend/.env.example: -------------------------------------------------------------------------------- 1 | BACKEND_API_URL=http://127.0.0.1:5000/api -------------------------------------------------------------------------------- /app/.env.production: -------------------------------------------------------------------------------- 1 | APP_ENV=production 2 | FASTIFY_LOG_LEVEL=warn 3 | -------------------------------------------------------------------------------- /app/frontend/__mocks__/fileMock.js: -------------------------------------------------------------------------------- 1 | module.exports = 'test-file-stub'; 2 | -------------------------------------------------------------------------------- /img/ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rh-aiservices-bu/rh-kb-chat/HEAD/img/ui.png -------------------------------------------------------------------------------- /app/frontend/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "printWidth": 120 4 | } 5 | -------------------------------------------------------------------------------- /products-documentation-ingestion/kfp_config.ini: -------------------------------------------------------------------------------- 1 | [Components] 2 | doc_ingest = documentation_ingestion.py -------------------------------------------------------------------------------- /app/frontend/.gitignore: -------------------------------------------------------------------------------- 1 | **/node_modules 2 | dist 3 | yarn-error.log 4 | yarn.lock 5 | stats.json 6 | coverage 7 | storybook-static 8 | .DS_Store 9 | .idea 10 | .env 11 | -------------------------------------------------------------------------------- /app/.dockerignore: -------------------------------------------------------------------------------- 1 | **/node_modules 2 | **/.env.* 3 | **/.env 4 | **/*.local 5 | .docs 6 | **/.vscode 7 | .gitignore 8 | **/.venv 9 | **/__pycache__ 10 | .dockerignore 11 | **/dist 12 | -------------------------------------------------------------------------------- /app/frontend/src/app/config.tsx: -------------------------------------------------------------------------------- 1 | const config = { 2 | backend_api_url: process.env.BACKEND_API_URL || window.location.protocol + '//' + window.location.hostname + '/api', 3 | }; 4 | 5 | export default config; -------------------------------------------------------------------------------- /app/backend/startdev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source /home/gmoutier/Dev/repos/rh-aiservices-bu/rh-kb-chat/app/backend/.venv/bin/activate 3 | watchmedo auto-restart --pattern "*.py" --recursive --signal SIGTERM python app.py -------------------------------------------------------------------------------- /app/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.access.redhat.com/ubi9/nodejs-18 2 | 3 | USER root 4 | 5 | RUN yum -y update && \ 6 | yum clean all 7 | 8 | COPY --chown=1001:0 . /tmp/src 9 | 10 | USER 1001 11 | 12 | RUN /usr/libexec/s2i/assemble 13 | 14 | CMD /usr/libexec/s2i/run 15 | -------------------------------------------------------------------------------- /products-documentation-ingestion/requirements.txt: -------------------------------------------------------------------------------- 1 | beautifulsoup4==4.13.3 2 | einops==0.8.1 3 | html2text==2024.2.26 4 | langchain==0.3.19 5 | langchain-community==0.3.17 6 | langchain-milvus==0.1.7 7 | lxml==5.3.1 8 | openai==1.63.2 9 | pymilvus==2.4.10 10 | pypdf==5.3.0 11 | python-dotenv==1.0.1 12 | tqdm==4.67.1 13 | kfp==2.11.0 -------------------------------------------------------------------------------- /app/deployment/service.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: rh-kb-chatbot 6 | labels: 7 | app: rh-kb-chatbot 8 | spec: 9 | ports: 10 | - name: http 11 | protocol: TCP 12 | port: 5000 13 | targetPort: 5000 14 | selector: 15 | app: rh-kb-chatbot 16 | sessionAffinity: None 17 | type: ClusterIP -------------------------------------------------------------------------------- /tekton/workspace.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: PersistentVolumeClaim 4 | metadata: 5 | name: shared-workspace 6 | annotations: 7 | volume.beta.kubernetes.io/storage-provisioner: kubernetes.io/aws-ebs 8 | spec: 9 | resources: 10 | requests: 11 | storage: 3Gi 12 | volumeMode: Filesystem 13 | accessModes: 14 | - ReadWriteOnce 15 | -------------------------------------------------------------------------------- /app/frontend/src/typings.d.ts: -------------------------------------------------------------------------------- 1 | declare module '*.png'; 2 | declare module '*.jpg'; 3 | declare module '*.jpeg'; 4 | declare module '*.gif'; 5 | declare module '*.svg'; 6 | declare module '*.css'; 7 | declare module '*.wav'; 8 | declare module '*.mp3'; 9 | declare module '*.m4a'; 10 | declare module '*.rdf'; 11 | declare module '*.ttl'; 12 | declare module '*.pdf'; 13 | -------------------------------------------------------------------------------- /products-documentation-ingestion/Containerfile: -------------------------------------------------------------------------------- 1 | FROM registry.redhat.io/ubi9/python-311@sha256:fc669a67a0ef9016c3376b2851050580b3519affd5ec645d629fd52d2a8b8e4a 2 | 3 | USER 0 4 | 5 | RUN dnf -y update && \ 6 | dnf -y clean all && \ 7 | rm -rf /var/cache/dnf 8 | 9 | USER 1001 10 | 11 | COPY . . 12 | 13 | RUN pip install --no-cache-dir -r requirements_frozen.txt 14 | -------------------------------------------------------------------------------- /app/deployment/route.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: Route 3 | apiVersion: route.openshift.io/v1 4 | metadata: 5 | name: rh-kb-chatbot 6 | labels: 7 | app: rh-kb-chatbot 8 | spec: 9 | to: 10 | kind: Service 11 | name: rh-kb-chatbot 12 | weight: 100 13 | port: 14 | targetPort: http 15 | tls: 16 | termination: edge 17 | insecureEdgeTerminationPolicy: Redirect 18 | wildcardPolicy: None -------------------------------------------------------------------------------- /app/frontend/.editorconfig: -------------------------------------------------------------------------------- 1 | # Editor configuration, see http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | indent_style = space 7 | indent_size = 2 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.snap] 12 | max_line_length = off 13 | trim_trailing_whitespace = false 14 | 15 | [*.md] 16 | max_line_length = off 17 | trim_trailing_whitespace = false 18 | -------------------------------------------------------------------------------- /rh-doc-splits-generation/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | langchain = "==0.1.12" 8 | beautifulsoup4 = "==4.12.2" 9 | html2text = "==2024.2.26" 10 | lxml = "==5.1.0" 11 | tqdm = "==4.66.2" 12 | 13 | [dev-packages] 14 | ipykernel = "*" 15 | 16 | [requires] 17 | python_version = "3.11" 18 | python_full_version = "3.11.7" 19 | -------------------------------------------------------------------------------- /app/frontend/dr-surge.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | const indexPath = path.resolve(__dirname, 'dist/index.html'); 4 | const targetFilePath = path.resolve(__dirname, 'dist/200.html'); 5 | // ensure we have bookmarkable url's when publishing to surge 6 | // https://surge.sh/help/adding-a-200-page-for-client-side-routing 7 | fs.createReadStream(indexPath).pipe(fs.createWriteStream(targetFilePath)); 8 | -------------------------------------------------------------------------------- /collections/edge_management.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "edge_management", 3 | "collection_full_name": "Edge management", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1-latest", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_insights.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_insights", 3 | "collection_full_name": "Red Hat Insights", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1-latest", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_device_edge.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_device_edge", 3 | "collection_full_name": "Red Hat Device Edge", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "4", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_customer_portal.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_customer_portal", 3 | "collection_full_name": "Red Hat Customer Portal", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_developer_tools.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_developer_tools", 3 | "collection_full_name": "Red Hat Developer Tools", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/hybrid_committed_spend.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "hybrid_committed_spend", 3 | "collection_full_name": "Hybrid committed spend", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1-latest", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_security_data_api.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_security_data_api", 3 | "collection_full_name": "Red Hat Security Data API", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.0", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_openshift_lightspeed.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openshift_lightspeed", 3 | "collection_full_name": "Red Hat OpenShift Lightspeed", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.0tp1", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_hardware_certification.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_hardware_certification", 3 | "collection_full_name": "Red Hat Hardware Certification", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "2025", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_hybrid_cloud_console.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_hybrid_cloud_console", 3 | "collection_full_name": "Red Hat Hybrid Cloud Console", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1-latest", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_software_certification.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_software_certification", 3 | "collection_full_name": "Red Hat Software Certification", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "2024", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/migration_toolkit_for_containers.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "migration_toolkit_for_containers", 3 | "collection_full_name": "Migration Toolkit for Containers", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.8", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_application_foundations.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_application_foundations", 3 | "collection_full_name": "Red Hat Application Foundations", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "2024", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_openshift_api_management.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openshift_api_management", 3 | "collection_full_name": "Red Hat OpenShift API Management", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_openshift_service_on_aws.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openshift_service_on_aws", 3 | "collection_full_name": "Red Hat OpenShift Service on AWS", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "4", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_openshift_ai_cloud_service.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openshift_ai_cloud_service", 3 | "collection_full_name": "Red Hat OpenShift AI Cloud Service", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /app/frontend/src/app/utils/useDocumentTitle.ts: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { useTranslation } from 'react-i18next'; 3 | 4 | // a custom hook for setting the page title 5 | export function useDocumentTitle(title: string) { 6 | const { t, i18n } = useTranslation(); 7 | React.useEffect(() => { 8 | const originalTitle = document.title; 9 | document.title = t(title); 10 | 11 | return () => { 12 | document.title = originalTitle; 13 | }; 14 | }, [title]); 15 | } 16 | -------------------------------------------------------------------------------- /collections/red_hat_product_life_cycle_data_api.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_product_life_cycle_data_api", 3 | "collection_full_name": "Red Hat Product Life Cycle Data API", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.0", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_openshift_virtualization_engine.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openshift_virtualization_engine", 3 | "collection_full_name": "Red Hat OpenShift Virtualization Engine", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "4", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /collections/red_hat_openstack_services_on_openshift.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openstack_services_on_openshift", 3 | "collection_full_name": "Red Hat OpenStack Services on OpenShift", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "18.0", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /tekton/trigger-rollout.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: tekton.dev/v1 3 | kind: Task 4 | metadata: 5 | name: trigger-rollout-chatbot 6 | spec: 7 | steps: 8 | - name: trigger-rollout-chatbot 9 | image: quay.io/openshift/origin-cli:4.14 10 | command: ["/bin/sh", "-c"] 11 | script: | 12 | #!/bin/sh 13 | 14 | # Start rollout for rh-kb-chatbot 15 | oc rollout restart deployment/rh-kb-chatbot -n rh-kb-chatbot 16 | 17 | # Check the status of rollout 18 | oc rollout status deployment/rh-kb-chatbot -n rh-kb-chatbot -------------------------------------------------------------------------------- /collections/red_hat_certified_cloud_and_service_provider_certification.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_certified_cloud_and_service_provider_certification", 3 | "collection_full_name": "Red Hat Certified Cloud and Service Provider Certification", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "2025", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /app/backend/.env.example: -------------------------------------------------------------------------------- 1 | # LLM 2 | INFERENCE_SERVER_URL=https://my-server-url/v1 3 | API_KEY=None 4 | MODEL_NAME=mistralai/Mistral-7B-Instruct-v0.2 5 | EMBEDDINGS_MODEL_NAME=nomic-ai/nomic-embed-text-v1 6 | MAX_TOKENS=512 7 | TOP_P=0.95 8 | TEMPERATURE=0.01 9 | PRESENCE_PENALTY=1.03 10 | PROMPT_FILE=default_prompt.txt 11 | # MILVUS 12 | MILVUS_HOST=127.0.0.1 13 | MILVUS_PORT=19530 14 | MILVUS_USERNAME=collection_reader 15 | MILVUS_PASSWORD=********* 16 | MAX_RETRIEVED_DOCS=4 17 | SCORE_THRESHOLD=0.90 18 | MILVUS_COLLECTIONS_FILE=collections.json 19 | MILVUS_COLLECTION=none 20 | -------------------------------------------------------------------------------- /collections/red_hat_ansible_lightspeed_with_ibm_watsonx_code_assistant.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_ansible_lightspeed_with_ibm_watsonx_code_assistant", 3 | "collection_full_name": "Red Hat Ansible Lightspeed with IBM watsonx Code Assistant", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "2.x_latest", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /app/frontend/src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Chat with the documentation 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /products-documentation-ingestion/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | beautifulsoup4 = "==4.13.3" 8 | einops = "==0.8.1" 9 | html2text = "==2024.2.26" 10 | langchain = "==0.3.19" 11 | langchain-community = "==0.3.17" 12 | langchain-milvus = "==0.1.7" 13 | lxml = "==5.3.1" 14 | openai = "==1.63.2" 15 | pymilvus = "==2.4.10" 16 | pypdf = "==5.3.0" 17 | python-dotenv = "==1.0.1" 18 | tqdm = "==4.67.1" 19 | 20 | [dev-packages] 21 | ipykernel = "==6.29.5" 22 | ipywidgets = "==8.1.5" 23 | 24 | [requires] 25 | python_version = "3.11" 26 | python_full_version = "3.11.7" -------------------------------------------------------------------------------- /app/frontend/src/i18n/config.ts: -------------------------------------------------------------------------------- 1 | import { esbuildMinify } from "css-minimizer-webpack-plugin"; 2 | import i18n from "i18next"; 3 | import HttpApi from "i18next-http-backend"; 4 | import { initReactI18next } from "react-i18next"; 5 | 6 | export const supportedLngs = { 7 | en: "English", 8 | fr: "Français", 9 | de: "Deutsch", 10 | es: "Español", 11 | cn: "中文", 12 | jp: "日本語", 13 | }; 14 | 15 | i18n 16 | .use(HttpApi) 17 | .use(initReactI18next) 18 | .init({ 19 | lng: "en", 20 | fallbackLng: "en", 21 | supportedLngs: Object.keys(supportedLngs), 22 | interpolation: { 23 | escapeValue: false, 24 | }, 25 | }); 26 | 27 | export default i18n; -------------------------------------------------------------------------------- /app/frontend/src/app/index.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import '@patternfly/react-core/dist/styles/base.css'; 3 | import { BrowserRouter as Router } from 'react-router-dom'; 4 | import { AppLayout } from '@app/components/AppLayout/AppLayout'; 5 | import { AppRoutes } from '@app/routes'; 6 | import '@app/app.css'; 7 | import '@patternfly/chatbot/dist/css/main.css'; 8 | import { UserProvider } from '@app/components/UserContext/UserContext'; 9 | 10 | const App: React.FunctionComponent = () => ( 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | ); 19 | 20 | export default App; 21 | -------------------------------------------------------------------------------- /app/README.md: -------------------------------------------------------------------------------- 1 | # Application image 2 | 3 | ## Building 4 | 5 | The npm build happens during the image build. To do it successfully, you may have to augment the limits on open files in your system. Ex: 6 | 7 | `podman build --no-cache --ulimit nofile=10000:10000 -t rh-kb-chat:1.1.0 .` 8 | 9 | ## Deployment 10 | 11 | - Create a secret from `backend/config.json` file: 12 | 13 | ```bash 14 | oc create secret generic kb-chatbot --from-file=backend/config.json 15 | ``` 16 | 17 | - Create the Deployment, Service and Route from their respective files in the `deployment` folder: 18 | 19 | ```bash 20 | oc create -f deployment.yaml 21 | oc create -f service.yaml 22 | oc create -f route.yaml 23 | ``` 24 | -------------------------------------------------------------------------------- /app/frontend/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import ReactDOM from "react-dom/client"; 3 | import App from '@app/index'; 4 | import './i18n/config'; 5 | 6 | if (process.env.NODE_ENV !== "production") { 7 | const config = { 8 | rules: [ 9 | { 10 | id: 'color-contrast', 11 | enabled: false 12 | } 13 | ] 14 | }; 15 | // eslint-disable-next-line @typescript-eslint/no-var-requires, no-undef 16 | const axe = require("react-axe"); 17 | axe(React, ReactDOM, 1000, config); 18 | } 19 | 20 | const root = ReactDOM.createRoot(document.getElementById("root") as Element); 21 | 22 | root.render( 23 | 24 | 25 | 26 | ) 27 | -------------------------------------------------------------------------------- /collections/red_hat_build_of_cryostat.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_build_of_cryostat", 3 | "collection_full_name": "Red Hat build of Cryostat", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "3", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "2", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | } 26 | ] 27 | } -------------------------------------------------------------------------------- /collections/red_hat_directory_server.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_directory_server", 3 | "collection_full_name": "Red Hat Directory Server", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "12", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "11", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | } 26 | ] 27 | } -------------------------------------------------------------------------------- /collections/red_hat_certificate_system.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_certificate_system", 3 | "collection_full_name": "Red Hat Certificate System", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "10", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "9", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | } 26 | ] 27 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # editor 2 | .idea/ 3 | .vscode/ 4 | 5 | # misc 6 | .DS_Store 7 | .gitkeep 8 | 9 | # env vars 10 | .env 11 | *.local 12 | local_collections 13 | 14 | # dependencies 15 | node_modules 16 | logs 17 | 18 | # build 19 | www 20 | www-cache 21 | app/backend/public/* 22 | !app/backend/public/.folder_keep 23 | 24 | # cache 25 | __pycache__ 26 | .cache 27 | .eslintcache 28 | **/.cache_* 29 | 30 | # helpers 31 | .docs 32 | 33 | # logs 34 | npm-debug.log* 35 | yarn-debug.log* 36 | yarn-error.log* 37 | 38 | # locks 39 | yarn.lock 40 | package-lock.json 41 | 42 | # local python env 43 | .venv 44 | 45 | # notebook checkpoints 46 | */*/.ipynb_checkpoints/* 47 | */*/*/.ipynb_checkpoints/* 48 | 49 | # config 50 | config.json 51 | -------------------------------------------------------------------------------- /tekton/rbac.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: rollout-controller 6 | namespace: rh-kb-chatbot 7 | rules: 8 | - apiGroups: ["apps"] 9 | resources: ["deployments"] 10 | verbs: ["get", "update"] 11 | - apiGroups: [""] 12 | resources: ["pods"] 13 | verbs: ["get", "list", "watch"] 14 | --- 15 | apiVersion: rbac.authorization.k8s.io/v1 16 | kind: RoleBinding 17 | metadata: 18 | name: rollout-controller-binding 19 | namespace: rh-kb-chatbot 20 | subjects: 21 | - kind: ServiceAccount 22 | name: pipeline 23 | namespace: rh-kb-chatbot-doc-ingestion 24 | roleRef: 25 | kind: Role 26 | name: rollout-controller 27 | apiGroup: rbac.authorization.k8s.io 28 | -------------------------------------------------------------------------------- /collections/red_hat_enterprise_linux_for_sap_solutions.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_enterprise_linux_for_sap_solutions", 3 | "collection_full_name": "Red Hat Enterprise Linux for SAP Solutions", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "9", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "8", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | } 26 | ] 27 | } -------------------------------------------------------------------------------- /collections/red_hat_advanced_developer_suite.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_advanced_developer_suite_-_software_supply_chain", 3 | "collection_full_name": "Red Hat Advanced Developer Suite", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.7", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.6", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /app/backend/Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [[source]] 7 | url = "https://download.pytorch.org/whl/cpu" 8 | verify_ssl = false 9 | name = "pytorch" 10 | 11 | [packages] 12 | einops = "~=0.8.1" 13 | fastapi = "~=0.115.8" 14 | httpx = "~=0.28.1" 15 | langchain = "==0.3.19" 16 | langchain-community = "==0.3.17" 17 | langchain-milvus = "==0.1.7" 18 | langchain-openai = "==0.3.6" 19 | openai = "==1.63.2" 20 | psycopg2-binary = "~=2.9.9" 21 | pymilvus = "~=2.4.10" 22 | python-dotenv = "~=1.0.1" 23 | python-multipart = "~=0.0.20" 24 | torch = {version = "==2.5.1+cpu", index = "pytorch"} 25 | uvicorn = "~=0.34.0" 26 | websockets = "~=15.0" 27 | 28 | [dev-packages] 29 | "watchdog[watchmedo]" = "~=5.0.3" 30 | black = "*" 31 | 32 | [requires] 33 | python_version = "3.11" 34 | -------------------------------------------------------------------------------- /products-documentation-ingestion/component_metadata/doc_ingest.yaml: -------------------------------------------------------------------------------- 1 | # PIPELINE DEFINITION 2 | # Name: doc-ingest 3 | components: 4 | comp-doc-ingest: 5 | executorLabel: exec-doc-ingest 6 | deploymentSpec: 7 | executors: 8 | exec-doc-ingest: 9 | container: 10 | args: 11 | - --executor_input 12 | - '{{$}}' 13 | - --function_to_execute 14 | - doc_ingest 15 | command: 16 | - python3 17 | - -m 18 | - kfp.dsl.executor_main 19 | image: quay.io/rh-aiservices-bu/rh-kb-doc-ingestion:1.0 20 | pipelineInfo: 21 | name: doc-ingest 22 | root: 23 | dag: 24 | tasks: 25 | doc-ingest: 26 | cachingOptions: 27 | enableCache: true 28 | componentRef: 29 | name: comp-doc-ingest 30 | taskInfo: 31 | name: doc-ingest 32 | schemaVersion: 2.1.0 33 | sdkVersion: kfp-2.11.0 34 | -------------------------------------------------------------------------------- /app/frontend/src/app/components/UserContext/UserContext.tsx: -------------------------------------------------------------------------------- 1 | import React, { createContext, useContext, useState } from 'react'; 2 | 3 | interface UserContextType { 4 | userName: string; 5 | setUserName: (name: string) => void; 6 | } 7 | 8 | const UserContext = createContext(undefined); 9 | 10 | export const UserProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => { 11 | const [userName, setUserName] = useState('DefaultUser'); // Replace with logic to fetch the username 12 | 13 | return ( 14 | 15 | {children} 16 | 17 | ); 18 | }; 19 | 20 | export const useUser = (): UserContextType => { 21 | const context = useContext(UserContext); 22 | if (!context) { 23 | throw new Error('useUser must be used within a UserProvider'); 24 | } 25 | return context; 26 | }; -------------------------------------------------------------------------------- /collections/open_liberty.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "open_liberty", 3 | "collection_full_name": "Open Liberty", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "24.0.0.12", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "24.0.0.9", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "24.0.0.6", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | } 36 | ] 37 | } -------------------------------------------------------------------------------- /app/frontend/scripts/dotenv.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const dotenv = require('dotenv'); 3 | 4 | /** 5 | * Setup, and access, a dotenv file and the related set of parameters. 6 | * 7 | * @param {string} path 8 | * @returns {*} 9 | */ 10 | const setupDotenvFile = path => dotenv.config({ path }); 11 | 12 | /** 13 | * Setup and access local and specific dotenv file parameters. 14 | * 15 | * @param {string} env 16 | */ 17 | const setupDotenvFilesForEnv = ({ env }) => { 18 | const RELATIVE_DIRNAME = path.resolve(__dirname, '..'); 19 | 20 | if (env) { 21 | setupDotenvFile(path.resolve(RELATIVE_DIRNAME, `.env.${env}.local`)); 22 | setupDotenvFile(path.resolve(RELATIVE_DIRNAME, `.env.${env}`)); 23 | } 24 | 25 | setupDotenvFile(path.resolve(RELATIVE_DIRNAME, '.env.local')); 26 | setupDotenvFile(path.resolve(RELATIVE_DIRNAME, '.env')); 27 | }; 28 | 29 | setupDotenvFilesForEnv({ env: process.env && process.env.NODE_ENV }); 30 | -------------------------------------------------------------------------------- /collections/red_hat_ansible_inside.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_ansible_inside", 3 | "collection_full_name": "Red Hat Ansible Inside", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.3", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.2", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.1", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | } 36 | ] 37 | } -------------------------------------------------------------------------------- /app/frontend/webpack.dev.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-var-requires */ 2 | 3 | const path = require('path'); 4 | const { merge } = require('webpack-merge'); 5 | const common = require('./webpack.common.js'); 6 | const { stylePaths } = require('./stylePaths.js'); 7 | const HOST = process.env.HOST || 'localhost'; 8 | const PORT = process.env.PORT || '9000'; 9 | 10 | module.exports = merge(common('development'), { 11 | mode: 'development', 12 | devtool: 'eval-source-map', 13 | devServer: { 14 | host: HOST, 15 | port: PORT, 16 | historyApiFallback: true, 17 | open: true, 18 | static: { 19 | directory: path.resolve(__dirname, 'dist'), 20 | }, 21 | client: { 22 | overlay: true, 23 | }, 24 | }, 25 | module: { 26 | rules: [ 27 | { 28 | test: /\.css$/, 29 | include: [...stylePaths], 30 | use: ['style-loader', 'css-loader'], 31 | }, 32 | ], 33 | }, 34 | }); 35 | -------------------------------------------------------------------------------- /collections/red_hat_build_of_keycloak.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_build_of_keycloak", 3 | "collection_full_name": "Red Hat build of Keycloak", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "26.0", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "24.0", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "22.0", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | } 36 | ] 37 | } -------------------------------------------------------------------------------- /collections/red_hat_software_collections.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_software_collections", 3 | "collection_full_name": "Red Hat Software Collections", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "3", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "2", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | } 36 | ] 37 | } -------------------------------------------------------------------------------- /collections/red_hat_service_interconnect.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_service_interconnect", 3 | "collection_full_name": "Red Hat Service Interconnect", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.8", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.5", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.4", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | } 36 | ] 37 | } -------------------------------------------------------------------------------- /app/backend/default_prompt.txt: -------------------------------------------------------------------------------- 1 | [INST] <> 2 | You are a helpful, respectful and honest assistant answering questions about products from the company "Red Hat". 3 | You need to answer a question I will ask. 4 | You must give your answer in {language} only, but don't translate any code. If your answer is not in English, don't give the English translation. 5 | If the question is about a specific product you will be given this product name, and references to provide you with additional information. 6 | If there are references, only use those references to create your answer. 7 | Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. 8 | If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. 9 | If you don't know the answer to a question, say you don't know. 10 | <> 11 | 12 | References: 13 | {{context}} 14 | 15 | Question: {{question}} [/INST] -------------------------------------------------------------------------------- /collections/red_hat_build_of_apache_camel.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_build_of_apache_camel", 3 | "collection_full_name": "Red Hat build of Apache Camel", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "4.8", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "4.4", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "4.0", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | } 36 | ] 37 | } -------------------------------------------------------------------------------- /collections/red_hat_build_of_optaplanner.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_build_of_optaplanner", 3 | "collection_full_name": "Red Hat build of OptaPlanner", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "8.38", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "8.33", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "8.29", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | } 36 | ] 37 | } -------------------------------------------------------------------------------- /collections/red_hat_connectivity_link.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_connectivity_link", 3 | "collection_full_name": "Red Hat Connectivity Link", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.2", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.1", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.0", 28 | "store_directive": "create_or_keep", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /collections/migration_toolkit_for_runtimes.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "migration_toolkit_for_runtimes", 3 | "collection_full_name": "Migration Toolkit for Runtimes", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.2", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.1", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.0", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | } 36 | ] 37 | } -------------------------------------------------------------------------------- /collections/red_hat_jboss_data_virtualization.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_jboss_data_virtualization", 3 | "collection_full_name": "Red Hat JBoss Data Virtualization", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "6.4", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "6.3", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "6.2", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | } 36 | ] 37 | } -------------------------------------------------------------------------------- /collections/red_hat_openshift_virtualization.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openshift_virtualization", 3 | "collection_full_name": "Red Hat OpenShift Virtualization", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "4.18", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "4.17", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "4.16", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | } 36 | ] 37 | } -------------------------------------------------------------------------------- /app/frontend/src/app/assets/bgimages/github-mark-white.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/frontend/src/app/assets/bgimages/github-mark.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /collections/red_hat_enterprise_linux_for_real_time.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_enterprise_linux_for_real_time", 3 | "collection_full_name": "Red Hat Enterprise Linux for Real Time", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "9", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "8", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "7", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | } 36 | ] 37 | } -------------------------------------------------------------------------------- /collections/red_hat_openshift_service_mesh.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openshift_service_mesh", 3 | "collection_full_name": "Red Hat OpenShift Service Mesh", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "3.2", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "3.1", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "3.0.0tp1", 28 | "store_directive": "create_or_keep", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /app/frontend/stylePaths.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | module.exports = { 3 | stylePaths: [ 4 | path.resolve(__dirname, 'src'), 5 | path.resolve(__dirname, 'node_modules/patternfly'), 6 | path.resolve(__dirname, 'node_modules/@patternfly/patternfly'), 7 | path.resolve(__dirname, 'node_modules/@patternfly/chatbot/dist/css/main.css'), 8 | path.resolve(__dirname, 'node_modules/@patternfly/react-styles/css'), 9 | path.resolve(__dirname, 'node_modules/@patternfly/react-core/dist/styles/base.css'), 10 | path.resolve(__dirname, 'node_modules/@patternfly/react-core/dist/esm/@patternfly/patternfly'), 11 | path.resolve(__dirname, 'node_modules/@patternfly/react-core/node_modules/@patternfly/react-styles/css'), 12 | path.resolve(__dirname, 'node_modules/@patternfly/react-table/node_modules/@patternfly/react-styles/css'), 13 | path.resolve(__dirname, 'node_modules/@patternfly/react-inline-edit-extension/node_modules/@patternfly/react-styles/css') 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /app/frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": ".", 4 | "rootDir": ".", 5 | "outDir": "dist", 6 | "module": "esnext", 7 | "target": "es5", 8 | "lib": ["es6", "dom"], 9 | "sourceMap": true, 10 | "jsx": "react", 11 | "moduleResolution": "node", 12 | "forceConsistentCasingInFileNames": true, 13 | "noImplicitReturns": true, 14 | "noImplicitThis": true, 15 | "noImplicitAny": false, 16 | "allowJs": true, 17 | "esModuleInterop": true, 18 | "allowSyntheticDefaultImports": true, 19 | "strict": true, 20 | "downlevelIteration": true, 21 | "paths": { 22 | "@app/*": ["src/app/*"], 23 | "@assets/*": ["node_modules/@patternfly/react-core/dist/styles/assets/*"] 24 | }, 25 | "importHelpers": true, 26 | "skipLibCheck": true 27 | }, 28 | "include": [ 29 | "**/*.ts", 30 | "**/*.tsx", 31 | "**/*.jsx", 32 | "**/*.js", 33 | "**/.eslintrc.js" 34 | ], 35 | "exclude": ["node_modules"] 36 | } 37 | -------------------------------------------------------------------------------- /app/frontend/src/app/assets/bgimages/star.svg: -------------------------------------------------------------------------------- 1 | 2 | 11 | 13 | 17 | 18 | -------------------------------------------------------------------------------- /app/frontend/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | [ 4 | '@babel/preset-env', 5 | { 6 | targets: { 7 | chrome: 100, 8 | }, 9 | }, 10 | ], 11 | '@babel/preset-react', 12 | '@babel/preset-typescript', 13 | ], 14 | plugins: [ 15 | // Devs tend to write `import { someIcon } from '@patternfly/react-icons';` 16 | // This transforms the import to be specific which prevents having to parse 2k+ icons 17 | // Also prevents potential bundle size blowups with CJS 18 | [ 19 | 'transform-imports', 20 | { 21 | '@patternfly/react-icons': { 22 | transform: (importName) => 23 | `@patternfly/react-icons/dist/js/icons/${ 24 | importName === 'PathMissingIcon' 25 | ? 'pathMissing-icon' 26 | : importName 27 | .split(/(?=[A-Z])/) 28 | .join('-') 29 | .toLowerCase() 30 | }`, 31 | preventFullImport: true, 32 | }, 33 | }, 34 | ], 35 | ], 36 | }; 37 | -------------------------------------------------------------------------------- /app/frontend/src/app/assets/bgimages/star-white.svg: -------------------------------------------------------------------------------- 1 | 2 | 11 | 13 | 17 | 18 | -------------------------------------------------------------------------------- /app/frontend/src/app/components/NotFound/NotFound.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { ExclamationTriangleIcon } from '@patternfly/react-icons'; 3 | import { 4 | Button, 5 | EmptyState, 6 | EmptyStateBody, 7 | EmptyStateFooter, 8 | PageSection, 9 | } from '@patternfly/react-core'; 10 | import { useNavigate } from 'react-router-dom'; 11 | 12 | const NotFound: React.FunctionComponent = () => { 13 | function GoHomeBtn() { 14 | const navigate = useNavigate(); 15 | function handleClick() { 16 | navigate('/'); 17 | } 18 | return ( 19 | 20 | ); 21 | } 22 | 23 | return ( 24 | 25 | 26 | 27 | We didn't find a page that matches the address you navigated to. 28 | 29 | 30 | 31 | 32 | ) 33 | }; 34 | 35 | export { NotFound }; -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 rh-aiservices-bu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /app/frontend/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Red Hat 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /collections/red_hat_amq.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_amq", 3 | "collection_full_name": "Red Hat AMQ", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "2021.q4", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "2021.q3", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "2021.q2", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "2021.q1", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | } 46 | ] 47 | } -------------------------------------------------------------------------------- /app/frontend/webpack.prod.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-var-requires */ 2 | 3 | const { merge } = require('webpack-merge'); 4 | const common = require('./webpack.common.js'); 5 | const { stylePaths } = require('./stylePaths.js'); 6 | const MiniCssExtractPlugin = require('mini-css-extract-plugin'); 7 | const CssMinimizerPlugin = require('css-minimizer-webpack-plugin'); 8 | const TerserJSPlugin = require('terser-webpack-plugin'); 9 | 10 | module.exports = merge(common('production'), { 11 | mode: 'production', 12 | devtool: 'source-map', 13 | optimization: { 14 | minimizer: [ 15 | new TerserJSPlugin({}), 16 | new CssMinimizerPlugin({ 17 | minimizerOptions: { 18 | preset: ['default', { mergeLonghand: false }], 19 | }, 20 | }), 21 | ], 22 | }, 23 | plugins: [ 24 | new MiniCssExtractPlugin({ 25 | filename: '[name].css', 26 | chunkFilename: '[name].bundle.css', 27 | }), 28 | ], 29 | module: { 30 | rules: [ 31 | { 32 | test: /\.css$/, 33 | include: [...stylePaths], 34 | use: [MiniCssExtractPlugin.loader, 'css-loader'], 35 | }, 36 | ], 37 | }, 38 | }); 39 | -------------------------------------------------------------------------------- /collections/red_hat_build_of_openjdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_build_of_openjdk", 3 | "collection_full_name": "Red Hat build of OpenJDK", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "21", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "17", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "11", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "8", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | } 46 | ] 47 | } -------------------------------------------------------------------------------- /collections/red_hat_developer_toolset.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_developer_toolset", 3 | "collection_full_name": "Red Hat Developer Toolset", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "12", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "11", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "10", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "9", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | } 46 | ] 47 | } -------------------------------------------------------------------------------- /collections/red_hat_amq_broker.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_amq_broker", 3 | "collection_full_name": "Red Hat AMQ Broker", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "7.13", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "7.12", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "7.11", 28 | "store_directive": "create_or_keep", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "7.10", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | } 46 | ] 47 | } 48 | -------------------------------------------------------------------------------- /collections/red_hat_enterprise_linux_ai.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_enterprise_linux_ai", 3 | "collection_full_name": "Red Hat Enterprise Linux AI", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.4", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.3", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.2", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "1.1", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | } 46 | ] 47 | } -------------------------------------------------------------------------------- /app/frontend/src/app/assets/bgimages/default-user.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 11 | 12 | 13 | 14 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /collections/builds_for_red_hat_openshift.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "builds_for_red_hat_openshift", 3 | "collection_full_name": "Builds for Red Hat OpenShift", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.3", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.2", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.1", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "1.0", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | } 46 | ] 47 | } -------------------------------------------------------------------------------- /app/backend/helpers.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | # Set up custom logging as we'll be intermixes with FastAPI/Uvicorn's logging 4 | class ColoredLogFormatter(logging.Formatter): 5 | COLOR_CODES = { 6 | logging.DEBUG: "\033[94m", # Blue 7 | logging.INFO: "\033[92m", # Green 8 | logging.WARNING: "\033[93m", # Yellow 9 | logging.ERROR: "\033[91m", # Red 10 | logging.CRITICAL: "\033[95m", # Magenta 11 | } 12 | RESET_CODE = "\033[0m" 13 | 14 | def format(self, record): 15 | color = self.COLOR_CODES.get(record.levelno, "") 16 | record.levelname = f"{color}{record.levelname}{self.RESET_CODE}" 17 | return super().format(record) 18 | 19 | 20 | def logging_config(): 21 | logging.basicConfig( 22 | level=logging.INFO, # Set the logging level 23 | format="%(levelname)s:\t%(name)s - %(message)s", 24 | datefmt="%H:%M:%S", 25 | ) 26 | 27 | # Override the formatter with the custom ColoredLogFormatter 28 | root_logger = logging.getLogger() # Get the root logger 29 | for handler in root_logger.handlers: # Iterate through existing handlers 30 | if handler.formatter: 31 | handler.setFormatter(ColoredLogFormatter(handler.formatter._fmt)) -------------------------------------------------------------------------------- /app/frontend/jest.config.js: -------------------------------------------------------------------------------- 1 | // For a detailed explanation regarding each configuration property, visit: 2 | // https://jestjs.io/docs/en/configuration.html 3 | 4 | module.exports = { 5 | // Automatically clear mock calls and instances between every test 6 | clearMocks: true, 7 | 8 | // Indicates whether the coverage information should be collected while executing the test 9 | collectCoverage: false, 10 | 11 | // The directory where Jest should output its coverage files 12 | coverageDirectory: 'coverage', 13 | 14 | // An array of directory names to be searched recursively up from the requiring module's location 15 | moduleDirectories: [ 16 | "node_modules", 17 | "/src" 18 | ], 19 | 20 | // A map from regular expressions to module names that allow to stub out resources with a single module 21 | moduleNameMapper: { 22 | '\\.(css|less)$': '/__mocks__/styleMock.js', 23 | "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "/__mocks__/fileMock.js", 24 | "@app/(.*)": '/src/app/$1' 25 | }, 26 | 27 | // A preset that is used as a base for Jest's configuration 28 | preset: "ts-jest/presets/js-with-ts", 29 | 30 | // The test environment that will be used for testing. 31 | testEnvironment: "jsdom", 32 | }; 33 | -------------------------------------------------------------------------------- /app/frontend/public/locales/cn/translation.json: -------------------------------------------------------------------------------- 1 | { 2 | "language_code": "zh-CN", 3 | "app_title": "文档助理演示", 4 | "app_menu": { 5 | "chat_documentation": "文档助理演示" 6 | }, 7 | "app_header": { 8 | "powered_by": "文档助理演示", 9 | "help": { 10 | "header": "信息", 11 | "body": "此应用程序不是Red Hat文档的官方聊天机器人。仅用于演示目的。", 12 | "footer": "源代码可在https://github.com/rh-aiservices-bu/rh-kb-chat获取" 13 | }, 14 | "select_language": "选择你的语言", 15 | "anonymous_user": "匿名用户" 16 | }, 17 | "chat": { 18 | "title": "文档助理", 19 | "filter": { 20 | "product": "红帽产品", 21 | "version": "版本", 22 | "language": "语言" 23 | }, 24 | "placeholder": "问我任何问题...", 25 | "new_chat": "开始新的聊天", 26 | "send": "发送你的查询", 27 | "disclaimer1": "此助手由AI提供支持。它可能会显示不准确的信息，请务必仔细检查响应。", 28 | "disclaimer2": "不是Red Hat文档的官方聊天机器人。仅用于演示目的。", 29 | "content": { 30 | "greeting": "你好！我是你的文档助理演示。我今天能帮你什么吗？", 31 | "change_product_prompt_start": "好的，我们现在谈论的是", 32 | "change_product_prompt_version": "版本", 33 | "change_product_prompt_none": "好的，我们没有谈论任何特定的产品。", 34 | "new_chat_product": "我们在谈论", 35 | "new_chat_version": "版本", 36 | "new_chat_no_product": "我们没有谈论任何特定的产品。请选择一个或告诉我你需要什么帮助！", 37 | "empty_query": "请输入内容以开始对话。", 38 | "references": "参考文献" 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /collections/red_hat_build_of_node.js.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_build_of_node.js", 3 | "collection_full_name": "Red Hat build of Node.js", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "20", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "18", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "16", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "14", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "12", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | } 56 | ] 57 | } -------------------------------------------------------------------------------- /collections/red_hat_virtualization.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_virtualization", 3 | "collection_full_name": "Red Hat Virtualization", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "4.4", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "4.3", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "4.2", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "4.1", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "4.0", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | } 56 | ] 57 | } -------------------------------------------------------------------------------- /app/backend/classes.py: -------------------------------------------------------------------------------- 1 | class Source: 2 | def __init__(self, ingestion_type, language=None, urls=None): 3 | self.ingestion_type: str = ingestion_type 4 | self.language: str | None = language 5 | self.urls: list[str] | None = urls 6 | 7 | def __repr__(self): 8 | return f"Source(ingestion_type={self.ingestion_type}, language={self.language}, urls={self.urls})" 9 | 10 | 11 | class VersionInfo: 12 | def __init__(self, version_number, store_directive, sources): 13 | self.version_number: str = version_number 14 | self.store_directive: str = store_directive 15 | self.sources: list[Source] = sources 16 | 17 | def __repr__(self): 18 | return f"VersionInfo(version_number={self.version_number}, store_directive={self.store_directive}, sources={self.sources}" 19 | 20 | 21 | class Collection: 22 | def __init__(self, collection_base_name, collection_full_name, versions, common_sources=None): 23 | self.collection_base_name: str = collection_base_name 24 | self.collection_full_name: str = collection_full_name 25 | self.versions: list[VersionInfo] = versions 26 | self.common_sources: list[Source] | None = common_sources 27 | 28 | def __repr__(self): 29 | return f"ProductInfo(collection_base_name={self.collection_base_name}, collection_full_name={self.collection_full_name}, versions={self.versions}, common_sources={self.common_sources})" -------------------------------------------------------------------------------- /collections/red_hat_update_infrastructure.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_update_infrastructure", 3 | "collection_full_name": "Red Hat Update Infrastructure", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "4", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "3.1", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "3.0", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "2.1", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "2.0", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | } 56 | ] 57 | } -------------------------------------------------------------------------------- /products-documentation-ingestion/classes.py: -------------------------------------------------------------------------------- 1 | class Source: 2 | def __init__(self, ingestion_type, language=None, urls=None): 3 | self.ingestion_type: str = ingestion_type 4 | self.language: str | None = language 5 | self.urls: list[str] | None = urls 6 | 7 | def __repr__(self): 8 | return f"Source(ingestion_type={self.ingestion_type}, language={self.language}, urls={self.urls})" 9 | 10 | 11 | class VersionInfo: 12 | def __init__(self, version_number, store_directive, sources): 13 | self.version_number: str = version_number 14 | self.store_directive: str = store_directive 15 | self.sources: list[Source] = sources 16 | 17 | def __repr__(self): 18 | return f"VersionInfo(version_number={self.version_number}, store_directive={self.store_directive}, sources={self.sources}" 19 | 20 | 21 | class Collection: 22 | def __init__(self, collection_base_name, collection_full_name, versions, common_sources=None): 23 | self.collection_base_name: str = collection_base_name 24 | self.collection_full_name: str = collection_full_name 25 | self.versions: list[VersionInfo] = versions 26 | self.common_sources: list[Source] | None = common_sources 27 | 28 | def __repr__(self): 29 | return f"ProductInfo(collection_base_name={self.collection_base_name}, collection_full_name={self.collection_full_name}, versions={self.versions}, common_sources={self.common_sources})" -------------------------------------------------------------------------------- /collections/red_hat_jboss_core_services.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_jboss_core_services", 3 | "collection_full_name": "Red Hat JBoss Core Services", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "2.4.57", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "2.4.51", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "2.4.37", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "2.4.29", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "2.4.23", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | } 56 | ] 57 | } -------------------------------------------------------------------------------- /collections/red_hat_ansible_automation_platform.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_ansible_automation_platform", 3 | "collection_full_name": "Red Hat Ansible Automation Platform", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "2.5", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "2.4", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "2.3", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "2.2", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "2.1", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | } 56 | ] 57 | } -------------------------------------------------------------------------------- /collections/red_hat_trusted_application_pipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_trusted_application_pipeline", 3 | "collection_full_name": "Red Hat Trusted Application Pipeline", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.5", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.4", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.3", 28 | "store_directive": "create_or_keep", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "1.2", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "1.0", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | } 56 | ] 57 | } 58 | -------------------------------------------------------------------------------- /collections/red_hat_trusted_artifact_signer.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_trusted_artifact_signer", 3 | "collection_full_name": "Red Hat Trusted Artifact Signer", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.3", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.2", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.1", 28 | "store_directive": "create_or_keep", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "1.0", 38 | "store_directive": "create_or_keep", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "1", 48 | "store_directive": "create_or_keep", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | } 56 | ] 57 | } 58 | -------------------------------------------------------------------------------- /app/deployment/deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: rh-kb-chatbot 6 | spec: 7 | replicas: 1 8 | selector: 9 | matchLabels: 10 | app: rh-kb-chatbot 11 | template: 12 | metadata: 13 | creationTimestamp: null 14 | labels: 15 | app: rh-kb-chatbot 16 | deployment: rh-kb-chatbot 17 | spec: 18 | containers: 19 | - name: rh-kb-chatbot 20 | image: quay.io/rh-aiservices-bu/rh-kb-chat:2.0.0-alpha 21 | ports: 22 | - containerPort: 5000 23 | protocol: TCP 24 | env: 25 | - name: CONFIG_FILE 26 | value: '/opt/app-rot/src/config/config.json' 27 | resources: {} 28 | terminationMessagePath: /dev/termination-log 29 | terminationMessagePolicy: File 30 | imagePullPolicy: Always 31 | volumeMounts: 32 | - name: config-volume 33 | mountPath: '/opt/app-rot/src/config' 34 | readOnly: true 35 | restartPolicy: Always 36 | terminationGracePeriodSeconds: 30 37 | dnsPolicy: ClusterFirst 38 | securityContext: {} 39 | schedulerName: default-scheduler 40 | volumes: 41 | - name: config-volume 42 | secret: 43 | secretName: kb-chatbot 44 | strategy: 45 | type: RollingUpdate 46 | rollingUpdate: 47 | maxUnavailable: 25% 48 | maxSurge: 25% 49 | revisionHistoryLimit: 10 50 | progressDeadlineSeconds: 600 -------------------------------------------------------------------------------- /app/frontend/public/locales/jp/translation.json: -------------------------------------------------------------------------------- 1 | { 2 | "language_code": "ja-JP", 3 | "app_title": "ドキュメントアシスタントのデモ", 4 | "app_menu": { 5 | "chat_documentation": "ドキュメントアシスタントのデモ" 6 | }, 7 | "app_header": { 8 | "powered_by": "ドキュメントアシスタントのデモ", 9 | "help": { 10 | "header": "情報", 11 | "body": "このアプリケーションはRed Hatのドキュメントの公式チャットボットではありません。デモンストレーション目的のみです。", 12 | "footer": "ソースコードはhttps://github.com/rh-aiservices-bu/rh-kb-chatで利用可能" 13 | }, 14 | "select_language": "言語を選択してください", 15 | "anonymous_user": "匿名ユーザー" 16 | }, 17 | "chat": { 18 | "title": "ドキュメンテーションアシスタント", 19 | "filter": { 20 | "product": "レッドハット製品", 21 | "version": "バージョン", 22 | "language": "言語" 23 | }, 24 | "placeholder": "何でも聞いてください...", 25 | "new_chat": "新しいチャットを開始", 26 | "send": "クエリを送信", 27 | "disclaimer1": "このアシスタントはAIによって提供されています。不正確な情報が表示される場合があるため、回答を必ず確認してください。", 28 | "disclaimer2": "Red Hatのドキュメントの公式チャットボットではありません。デモンストレーション目的のみです。", 29 | "content": { 30 | "greeting": "こんにちは！私はあなたのドキュメンテーションアシスタントです。今日はどのようにお手伝いできますか？", 31 | "change_product_prompt_start": "わかりました、今は", 32 | "change_product_prompt_version": "バージョンについて話しています。", 33 | "change_product_prompt_none": "わかりました、特定の製品については話していません。", 34 | "new_chat_product": "話している製品は ", 35 | "new_chat_version": "バージョン", 36 | "new_chat_no_product": "特定の製品については話していません。1つ選択するか、どのようにお手伝いできるか教えてください！", 37 | "empty_query": "会話を始めるには何か入力してください。", 38 | "references": "参考文献" 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /app/frontend/src/app/routes.tsx: -------------------------------------------------------------------------------- 1 | import { NotFound } from '@app/components/NotFound/NotFound'; 2 | import { useDocumentTitle } from '@app/utils/useDocumentTitle'; 3 | import * as React from 'react'; 4 | import { Navigate, Route, Routes, useLocation } from 'react-router-dom'; 5 | import { Chat } from './components/Chat/Chat'; 6 | 7 | 8 | export interface IAppRoute { 9 | label?: string; // Excluding the label will exclude the route from the nav sidebar in AppLayout 10 | element: JSX.Element; 11 | path: string; 12 | title: string; 13 | routes?: undefined; 14 | bottomRoutes?: undefined; 15 | disabled?: boolean; 16 | } 17 | 18 | export interface IAppRouteGroup { 19 | label: string; 20 | routes: IAppRoute[]; 21 | } 22 | 23 | export type AppRouteConfig = IAppRoute | IAppRouteGroup; 24 | 25 | 26 | const routes: AppRouteConfig[] = [ 27 | { 28 | element: , 29 | path: '/', 30 | title: 'Redirect', 31 | }, 32 | { 33 | element: , 34 | label: 'app_menu.chat_documentation', 35 | path: '/Chat', 36 | title: 'app_menu.chat_documentation', 37 | } 38 | ]; 39 | 40 | 41 | const flattenedRoutes: IAppRoute[] = routes.reduce( 42 | (flattened, route) => [...flattened, ...(route.routes ? route.routes : [route])], 43 | [] as IAppRoute[] 44 | ); 45 | 46 | const AppRoutes = (): React.ReactElement => ( 47 | 48 | {flattenedRoutes.map((route, idx) => ( 49 | 50 | ))} 51 | } /> 52 | 53 | ); 54 | 55 | export { AppRoutes, routes }; 56 | -------------------------------------------------------------------------------- /collections/red_hat_gluster_storage.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_gluster_storage", 3 | "collection_full_name": "Red Hat Gluster Storage", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "3.5", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "3.4", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "3.3", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "3.2", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "3.1", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "3", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | } 66 | ] 67 | } -------------------------------------------------------------------------------- /collections/red_hat_build_of_debezium.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_build_of_debezium", 3 | "collection_full_name": "Red Hat build of Debezium", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "2.7.3", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "2.5.4", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "2.3.7", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "2.3.4", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "2.1.4", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "1.9.7", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | } 66 | ] 67 | } -------------------------------------------------------------------------------- /collections/red_hat_build_of_eclipse_vert.x.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_build_of_eclipse_vert.x", 3 | "collection_full_name": "Red Hat build of Eclipse Vert.x", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "4.3", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "4.2", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "4.1", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "4.0", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "3.9", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "3.8", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | } 66 | ] 67 | } -------------------------------------------------------------------------------- /collections/red_hat_support_for_spring_boot.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_support_for_spring_boot", 3 | "collection_full_name": "Red Hat support for Spring Boot", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "2.7", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "2.5", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "2.4", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "2.3", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "2.2", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "2.1", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | } 66 | ] 67 | } -------------------------------------------------------------------------------- /collections/red_hat_build_of_apicurio_registry.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_build_of_apicurio_registry", 3 | "collection_full_name": "Red Hat build of Apicurio Registry", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "3.0-Beta", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "2.6", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "2.5", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "2.4", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "2.3", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "2.0", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | } 66 | ] 67 | } -------------------------------------------------------------------------------- /collections/red_hat_hyperconverged_infrastructure_for_virtualization.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_hyperconverged_infrastructure_for_virtualization", 3 | "collection_full_name": "Red Hat Hyperconverged Infrastructure for Virtualization", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.8", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.7", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.6", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "1.5", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "1.1", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "1.0", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | } 66 | ] 67 | } -------------------------------------------------------------------------------- /app/frontend/src/app/components/Chat/classes.tsx: -------------------------------------------------------------------------------- 1 | export class Query { 2 | content: string; 3 | collection: string; 4 | collectionFullName: string; 5 | selectedVersion: string; 6 | language: string; 7 | timestamp: Date; 8 | type = 'Query'; 9 | 10 | constructor( 11 | content: string = '', 12 | collection: string = '', 13 | collectionFullName: string = '', 14 | selectedVersion: string = '', 15 | language: string = '', 16 | timestamp: Date = new Date() 17 | ) { 18 | this.content = content; 19 | this.collection = collection; 20 | this.collectionFullName = collectionFullName; 21 | this.selectedVersion = selectedVersion; 22 | this.language = language; 23 | this.timestamp = timestamp; 24 | } 25 | } 26 | 27 | export class Answer { 28 | content: string[]; 29 | sources: Source[]; 30 | timestamp: Date; 31 | type = 'Answer'; 32 | 33 | constructor(content: string[], sources, timestamp) { 34 | this.content = content; 35 | this.sources = sources; 36 | this.timestamp = timestamp; 37 | } 38 | } 39 | 40 | export class Source { 41 | content: string; 42 | score: number; 43 | 44 | constructor(content: string, score: number) { 45 | this.content = content; 46 | this.score = score; 47 | } 48 | } 49 | 50 | export class MessageContent { 51 | messageContent: Query | Answer ; 52 | 53 | constructor(messageContent: Query | Answer ) { 54 | this.messageContent = messageContent; 55 | } 56 | } 57 | 58 | export class MessageHistory { 59 | message: MessageContent[]; 60 | 61 | constructor(message: MessageContent[]) { 62 | this.message = message; 63 | } 64 | } 65 | 66 | export class Models { 67 | name: string; 68 | 69 | constructor(name: string) { 70 | this.name = name; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /products-documentation-ingestion/doc_processing_docling_server.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from dotenv import load_dotenv 3 | import os 4 | import md_splitter 5 | from langchain_core.documents import Document 6 | 7 | load_dotenv() 8 | 9 | 10 | def docling_processing(url): 11 | # Process the given URL using the Docling API 12 | api_address = os.getenv("DOCLING_API_URL") 13 | api_key = os.getenv("DOCLING_API_KEY") 14 | headers = {"Authorization": f"Bearer {api_key}"} 15 | print(f"Processing: {url}") 16 | payload = { 17 | "http_sources": [{"url": url}], 18 | "options": {"to_formats": ["md"], "image_export_mode": "placeholder"}, 19 | } 20 | response = requests.post( 21 | f"{api_address}/v1alpha/convert/source", json=payload, headers=headers 22 | ) 23 | md_content = response.json()["document"]["md_content"] 24 | return md_content 25 | 26 | 27 | def generate_splits(urls, product, product_full_name, chunk_size, chunk_overlap): 28 | # Generate splits from the given URLs 29 | all_splits = [] 30 | for url in urls: 31 | print(f"Processing: {url}") 32 | md_content = docling_processing(url) 33 | # Get the title of the document from the last part of the URL 34 | title = url.split("/")[-1] 35 | # Add metadata 36 | metadata = {"source": url, "title": title} 37 | md_doc = [Document(page_content=md_content, metadata=metadata)] 38 | splits = md_splitter.split( 39 | md_doc, 40 | product, 41 | product_full_name, 42 | url=url, 43 | chunk_size=chunk_size, 44 | chunk_overlap=chunk_overlap, 45 | ) 46 | all_splits.extend(splits) 47 | print(f"Generated {len(all_splits)} splits.") 48 | 49 | return all_splits -------------------------------------------------------------------------------- /collections/red_hat_single_sign-on.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_single_sign-on", 3 | "collection_full_name": "Red Hat Single Sign-On", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "7.6", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "7.5", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "7.4", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "7.3", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "7.2", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "7.1", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "7.0", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | } 76 | ] 77 | } -------------------------------------------------------------------------------- /app/frontend/public/locales/en/translation.json: -------------------------------------------------------------------------------- 1 | { 2 | "language_code": "en-US", 3 | "app_title": "Documentation Assistant Demo", 4 | "app_menu": { 5 | "chat_documentation": "Documentation Assistant Demo" 6 | }, 7 | "app_header": { 8 | "powered_by": "Documentation Assistant Demo", 9 | "help": { 10 | "header": "Information", 11 | "body": "This application is not an official chatbot for Red Hat documentation. For demonstration purposes only.", 12 | "footer": "Source code available at https://github.com/rh-aiservices-bu/rh-kb-chat" 13 | }, 14 | "select_language": "Select your language", 15 | "anonymous_user": "Anonymous User" 16 | }, 17 | "chat": { 18 | "title": "Documentation Assistant", 19 | "filter": { 20 | "product": "Red Hat Product", 21 | "version": "Version", 22 | "language": "Language" 23 | }, 24 | "placeholder": "Ask me anything...", 25 | "new_chat": "Start a new chat", 26 | "send": "Send your query", 27 | "disclaimer1": "This assistant is powered by an AI. It may display inaccurate information, so please double-check the responses.", 28 | "disclaimer2": "Not an official chatbot for Red Hat documentation. For demonstration purposes only.", 29 | "content": { 30 | "greeting": "Hi! I am your documentation Assistant. How can I help you today?", 31 | "change_product_prompt_start": "Ok, we are now talking about", 32 | "change_product_prompt_version": "version", 33 | "change_product_prompt_none": "Ok, we are not talking about any specific product.", 34 | "new_chat_product": "We are talking about ", 35 | "new_chat_version": "version", 36 | "new_chat_no_product": "We are not talking about any specific product. Please choose one or tell me how can I help you!", 37 | "empty_query": "Please enter something to start the conversation.", 38 | "references": "References" 39 | } 40 | } 41 | } -------------------------------------------------------------------------------- /collections/red_hat_build_of_microshift.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_build_of_microshift", 3 | "collection_full_name": "Red Hat build of MicroShift", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "4.18", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "4.17", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "4.16", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "4.15", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "4.14", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "4.13", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "4.12", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | } 76 | ] 77 | } -------------------------------------------------------------------------------- /collections/migration_toolkit_for_virtualization.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "migration_toolkit_for_virtualization", 3 | "collection_full_name": "Migration Toolkit for Virtualization", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "2.7", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "2.6", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "2.5", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "2.4", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "2.3", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "2.2", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "2.1", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | } 76 | ] 77 | } -------------------------------------------------------------------------------- /collections/red_hat_enterprise_linux.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_enterprise_linux", 3 | "collection_full_name": "Red Hat Enterprise Linux", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "4", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "5", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "6", 28 | "store_directive": "create_or_keep", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "7", 38 | "store_directive": "create_or_keep", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "8", 48 | "store_directive": "create_or_keep", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "9", 58 | "store_directive": "create_or_keep", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "10-beta", 68 | "store_directive": "create_or_keep", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | } 76 | ] 77 | } -------------------------------------------------------------------------------- /collections/red_hat_trusted_profile_analyzer.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_trusted_profile_analyzer", 3 | "collection_full_name": "Red Hat Trusted Profile Analyzer", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "2.1", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "2.0", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.3", 28 | "store_directive": "create_or_keep", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "1.2", 38 | "store_directive": "create_or_keep", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "1.1", 48 | "store_directive": "create_or_keep", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "1.0", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "1", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | } 76 | ] 77 | } 78 | -------------------------------------------------------------------------------- /collections/red_hat_jboss_enterprise_application_platform.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_jboss_enterprise_application_platform", 3 | "collection_full_name": "Red Hat JBoss Enterprise Application Platform", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "8.0", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "7.4", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "7.3", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "7.2", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "7.1", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "7.0", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "6.4", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | } 76 | ] 77 | } -------------------------------------------------------------------------------- /app/frontend/public/locales/de/translation.json: -------------------------------------------------------------------------------- 1 | { 2 | "language_code": "de-DE", 3 | "app_title": "Demo des Dokumentationsassistenten", 4 | "app_menu": { 5 | "chat_documentation": "Demo des Dokumentationsassistenten" 6 | }, 7 | "app_header": { 8 | "powered_by": "Demo des Dokumentationsassistenten", 9 | "help": { 10 | "header": "Information", 11 | "body": "Diese Anwendung ist kein offizieller Chatbot für Red Hat-Dokumentationen. Nur für Demonstrationszwecke.", 12 | "footer": "Quellcode verfügbar unter https://github.com/rh-aiservices-bu/rh-kb-chat" 13 | }, 14 | "select_language": "Wählen Sie Ihre Sprache", 15 | "anonymous_user": "Anonymer Benutzer" 16 | }, 17 | "chat": { 18 | "title": "Dokumentationsassistent", 19 | "filter": { 20 | "product": "Red Hat-Produkt", 21 | "version": "Version", 22 | "language": "Sprache" 23 | }, 24 | "placeholder": "Fragen Sie mich alles...", 25 | "new_chat": "Neuen Chat starten", 26 | "send": "Ihre Anfrage senden", 27 | "disclaimer1": "Dieser Assistent wird von einer KI unterstützt. Er kann ungenaue Informationen anzeigen, daher überprüfen Sie bitte die Antworten.", 28 | "disclaimer2": "Kein offizieller Chatbot für Red Hat-Dokumentationen. Nur für Demonstrationszwecke.", 29 | "content": { 30 | "greeting": "Hallo! Ich bin Ihr Dokumentationsassistent. Wie kann ich Ihnen heute helfen?", 31 | "change_product_prompt_start": "Ok, wir sprechen jetzt über", 32 | "change_product_prompt_version": "Version", 33 | "change_product_prompt_none": "Ok, wir sprechen nicht über ein bestimmtes Produkt.", 34 | "new_chat_product": "Wir sprechen über ", 35 | "new_chat_version": "Version", 36 | "new_chat_no_product": "Wir sprechen nicht über ein bestimmtes Produkt. Bitte wählen Sie eines oder sagen Sie mir, wie ich Ihnen helfen kann!", 37 | "empty_query": "Bitte geben Sie etwas ein, um das Gespräch zu beginnen.", 38 | "references": "Referenzen" 39 | } 40 | } 41 | } -------------------------------------------------------------------------------- /app/frontend/public/locales/es/translation.json: -------------------------------------------------------------------------------- 1 | { 2 | "language_code": "es-ES", 3 | "app_title": "Demo del asistente de documentación", 4 | "app_menu": { 5 | "chat_documentation": "Demo del asistente de documentación" 6 | }, 7 | "app_header": { 8 | "powered_by": "Demo del asistente de documentación", 9 | "help": { 10 | "header": "Información", 11 | "body": "Esta aplicación no es un chatbot oficial para la documentación de Red Hat. Solo para fines de demostración.", 12 | "footer": "Código fuente disponible en https://github.com/rh-aiservices-bu/rh-kb-chat" 13 | }, 14 | "select_language": "Seleccione su idioma", 15 | "anonymous_user": "Usuario Anónimo" 16 | }, 17 | "chat": { 18 | "title": "Asistente de Documentación", 19 | "filter": { 20 | "product": "Producto Red Hat", 21 | "version": "Versión", 22 | "language": "Idioma" 23 | }, 24 | "placeholder": "Pregúntame cualquier cosa...", 25 | "new_chat": "Iniciar un nuevo chat", 26 | "send": "Enviar su consulta", 27 | "disclaimer1": "Este asistente está impulsado por una IA. Puede mostrar información inexacta, así que por favor verifique las respuestas.", 28 | "disclaimer2": "No es un chatbot oficial para la documentación de Red Hat. Solo para fines de demostración.", 29 | "content": { 30 | "greeting": "¡Hola! Soy tu Asistente de Documentación. ¿Cómo puedo ayudarte hoy?", 31 | "change_product_prompt_start": "Ok, ahora estamos hablando sobre", 32 | "change_product_prompt_version": "versión", 33 | "change_product_prompt_none": "Ok, no estamos hablando sobre ningún producto específico.", 34 | "new_chat_product": "Estamos hablando sobre", 35 | "new_chat_version": "versión", 36 | "new_chat_no_product": "No estamos hablando sobre ningún producto específico. ¡Por favor elija uno o dígame cómo puedo ayudarle!", 37 | "empty_query": "Por favor ingrese algo para iniciar la conversación.", 38 | "references": "Referencias" 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /app/frontend/public/locales/fr/translation.json: -------------------------------------------------------------------------------- 1 | { 2 | "language_code": "fr-FR", 3 | "app_title": "Démo Assistant Documentaire", 4 | "app_menu": { 5 | "chat_documentation": "Démo Assistant Documentaire" 6 | }, 7 | "app_header": { 8 | "powered_by": "Démo Assistant Documentaire", 9 | "help": { 10 | "header": "Information", 11 | "body": "Cette application n'est pas un chatbot officiel pour la documentation Red Hat. À des fins de démonstration uniquement.", 12 | "footer": "Code source disponible sur https://github.com/rh-aiservices-bu/rh-kb-chat" 13 | }, 14 | "select_language": "Choisissez une langue", 15 | "anonymous_user": "Utilisateur anonyme" 16 | }, 17 | "chat": { 18 | "title": "Assistant Documentaire", 19 | "filter": { 20 | "product": "Produit Red Hat", 21 | "version": "Version", 22 | "language": "Langue" 23 | }, 24 | "placeholder": "Posez moi une question...", 25 | "new_chat": "Démarrer un nouveau chat", 26 | "send": "Envoyer", 27 | "disclaimer1": "Cet assistant est alimenté par une IA. Il peut afficher des informations inexactes, veuillez donc vérifier les réponses.", 28 | "disclaimer2": "Cette application n'est pas un chatbot officiel pour la documentation Red Hat. À des fins de démonstration uniquement.", 29 | "content": { 30 | "greeting": "Bonjour! Je suis votre assistant documentaire. Comment puis-je vous aider aujourd'hui ?", 31 | "change_product_prompt_start": "D'accord, nous parlons maintenant de", 32 | "change_product_prompt_version": "version", 33 | "change_product_prompt_none": "Ok, we are not talking about any specific product.", 34 | "new_chat_product": "Nous parlons de", 35 | "new_chat_version": "version", 36 | "new_chat_no_product": "Nous ne parlons d'aucun produit spécifique. Choisissez un produit svp ou dîtes-moi comment je peux vous aider.", 37 | "empty_query": "Entrez quelque chose pour commencer la conversation.", 38 | "references": "References" 39 | } 40 | } 41 | } -------------------------------------------------------------------------------- /products-documentation-ingestion/requirements_frozen.txt: -------------------------------------------------------------------------------- 1 | aiohappyeyeballs==2.4.6 2 | aiohttp==3.11.13 3 | aiosignal==1.3.2 4 | annotated-types==0.7.0 5 | anyio==4.8.0 6 | attrs==25.1.0 7 | beautifulsoup4==4.13.3 8 | cachetools==5.5.2 9 | certifi==2025.1.31 10 | charset-normalizer==3.4.1 11 | click==8.1.8 12 | dataclasses-json==0.6.7 13 | distro==1.9.0 14 | docstring_parser==0.16 15 | einops==0.8.1 16 | frozenlist==1.5.0 17 | google-api-core==2.24.1 18 | google-auth==2.38.0 19 | google-cloud-core==2.4.2 20 | google-cloud-storage==2.19.0 21 | google-crc32c==1.6.0 22 | google-resumable-media==2.7.2 23 | googleapis-common-protos==1.68.0 24 | grpcio==1.67.1 25 | h11==0.14.0 26 | html2text==2024.2.26 27 | httpcore==1.0.7 28 | httpx==0.28.1 29 | httpx-sse==0.4.0 30 | idna==3.10 31 | jiter==0.8.2 32 | jsonpatch==1.33 33 | jsonpointer==3.0.0 34 | kfp==2.11.0 35 | kfp-pipeline-spec==0.6.0 36 | kfp-server-api==2.3.0 37 | kubernetes==30.1.0 38 | langchain==0.3.19 39 | langchain-community==0.3.17 40 | langchain-core==0.3.39 41 | langchain-milvus==0.1.7 42 | langchain-text-splitters==0.3.6 43 | langsmith==0.1.129 44 | lxml==5.3.1 45 | marshmallow==3.26.1 46 | milvus-lite==2.4.11 47 | multidict==6.1.0 48 | mypy-extensions==1.0.0 49 | numpy==1.26.4 50 | oauthlib==3.2.2 51 | openai==1.63.2 52 | orjson==3.10.15 53 | packaging==24.2 54 | pandas==2.2.3 55 | propcache==0.3.0 56 | proto-plus==1.26.0 57 | protobuf==4.25.6 58 | pyasn1==0.6.1 59 | pyasn1_modules==0.4.1 60 | pydantic==2.10.6 61 | pydantic-settings==2.8.0 62 | pydantic_core==2.27.2 63 | pymilvus==2.4.10 64 | pypdf==5.3.0 65 | python-dateutil==2.9.0.post0 66 | python-dotenv==1.0.1 67 | pytz==2025.1 68 | PyYAML==6.0.2 69 | requests==2.32.3 70 | requests-oauthlib==2.0.0 71 | requests-toolbelt==0.10.1 72 | rsa==4.9 73 | six==1.17.0 74 | sniffio==1.3.1 75 | soupsieve==2.6 76 | SQLAlchemy==2.0.38 77 | tabulate==0.9.0 78 | tenacity==9.0.0 79 | tqdm==4.67.1 80 | typing-inspect==0.9.0 81 | typing_extensions==4.12.2 82 | tzdata==2025.1 83 | ujson==5.10.0 84 | urllib3==1.26.20 85 | websocket-client==1.8.0 86 | yarl==1.18.3 87 | -------------------------------------------------------------------------------- /app/frontend/src/app/app.test.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import App from '@app/index'; 3 | import { render, screen, } from '@testing-library/react'; 4 | import userEvent from '@testing-library/user-event' 5 | import '@testing-library/jest-dom' 6 | 7 | describe('App tests', () => { 8 | test('should render default App component', () => { 9 | const { asFragment } = render(); 10 | 11 | expect(asFragment()).toMatchSnapshot(); 12 | }); 13 | 14 | it('should render a nav-toggle button', () => { 15 | render(); 16 | 17 | expect(screen.getByRole('button', { name: "Global navigation"})).toBeVisible(); 18 | }); 19 | 20 | // I'm fairly sure that this test not going to work properly no matter what we do since JSDOM doesn't actually 21 | // draw anything. We could potentially make something work, likely using a different test environment, but 22 | // using Cypress for this kind of test would be more efficient. 23 | it.skip('should hide the sidebar on smaller viewports', () => { 24 | Object.defineProperty(window, 'innerWidth', { writable: true, configurable: true, value: 600 }); 25 | 26 | render(); 27 | 28 | window.dispatchEvent(new Event('resize')); 29 | 30 | expect(screen.queryByRole('link', { name: "Dashboard"})).not.toBeInTheDocument(); 31 | }); 32 | 33 | it('should expand the sidebar on larger viewports', () => { 34 | render(); 35 | 36 | window.dispatchEvent(new Event('resize')); 37 | 38 | expect(screen.getByRole('link', { name: "Dashboard"})).toBeVisible(); 39 | }); 40 | 41 | it('should hide the sidebar when clicking the nav-toggle button', async () => { 42 | const user = userEvent.setup() 43 | 44 | render(); 45 | 46 | window.dispatchEvent(new Event('resize')); 47 | const button = screen.getByRole('button', { name: "Global navigation"}) 48 | 49 | expect(screen.getByRole('link', { name: "Dashboard"})).toBeVisible(); 50 | 51 | await user.click(button); 52 | 53 | expect(screen.queryByRole('link', { name: "Dashboard"})).not.toBeInTheDocument(); 54 | }); 55 | }); 56 | -------------------------------------------------------------------------------- /collections/red_hat_ceph_storage.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_ceph_storage", 3 | "collection_full_name": "Red Hat Ceph Storage", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "8", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "7", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "6", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "5", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "4", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "3", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "2", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "1.3", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | } 86 | ] 87 | } -------------------------------------------------------------------------------- /collections/workload_availability_for_red_hat_openshift.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "workload_availability_for_red_hat_openshift", 3 | "collection_full_name": "Workload Availability for Red Hat OpenShift", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "25.1", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "24.4", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "24.3", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "24.2", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "24.1", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "23.3", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "23.2", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "23.1", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | } 86 | ] 87 | } -------------------------------------------------------------------------------- /products-documentation-ingestion/md_splitter.py: -------------------------------------------------------------------------------- 1 | from langchain_text_splitters import ( 2 | MarkdownHeaderTextSplitter, 3 | RecursiveCharacterTextSplitter, 4 | ) 5 | 6 | 7 | def split(md_docs, product, product_full_name, version=None, language=None, url=None, chunk_size=768, chunk_overlap=128): 8 | # Generates splits from a list of Markdown documents 9 | 10 | # Markdown splitter config 11 | headers_to_split_on = [ 12 | ("#", "Header 1"), 13 | ("##", "Header 2"), 14 | ("###", "Header 3"), 15 | ] 16 | 17 | markdown_splitter = MarkdownHeaderTextSplitter( 18 | headers_to_split_on=headers_to_split_on, strip_headers=True 19 | ) 20 | 21 | # Markdown split 22 | new_splits = [] 23 | for doc in md_docs: 24 | md_header_splits = markdown_splitter.split_text(doc.page_content) 25 | for split in md_header_splits: 26 | split.metadata = split.metadata | doc.metadata 27 | split.metadata["product"] = product 28 | split.metadata["product_full_name"] = product_full_name 29 | if version is not None: 30 | split.metadata["version"] = version 31 | if language is not None: 32 | split.metadata["language"] = language 33 | if url is not None: 34 | split.metadata["url"] = url 35 | 36 | new_splits.extend(md_header_splits) 37 | 38 | # Char-level splitter config 39 | text_splitter = RecursiveCharacterTextSplitter( 40 | chunk_size=chunk_size, chunk_overlap=chunk_overlap 41 | ) 42 | 43 | # Char-level split in case the Markdown split is too large 44 | splits = text_splitter.split_documents(new_splits) 45 | 46 | # Add content headers to each split 47 | for split in splits: 48 | # 'search_document:' part added to guide nomic in the embeddings 49 | content_header = f"search_document: Section: {split.metadata['title']}" 50 | for header_name in ["Header 1", "Header 2", "Header 3"]: 51 | if header_name in split.metadata: 52 | content_header += f" / {split.metadata[header_name]}" 53 | content_header += "\n\nContent:\n" 54 | split.page_content = content_header + split.page_content 55 | 56 | return splits 57 | -------------------------------------------------------------------------------- /tekton/doc-process-status-task.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: tekton.dev/v1 3 | kind: Task 4 | metadata: 5 | name: doc-process-status 6 | spec: 7 | params: 8 | - name: KFP_RUN_ID 9 | description: Run ID of the Data Science Pipeline 10 | type: string 11 | steps: 12 | - name: doc-process-status 13 | image: quay.io/openshift/origin-cli:4.14 14 | script: | 15 | #!/bin/bash 16 | 17 | LABEL="pipeline/runid=$(params.KFP_RUN_ID)" 18 | LOG_FILE="/tekton/results/pod-logs.txt" 19 | 20 | echo "Waiting for the system-container-impl pod with label $LABEL to start running...🥱" 21 | 22 | while true; do 23 | POD_NAME=$(oc get pods -l $LABEL -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep "system-container-impl") 24 | 25 | if [[ -n "$POD_NAME" ]]; then 26 | POD_STATUS=$(oc get pod "$POD_NAME" -o jsonpath='{.status.phase}' ) 27 | 28 | if [[ "$POD_STATUS" == "Running" ]]; then 29 | echo "Pod $POD_NAME is now Running. Starting log streaming...💪" 30 | break 31 | fi 32 | fi 33 | 34 | echo "Waiting for the pod to start...🥱" 35 | sleep 5 36 | done 37 | 38 | # Start `oc logs -f` in the background and redirect output to stdout 39 | oc logs -f "$POD_NAME" | grep --line-buffered -A2 PROCESSING 2>&1 & 40 | 41 | # Wait until the pod reaches Succeeded (or Failed) 42 | while true; do 43 | POD_STATUS=$(oc get pod "$POD_NAME" -o jsonpath='{.status.phase}' ) 44 | 45 | if [[ "$POD_STATUS" != "Running" ]]; then 46 | echo "🙅‍♀️ Pod $POD_NAME is not running anymore. Stopping log streaming..." 47 | break 48 | fi 49 | 50 | sleep 5 51 | done 52 | 53 | # Show the last 10 log lines 54 | echo "🤓 Fetching the last 1 minute logs before exit:" 55 | oc logs "$POD_NAME" --since=1m 56 | 57 | # Exit based on final pod status 58 | if [[ "$POD_STATUS" == "Succeeded" ]]; then 59 | echo "💚 Pod completed successfully. Exiting with success." 60 | exit 0 61 | else 62 | echo "💔 Pod failed. Exiting with failure." 63 | exit 1 64 | fi 65 | 66 | -------------------------------------------------------------------------------- /rh-doc-splits-generation/requirements.txt: -------------------------------------------------------------------------------- 1 | -i https://pypi.org/simple 2 | aiohttp==3.9.5; python_version >= '3.8' 3 | aiosignal==1.3.1; python_version >= '3.7' 4 | annotated-types==0.6.0; python_version >= '3.8' 5 | attrs==23.2.0; python_version >= '3.7' 6 | beautifulsoup4==4.12.2; python_full_version >= '3.6.0' 7 | certifi==2024.2.2; python_version >= '3.6' 8 | charset-normalizer==3.3.2; python_full_version >= '3.7.0' 9 | dataclasses-json==0.6.4; python_version >= '3.7' and python_version < '4.0' 10 | frozenlist==1.4.1; python_version >= '3.8' 11 | greenlet==3.0.3; platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))) 12 | html2text==2024.2.26; python_version >= '3.8' 13 | idna==3.7; python_version >= '3.5' 14 | jsonpatch==1.33; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6' 15 | jsonpointer==2.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6' 16 | langchain==0.1.12; python_version < '4.0' and python_full_version >= '3.8.1' 17 | langchain-community==0.0.34; python_version < '4.0' and python_full_version >= '3.8.1' 18 | langchain-core==0.1.45; python_version < '4.0' and python_full_version >= '3.8.1' 19 | langchain-text-splitters==0.0.1; python_version < '4.0' and python_full_version >= '3.8.1' 20 | langsmith==0.1.50; python_version < '4.0' and python_full_version >= '3.8.1' 21 | lxml==5.1.0; python_version >= '3.6' 22 | marshmallow==3.21.1; python_version >= '3.8' 23 | multidict==6.0.5; python_version >= '3.7' 24 | mypy-extensions==1.0.0; python_version >= '3.5' 25 | numpy==1.26.4; python_version >= '3.9' 26 | orjson==3.10.1; python_version >= '3.8' 27 | packaging==23.2; python_version >= '3.7' 28 | pydantic==2.7.1; python_version >= '3.8' 29 | pydantic-core==2.18.2; python_version >= '3.8' 30 | pyyaml==6.0.1; python_version >= '3.6' 31 | requests==2.31.0; python_version >= '3.7' 32 | soupsieve==2.5; python_version >= '3.8' 33 | sqlalchemy==2.0.29; python_version >= '3.7' 34 | tenacity==8.2.3; python_version >= '3.7' 35 | tqdm==4.66.2; python_version >= '3.7' 36 | typing-extensions==4.11.0; python_version >= '3.8' 37 | typing-inspect==0.9.0 38 | urllib3==2.2.1; python_version >= '3.8' 39 | yarl==1.9.4; python_version >= '3.7' 40 | -------------------------------------------------------------------------------- /collections/red_hat_build_of_quarkus.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_build_of_quarkus", 3 | "collection_full_name": "Red Hat build of Quarkus", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "3.15", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "3.8", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "3.2", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "2.13", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "2.7", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "2.2", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "1.11", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "1.7", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | }, 86 | { 87 | "version_number": "1.3", 88 | "store_directive": "delete", 89 | "sources": [ 90 | { 91 | "ingestion_type": "redhat_doc", 92 | "language": "en-US" 93 | } 94 | ] 95 | } 96 | ] 97 | } -------------------------------------------------------------------------------- /collections/red_hat_openstack_platform.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openstack_platform", 3 | "collection_full_name": "Red Hat OpenStack Platform", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "17.1", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "17.0", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "16.2", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "16.1", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "16.0", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "15", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "13", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "10", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | }, 86 | { 87 | "version_number": "8", 88 | "store_directive": "delete", 89 | "sources": [ 90 | { 91 | "ingestion_type": "redhat_doc", 92 | "language": "en-US" 93 | } 94 | ] 95 | } 96 | ] 97 | } -------------------------------------------------------------------------------- /app/frontend/.eslintrc.js: -------------------------------------------------------------------------------- 1 | export default { 2 | // tells eslint to use the TypeScript parser 3 | "parser": "@typescript-eslint/parser", 4 | // tell the TypeScript parser that we want to use JSX syntax 5 | "parserOptions": { 6 | "tsx": true, 7 | "jsx": true, 8 | "js": true, 9 | "useJSXTextNode": true, 10 | "project": "./tsconfig.json", 11 | "tsconfigRootDir": __dirname 12 | }, 13 | // we want to use the recommended rules provided from the typescript plugin 14 | "extends": [ 15 | "@redhat-cloud-services/eslint-config-redhat-cloud-services", 16 | "eslint:recommended", 17 | "plugin:react/recommended", 18 | "plugin:@typescript-eslint/recommended" 19 | ], 20 | "globals": { 21 | "window": "readonly", 22 | "describe": "readonly", 23 | "test": "readonly", 24 | "expect": "readonly", 25 | "it": "readonly", 26 | "process": "readonly", 27 | "document": "readonly", 28 | "insights": "readonly", 29 | "shallow": "readonly", 30 | "render": "readonly", 31 | "mount": "readonly" 32 | }, 33 | "overrides": [ 34 | { 35 | "files": ["src/**/*.ts", "src/**/*.tsx"], 36 | "parser": "@typescript-eslint/parser", 37 | "plugins": ["@typescript-eslint"], 38 | "extends": ["plugin:@typescript-eslint/recommended"], 39 | "rules": { 40 | "react/prop-types": "off", 41 | "@typescript-eslint/no-unused-vars": "error" 42 | }, 43 | }, 44 | ], 45 | "settings": { 46 | "react": { 47 | "version": "^16.11.0" 48 | } 49 | }, 50 | // includes the typescript specific rules found here: https://github.com/typescript-eslint/typescript-eslint/tree/master/packages/eslint-plugin#supported-rules 51 | "plugins": [ 52 | "@typescript-eslint", 53 | "react-hooks", 54 | "eslint-plugin-react-hooks" 55 | ], 56 | "rules": { 57 | "sort-imports": [ 58 | "error", 59 | { 60 | "ignoreDeclarationSort": true 61 | } 62 | ], 63 | "@typescript-eslint/explicit-function-return-type": "off", 64 | "react-hooks/rules-of-hooks": "error", 65 | "react-hooks/exhaustive-deps": "warn", 66 | "@typescript-eslint/interface-name-prefix": "off", 67 | "prettier/prettier": "off", 68 | "import/no-unresolved": "off", 69 | "import/extensions": "off", 70 | "react/prop-types": "off" 71 | }, 72 | "env": { 73 | "browser": true, 74 | "node": true 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /collections/red_hat_developer_hub.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_developer_hub", 3 | "collection_full_name": "Red Hat Developer Hub", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.8", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.7", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.6", 28 | "store_directive": "create_or_keep", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "1.5", 38 | "store_directive": "create_or_keep", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "1.4", 48 | "store_directive": "create_or_keep", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "1.3", 58 | "store_directive": "create_or_keep", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "1.2", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "1.1", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | }, 86 | { 87 | "version_number": "1.0", 88 | "store_directive": "delete", 89 | "sources": [ 90 | { 91 | "ingestion_type": "redhat_doc", 92 | "language": "en-US" 93 | } 94 | ] 95 | } 96 | ] 97 | } 98 | -------------------------------------------------------------------------------- /collections/red_hat_openshift_data_foundation.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openshift_data_foundation", 3 | "collection_full_name": "Red Hat OpenShift Data Foundation", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "4.17", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "4.16", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "4.15", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "4.14", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "4.13", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "4.12", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "4.11", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "4.10", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | }, 86 | { 87 | "version_number": "4.9", 88 | "store_directive": "delete", 89 | "sources": [ 90 | { 91 | "ingestion_type": "redhat_doc", 92 | "language": "en-US" 93 | } 94 | ] 95 | } 96 | ] 97 | } -------------------------------------------------------------------------------- /collections/red_hat_openshift_serverless.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openshift_serverless", 3 | "collection_full_name": "Red Hat OpenShift Serverless", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.36", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.35", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.34", 28 | "store_directive": "create_or_keep", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "1.33", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "1.32", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "1.31", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "1.30", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "1.29", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | }, 86 | { 87 | "version_number": "1.28", 88 | "store_directive": "delete", 89 | "sources": [ 90 | { 91 | "ingestion_type": "redhat_doc", 92 | "language": "en-US" 93 | } 94 | ] 95 | } 96 | ] 97 | } 98 | -------------------------------------------------------------------------------- /tekton/execute-doc-ingestion-task.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: tekton.dev/v1 3 | kind: Task 4 | metadata: 5 | name: execute-doc-ingestion-pipeline 6 | spec: 7 | workspaces: 8 | - name: output 9 | params: 10 | - name: WORK_DIRECTORY 11 | description: Directory to start build in (handle multiple branches) 12 | type: string 13 | - name: APPLICATION_NAME 14 | description: Name of the model 15 | type: string 16 | results: 17 | - name: KFP_RUN_ID 18 | description: Run ID of the Data Science Pipeline 19 | steps: 20 | - name: execute-doc-ingestion 21 | workingDir: $(workspaces.output.path)/$(params.WORK_DIRECTORY) 22 | image: registry.redhat.io/ubi9/python-311@sha256:fc669a67a0ef9016c3376b2851050580b3519affd5ec645d629fd52d2a8b8e4a 23 | command: ["/bin/sh", "-c"] 24 | args: 25 | - | 26 | python3 -m pip install kfp.kubernetes 27 | cat << 'EOF' | python3 28 | import kfp 29 | import json 30 | from doc_ingestion_pipeline import doc_ingestion_pipeline 31 | 32 | namespace_file_path =\ 33 | '/var/run/secrets/kubernetes.io/serviceaccount/namespace' 34 | with open(namespace_file_path, 'r') as namespace_file: 35 | namespace = namespace_file.read() 36 | 37 | kubeflow_endpoint =\ 38 | f'https://ds-pipeline-dspa.{namespace}.svc:8443' 39 | 40 | sa_token_file_path = '/var/run/secrets/kubernetes.io/serviceaccount/token' 41 | with open(sa_token_file_path, 'r') as token_file: 42 | bearer_token = token_file.read() 43 | 44 | ssl_ca_cert =\ 45 | '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' 46 | 47 | print(f'Connecting to Data Science Pipelines: {kubeflow_endpoint}') 48 | client = kfp.Client( 49 | host=kubeflow_endpoint, 50 | existing_token=bearer_token, 51 | ssl_ca_cert=ssl_ca_cert 52 | ) 53 | 54 | # start a run 55 | print("🏃‍♂️ start a run") 56 | run_id = client.create_run_from_pipeline_func( 57 | doc_ingestion_pipeline, 58 | experiment_name="$(params.APPLICATION_NAME)-doc-ingestion", 59 | namespace=namespace, 60 | enable_caching=False 61 | ) 62 | 63 | # save run id to query the logs 64 | KFP_RUN_ID = run_id.run_id 65 | path = "$(results.KFP_RUN_ID.path)" 66 | with open(path, "w") as file: 67 | file.write(KFP_RUN_ID) 68 | 69 | # print("🎉 job finished 🙌") 70 | EOF 71 | 72 | -------------------------------------------------------------------------------- /app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "documentation-chatbot", 3 | "version": "3.0.0", 4 | "description": "Documentation Chabot.", 5 | "author": "", 6 | "license": "MIT", 7 | "private": true, 8 | "repository": { 9 | "type": "git", 10 | "url": "git+https://github.com/rh-aiservices-bu/rh-kb-chat.git" 11 | }, 12 | "homepage": "https://github.com/rh-aiservices-bu/rh-kb-chat.git/README.md", 13 | "bugs": { 14 | "url": "https://github.com/rh-aiservices-bu/rh-kb-chat.git/issues" 15 | }, 16 | "engines": { 17 | "node": ">=18.0.0" 18 | }, 19 | "scripts": { 20 | "build": "run-p -l build:*", 21 | "build:backend": "cd ./backend && export PATH=$PATH:/opt/app-root/src/.local/bin && python3 -m ensurepip --upgrade && pip3 install -r requirements.txt", 22 | "build:frontend": "cd ./frontend && npm run build && cp -r ./dist/* ../backend/public", 23 | "dev": "run-p -l dev:*", 24 | "dev:backend": "cd ./backend && source ./.venv/bin/activate && nodemon app.py dev", 25 | "dev:frontend": "cd ./frontend && npm run start:dev", 26 | "format": "prettier --write \"backend/**/*.ts\" \"frontend/**/*.ts\" \"frontend/**/*.tsx\"", 27 | "make": "make", 28 | "make:build": "make build", 29 | "make:deploy": "make deploy", 30 | "make:login": "make login", 31 | "make:undeploy": "make undeploy", 32 | "make:push": "make push", 33 | "postinstall": "run-p postinstall:*", 34 | "postinstall:backend": "echo \"backend venv needs to be activated\"", 35 | "postinstall:frontend": "cd ./frontend && npm install", 36 | "start": "run-p start:*", 37 | "start:backend": "cd ./backend && python3 app.py", 38 | "start:frontend": "echo \"...available at ./frontend/public\"", 39 | "test": "run-s test:backend test:frontend", 40 | "test:backend": "cd ./backend && npm run test", 41 | "test:frontend": "cd ./frontend && npm run test", 42 | "test:e2e": "cd ./frontend && npm run test:e2e", 43 | "test:accessibility": "cd ./frontend && npm run test:accessibility", 44 | "test:integration": "cd ./frontend && npm run test:integration", 45 | "test:unit": "cd ./frontend && npm run test:unit", 46 | "test:fix": "run-s test:fix-backend test:fix-frontend", 47 | "test:fix-backend": "cd ./backend && npm run test:fix", 48 | "test:fix-frontend": "cd ./frontend && npm run test:fix" 49 | }, 50 | "dependencies": { 51 | "dotenv": "^8.2.0", 52 | "dotenv-expand": "^5.1.0", 53 | "npm-run-all": "^4.1.5", 54 | "prettier": "^2.2.1" 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /tekton/README.md: -------------------------------------------------------------------------------- 1 | # Doc Ingestion Pipeline 2 | 3 | This pipeline automates the process of ingesting documentation into a data science pipeline ([doc ingestion pipeline](./../products-documentation-ingestion/)) when changes are detected in the [collections](./../collections/) directory. It uses GitHub webhooks to detect pushes to the main branch and executes a series of tasks to process the documentation. 4 | 5 | ## Components 6 | 7 | ### PersistentVolumeClaim 8 | - **Name**: `shared-workspace` 9 | - **Storage**: 3Gi 10 | - **Access Mode**: ReadWriteOnce 11 | - **Provisioner**: AWS EBS 12 | 13 | ### Event Listener 14 | - **Name**: `doc-ingestion-listener` 15 | - Listens for GitHub push events via a webhook 16 | 17 | ### Trigger 18 | - **Name**: `doc-ingestion-trigger` 19 | - Includes a CEL interceptor that filters for: 20 | - Push events to the `main` branch 21 | - Changes (modified, added, or removed) in the `collections` directory 22 | 23 | ### Pipeline 24 | - **Name**: `doc-ingestion` 25 | - **Parameters**: 26 | - `APPLICATION_NAME`: Repository name 27 | - `GIT_URL`: Git repository URL 28 | - `GIT_BRANCH`: Git branch name (default: main) 29 | 30 | ### Tasks 31 | 32 | 1. **fetch-ds-pipeline-repository** 33 | - Clones the Git repository to a shared workspace 34 | - Uses the OpenShift Pipelines git-clone task 35 | 36 | 2. **execute-doc-ingestion-pipeline** 37 | - Executes the data science pipeline for document ingestion 38 | - Uses a custom task that connects to Data Science Pipelines via Kubernetes API 39 | - Creates a run in the Data Science Pipeline environment 40 | 41 | 3. **doc-process-status** 42 | - Monitors the status of the data science pipeline run 43 | - Streams logs from the system-container-impl pod 44 | - Reports success or failure based on the pod status 45 | - Times out after 10 hours 46 | 47 | ## Execution Flow 48 | 49 | 1. A push to the main branch with changes in the collections directory triggers the pipeline 50 | 2. The repository is cloned to a shared workspace 51 | 3. The document ingestion data science pipeline is executed 52 | 4. Logs are streamed and the status is monitored until completion 53 | 54 | ## Route Configuration 55 | 56 | The EventListener is exposed via an OpenShift Route with TLS edge termination. 57 | 58 | ## Requirements 59 | 60 | - OpenShift cluster with Tekton Pipelines installed 61 | - Data Science Pipeline Application is deployed in the same namespace 62 | - GitHub repository configured with webhooks pointing to the EventListener route -------------------------------------------------------------------------------- /tekton/trigger.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: triggers.tekton.dev/v1beta1 3 | kind: EventListener 4 | metadata: 5 | name: doc-ingestion-listener 6 | spec: 7 | triggers: 8 | - triggerRef: doc-ingestion-trigger 9 | --- 10 | kind: Route 11 | apiVersion: route.openshift.io/v1 12 | metadata: 13 | name: el-doc-ingestion-listener 14 | spec: 15 | to: 16 | kind: Service 17 | name: el-doc-ingestion-listener 18 | weight: 100 19 | port: 20 | targetPort: http-listener 21 | wildcardPolicy: None 22 | tls: 23 | termination: edge 24 | insecureEdgeTerminationPolicy: Redirect 25 | --- 26 | apiVersion: triggers.tekton.dev/v1beta1 27 | kind: TriggerTemplate 28 | metadata: 29 | name: doc-ingestion-trigger-template 30 | spec: 31 | params: 32 | - name: gitrepositoryurl 33 | description: The git repository url 34 | - name: git-repo-name 35 | description: The name of the deployment to be created / patched 36 | - name: git-branch 37 | description: The git branch name 38 | resourcetemplates: 39 | - apiVersion: tekton.dev/v1 40 | kind: PipelineRun 41 | metadata: 42 | generateName: doc-ingestion- 43 | spec: 44 | pipelineRef: 45 | name: doc-ingestion 46 | params: 47 | - name: APPLICATION_NAME 48 | value: $(tt.params.git-repo-name) 49 | - name: GIT_URL 50 | value: $(tt.params.gitrepositoryurl) 51 | - name: GIT_BRANCH 52 | value: $(tt.params.git-branch) 53 | workspaces: 54 | - name: shared-workspace 55 | persistentVolumeClaim: 56 | claimName: shared-workspace 57 | --- 58 | apiVersion: triggers.tekton.dev/v1beta1 59 | kind: Trigger 60 | metadata: 61 | name: doc-ingestion-trigger 62 | spec: 63 | interceptors: 64 | - ref: 65 | name: "cel" 66 | params: 67 | - name: filter 68 | value: (header.match('X-GitHub-Event', 'push') && body.ref.split('/')[2] == 'main' && body.commits.exists(c, c.modified.exists(m, m.contains('collections')) || c.added.exists(a, a.contains('collections')) || c.removed.exists(r, r.contains('collections')))) 69 | - name: overlays 70 | value: 71 | - expression: body.ref.split('/')[2] 72 | key: branch_name 73 | bindings: 74 | - name: git-branch 75 | value: $(extensions.branch_name) 76 | - name: gitrepositoryurl 77 | value: $(body.repository.clone_url) 78 | - name: git-repo-name 79 | value: $(body.repository.name) 80 | template: 81 | ref: doc-ingestion-trigger-template 82 | -------------------------------------------------------------------------------- /tekton/pipeline.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: tekton.dev/v1 3 | kind: Pipeline 4 | metadata: 5 | name: doc-ingestion 6 | spec: 7 | workspaces: 8 | - name: shared-workspace 9 | params: 10 | - name: APPLICATION_NAME 11 | type: string 12 | default: "" 13 | - name: GIT_URL 14 | type: string 15 | default: "" 16 | - name: GIT_BRANCH 17 | type: string 18 | default: "" 19 | tasks: 20 | # Clone the repository 21 | - name: fetch-ds-pipeline-repository 22 | taskRef: 23 | resolver: cluster 24 | params: 25 | - name: kind 26 | value: task 27 | - name: name 28 | value: git-clone 29 | - name: namespace 30 | value: openshift-pipelines 31 | workspaces: 32 | - name: output 33 | workspace: shared-workspace 34 | params: 35 | - name: URL 36 | value: "$(params.GIT_URL)" 37 | - name: REVISION 38 | value: "$(params.GIT_BRANCH)" 39 | - name: SUBDIRECTORY 40 | value: "$(params.APPLICATION_NAME)/$(params.GIT_BRANCH)" 41 | - name: DELETE_EXISTING 42 | value: "true" 43 | - name: SSL_VERIFY 44 | value: "false" 45 | # Execute Doc Ingestion Data Science Pipeline 46 | - name: execute-doc-ingestion-pipeline 47 | taskRef: 48 | name: execute-doc-ingestion-pipeline 49 | kind: Task 50 | workspaces: 51 | - name: output 52 | workspace: shared-workspace 53 | params: 54 | - name: WORK_DIRECTORY 55 | value: "$(params.APPLICATION_NAME)/$(params.GIT_BRANCH)/products-documentation-ingestion" 56 | - name: APPLICATION_NAME 57 | value: "$(params.APPLICATION_NAME)" 58 | runAfter: 59 | - fetch-ds-pipeline-repository 60 | # Check the status of document processing 61 | - name: doc-process-status 62 | taskRef: 63 | name: doc-process-status 64 | kind: Task 65 | params: 66 | - name: KFP_RUN_ID 67 | value: "$(tasks.execute-doc-ingestion-pipeline.results.KFP_RUN_ID)" 68 | runAfter: 69 | - execute-doc-ingestion-pipeline 70 | timeout: 10h0m0s 71 | # Trigger rollout for the Chatbot app 72 | - name: trigger-rollout-chatbot 73 | taskRef: 74 | name: trigger-rollout-chatbot 75 | kind: Task 76 | runAfter: 77 | - doc-process-status 78 | results: 79 | - name: KFP_RUN_ID 80 | description: Run ID of the Data Science Pipeline 81 | value: "$(tasks.execute-doc-ingestion-pipeline.results.KFP_RUN_ID)" 82 | -------------------------------------------------------------------------------- /products-documentation-ingestion/doc_ingestion_pipeline.py: -------------------------------------------------------------------------------- 1 | # kfp imports 2 | import kfp 3 | import kfp.dsl as dsl 4 | from kfp.dsl import ( 5 | component 6 | ) 7 | from kfp import kubernetes 8 | 9 | # Misc imports 10 | import os 11 | 12 | # Component imports 13 | from documentation_ingestion import doc_ingest 14 | 15 | ######### Pipeline definition ######### 16 | 17 | ingestion_secret_name = 'doc-ingestion' 18 | 19 | # Create pipeline 20 | @dsl.pipeline( 21 | name='doc-ingestion-pipeline', 22 | description='Ingest ALL the documents📚💪' 23 | ) 24 | def doc_ingestion_pipeline(): 25 | ### 🐶 Document ingestion 26 | doc_ingestion_task = doc_ingest() 27 | kubernetes.use_secret_as_env( 28 | doc_ingestion_task, 29 | secret_name=ingestion_secret_name, 30 | secret_key_to_env={ 31 | 'EMBEDDINGS_API_KEY': 'EMBEDDINGS_API_KEY', 32 | 'EMBEDDINGS_API_URL': 'EMBEDDINGS_API_URL', 33 | 'MILVUS_URI': 'MILVUS_URI', 34 | 'MILVUS_PASSWORD': 'MILVUS_PASSWORD', 35 | 'MILVUS_USERNAME': 'MILVUS_USERNAME', 36 | 'MILVUS_DB': 'MILVUS_DB', 37 | 'MILVUS_BATCH_SIZE': 'MILVUS_BATCH_SIZE', 38 | 'CHUNK_SIZE': 'CHUNK_SIZE', 39 | 'CHUNK_OVERLAP': 'CHUNK_OVERLAP', 40 | 'EMBEDDINGS_API_URL': 'EMBEDDINGS_API_URL', 41 | 'EMBEDDINGS_API_KEY': 'EMBEDDINGS_API_KEY', 42 | 'EMBEDDINGS_MODEL_NAME': 'EMBEDDINGS_MODEL_NAME', 43 | 'DOCLING_API_URL': 'DOCLING_API_URL', 44 | 'DOCLING_API_KEY': 'DOCLING_API_KEY', 45 | 'COLLECTIONS_PATH': 'COLLECTIONS_PATH', 46 | 'COLLECTIONS_GIT_REPO_NAME': 'COLLECTIONS_GIT_REPO_NAME', 47 | 'COLLECTIONS_GIT_REPO_PATH': 'COLLECTIONS_GIT_REPO_PATH', 48 | 'COLLECTIONS_GIT_REPO_BRANCH': 'COLLECTIONS_GIT_REPO_BRANCH' 49 | }, 50 | ) 51 | 52 | if __name__ == '__main__': 53 | 54 | namespace_file_path =\ 55 | '/var/run/secrets/kubernetes.io/serviceaccount/namespace' 56 | with open(namespace_file_path, 'r') as namespace_file: 57 | namespace = namespace_file.read() 58 | 59 | kubeflow_endpoint =\ 60 | f'https://ds-pipeline-dspa.{namespace}.svc:8443' 61 | 62 | sa_token_file_path = '/var/run/secrets/kubernetes.io/serviceaccount/token' 63 | with open(sa_token_file_path, 'r') as token_file: 64 | bearer_token = token_file.read() 65 | 66 | ssl_ca_cert =\ 67 | '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' 68 | 69 | print(f'Connecting to Data Science Pipelines: {kubeflow_endpoint}') 70 | client = kfp.Client( 71 | host=kubeflow_endpoint, 72 | existing_token=bearer_token, 73 | ssl_ca_cert=ssl_ca_cert 74 | ) 75 | 76 | client.create_run_from_pipeline_func( 77 | doc_ingestion_pipeline, 78 | experiment_name="doc-ingestion", 79 | enable_caching=True 80 | ) 81 | -------------------------------------------------------------------------------- /products-documentation-ingestion/collections_loader.py: -------------------------------------------------------------------------------- 1 | from classes import Collection, Source, VersionInfo 2 | import os 3 | import requests 4 | import json 5 | 6 | class CollectionLoader: 7 | """Class to load collections from JSON files or from a git repository""" 8 | 9 | def load_collection_from_json(self, data: str) -> Collection: 10 | # Parse common_sources 11 | common_sources = [ 12 | Source( 13 | ingestion_type=source.get("ingestion_type"), 14 | language=source.get("language"), 15 | urls=source.get("urls") 16 | ) for source in data.get("common_sources", []) 17 | ] 18 | 19 | # Parse versions 20 | versions = [ 21 | VersionInfo( 22 | version_number=version.get("version_number"), 23 | store_directive=version.get("directive", version.get("store_directive")), # Handle possible key mismatch 24 | sources=[ 25 | Source( 26 | ingestion_type=src.get("ingestion_type"), 27 | language=src.get("language"), 28 | urls=src.get("urls") 29 | ) for src in version.get("sources", []) 30 | ] 31 | ) for version in data.get("versions", []) 32 | ] 33 | 34 | # Create the Collection object 35 | collection = Collection( 36 | collection_base_name=data.get("collection_base_name"), 37 | collection_full_name=data.get("collection_full_name"), 38 | versions=versions, 39 | common_sources=common_sources 40 | ) 41 | 42 | return collection 43 | 44 | 45 | def fetch_collections_from_git(self, collections, git_repo_name, git_repo_path, git_branch="main"): 46 | repo_url = f"https://api.github.com/repos/{git_repo_name}/contents/{git_repo_path}?ref={git_branch}" 47 | response = requests.get(repo_url) 48 | response.raise_for_status() 49 | files = response.json() 50 | 51 | for file in files: 52 | if file["name"].endswith(".json"): 53 | file_url = file["download_url"] 54 | file_response = requests.get(file_url) 55 | file_response.raise_for_status() 56 | data = file_response.json() 57 | collection = self.load_collection_from_json(data) 58 | collections.append(collection) 59 | 60 | return collections 61 | 62 | def fetch_collections_from_path(self, collections, path): 63 | for root, _, files in os.walk(path): 64 | for file in files: 65 | if file.endswith(".json"): 66 | with open(os.path.join(root, file), "r") as f: 67 | data = json.load(f) 68 | collection = self.load_collection_from_json(data) 69 | collections.append(collection) 70 | 71 | return collections -------------------------------------------------------------------------------- /collections/streams_for_apache_kafka.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "streams_for_apache_kafka", 3 | "collection_full_name": "Streams for Apache Kafka", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "3.0", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "2.9", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "2.8", 28 | "store_directive": "create_or_keep", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "2.7", 38 | "store_directive": "create_or_keep", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "2.6", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "2.5", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "2.4", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "2.3", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | }, 86 | { 87 | "version_number": "2.2", 88 | "store_directive": "delete", 89 | "sources": [ 90 | { 91 | "ingestion_type": "redhat_doc", 92 | "language": "en-US" 93 | } 94 | ] 95 | }, 96 | { 97 | "version_number": "2.1", 98 | "store_directive": "delete", 99 | "sources": [ 100 | { 101 | "ingestion_type": "redhat_doc", 102 | "language": "en-US" 103 | } 104 | ] 105 | }, 106 | { 107 | "version_number": "2.0", 108 | "store_directive": "delete", 109 | "sources": [ 110 | { 111 | "ingestion_type": "redhat_doc", 112 | "language": "en-US" 113 | } 114 | ] 115 | } 116 | ] 117 | } 118 | -------------------------------------------------------------------------------- /collections/red_hat_openshift_gitops.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openshift_gitops", 3 | "collection_full_name": "Red Hat OpenShift GitOps", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.18", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.17", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.16", 28 | "store_directive": "create_or_keep", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "1.15", 38 | "store_directive": "create_or_keep", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "1.14", 48 | "store_directive": "create_or_keep", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "1.13", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "1.12", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "1.11", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | }, 86 | { 87 | "version_number": "1.10", 88 | "store_directive": "delete", 89 | "sources": [ 90 | { 91 | "ingestion_type": "redhat_doc", 92 | "language": "en-US" 93 | } 94 | ] 95 | }, 96 | { 97 | "version_number": "1.9", 98 | "store_directive": "delete", 99 | "sources": [ 100 | { 101 | "ingestion_type": "redhat_doc", 102 | "language": "en-US" 103 | } 104 | ] 105 | }, 106 | { 107 | "version_number": "1.8", 108 | "store_directive": "delete", 109 | "sources": [ 110 | { 111 | "ingestion_type": "redhat_doc", 112 | "language": "en-US" 113 | } 114 | ] 115 | } 116 | ] 117 | } 118 | -------------------------------------------------------------------------------- /collections/red_hat_openshift_pipelines.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openshift_pipelines", 3 | "collection_full_name": "Red Hat OpenShift Pipelines", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "1.20", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "1.19", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "1.18", 28 | "store_directive": "create_or_keep", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "1.17", 38 | "store_directive": "create_or_keep", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "1.16", 48 | "store_directive": "create_or_keep", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "1.15", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "1.14", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "1.13", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | }, 86 | { 87 | "version_number": "1.12", 88 | "store_directive": "delete", 89 | "sources": [ 90 | { 91 | "ingestion_type": "redhat_doc", 92 | "language": "en-US" 93 | } 94 | ] 95 | }, 96 | { 97 | "version_number": "1.11", 98 | "store_directive": "delete", 99 | "sources": [ 100 | { 101 | "ingestion_type": "redhat_doc", 102 | "language": "en-US" 103 | } 104 | ] 105 | }, 106 | { 107 | "version_number": "1.10", 108 | "store_directive": "delete", 109 | "sources": [ 110 | { 111 | "ingestion_type": "redhat_doc", 112 | "language": "en-US" 113 | } 114 | ] 115 | } 116 | ] 117 | } 118 | -------------------------------------------------------------------------------- /app/backend/milvus_retriever_with_score_threshold.py: -------------------------------------------------------------------------------- 1 | """Milvus Retriever with Score Threshold""" 2 | import warnings 3 | from typing import Any, Dict, List, Optional 4 | 5 | from langchain_core.callbacks import CallbackManagerForRetrieverRun 6 | from langchain_core.documents import Document 7 | from langchain_core.embeddings import Embeddings 8 | from pydantic import model_validator 9 | from langchain_milvus import Milvus 10 | from langchain_core.retrievers import BaseRetriever 11 | 12 | class MilvusRetrieverWithScoreThreshold(BaseRetriever): 13 | """`Milvus API` retriever.""" 14 | 15 | embedding_function: Embeddings 16 | collection_name: str = "LangChainCollection" 17 | collection_description: str = "" 18 | collection_properties: Optional[Dict[str, Any]] = None 19 | connection_args: Optional[Dict[str, Any]] = None 20 | consistency_level: str = "Session" 21 | search_params: Optional[dict] = None 22 | k: int = 4 23 | score_threshold: float = 0.99 24 | enable_dynamic_field: bool =True 25 | text_field: str = "page_content" 26 | logger: Any 27 | 28 | store: Milvus 29 | retriever: BaseRetriever 30 | 31 | @model_validator(mode="before") 32 | def create_retriever(cls, values: Dict) -> Dict: 33 | """Create the Milvus store and retriever.""" 34 | values["store"] = Milvus( 35 | embedding_function=values["embedding_function"], 36 | collection_name=values["collection_name"], 37 | collection_description=values["collection_description"], 38 | collection_properties=values["collection_properties"], 39 | connection_args=values["connection_args"], 40 | consistency_level=values["consistency_level"], 41 | enable_dynamic_field=True, 42 | text_field="page_content" 43 | ) 44 | values["retriever"] = values["store"].as_retriever( 45 | search_kwargs={"param": values["search_params"]} 46 | ) 47 | return values 48 | 49 | def add_texts( 50 | self, texts: List[str], metadatas: Optional[List[dict]] = None 51 | ) -> None: 52 | """Add text to the Milvus store 53 | 54 | Args: 55 | texts (List[str]): The text 56 | metadatas (List[dict]): Metadata dicts, must line up with existing store 57 | """ 58 | self.store.add_texts(texts, metadatas) 59 | 60 | def _get_relevant_documents( 61 | self, 62 | query: str, 63 | *, 64 | run_manager: CallbackManagerForRetrieverRun, 65 | **kwargs: Any, 66 | ) -> List[Document]: 67 | self.logger.info(f"Store: {self.store.collection_name}") 68 | docs_and_scores = self.store.similarity_search_with_score(query, k=self.k, return_metadata=True) 69 | docs_and_scores = [(doc, score) for doc, score in docs_and_scores if score < self.score_threshold] 70 | for doc, score in docs_and_scores: 71 | doc.metadata = {**doc.metadata, **{"score": score}} 72 | return [doc for (doc, _) in docs_and_scores] 73 | -------------------------------------------------------------------------------- /rh-doc-splits-generation/rh-doc-splits-generation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "bb6d3a63", 6 | "metadata": {}, 7 | "source": [ 8 | "## Red Hat documentation splits generator\n", 9 | "\n", 10 | "This notebook shows how to generates splits for a Red Hat product documentation." 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "id": "55c77c4a", 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "# If you don't have the needed libraries, install them with:\n", 21 | "# !pip install -r requirements.txt" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "id": "4adbeacb-f78b-41bf-b33b-37163188a112", 28 | "metadata": { 29 | "tags": [] 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "import rh_documentation_processing as rhdp" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": null, 39 | "id": "456bae8d", 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "# Set the variables for the documentation\n", 44 | "# For this example the original URL was: https://access.redhat.com/documentation/en-us/red_hat_openshift_ai_self-managed/2.8\n", 45 | "# You must extract the following information from the URL:\n", 46 | "\n", 47 | "product = 'red_hat_openshift_ai_self-managed' # The second last part of the URL\n", 48 | "product_full_name = 'Red Hat OpenShift AI Self-Managed' # From the Web Page\n", 49 | "version = '2.8' # The last part of the URL\n", 50 | "language = 'en-US' # The language of the documentation" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": null, 56 | "id": "dcac987e", 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "# Generate the splits\n", 61 | "\n", 62 | "splits = rhdp.generate_splits(product, product_full_name, version, language)" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "id": "7768f97a", 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "# Print the splits\n", 73 | "\n", 74 | "for split in splits:\n", 75 | " print(f\"Metadata: {split.metadata}\")\n", 76 | " print(f\"Page Content:\\n-------------\\n{split.page_content}\")\n", 77 | " print(\"\\n\")" 78 | ] 79 | } 80 | ], 81 | "metadata": { 82 | "kernelspec": { 83 | "display_name": "Python 3.9", 84 | "language": "python", 85 | "name": "python3" 86 | }, 87 | "language_info": { 88 | "codemirror_mode": { 89 | "name": "ipython", 90 | "version": 3 91 | }, 92 | "file_extension": ".py", 93 | "mimetype": "text/x-python", 94 | "name": "python", 95 | "nbconvert_exporter": "python", 96 | "pygments_lexer": "ipython3", 97 | "version": "3.11.7" 98 | } 99 | }, 100 | "nbformat": 4, 101 | "nbformat_minor": 5 102 | } 103 | -------------------------------------------------------------------------------- /products-documentation-ingestion/README.md: -------------------------------------------------------------------------------- 1 | # Document Ingestion Data Science Pipeline 2 | 3 | This data science pipeline automates the process of ingesting documentation from collections and storing them in a Milvus vector database with embeddings for retrieval. 4 | 5 | ## Overview 6 | 7 | The pipeline processes documentation collections, generates embeddings, and stores them in Milvus to be used in retrieval systems. It handles various operations (create, update, delete) based on directives specified in the collection configuration. 8 | 9 | ## Components 10 | 11 | ### Main Pipeline (`doc_ingestion_pipeline.py`) 12 | - Defines a Kubeflow Pipeline for document ingestion 13 | - Connects to a Data Science Pipelines Application server 14 | - Executes the document ingestion task with necessary secrets 15 | 16 | ### Document Processor (`doc_processing_docling.py`) 17 | - KFP component that handles the actual document ingestion process 18 | - Loads collections from either a local path or Git repository 19 | - Processes documents according to their storage directives 20 | - Manages connections to Milvus and embedding services 21 | - Provides detailed progress logging 22 | 23 | ## Requirements 24 | 25 | ### Custom Image 26 | The pipeline requires a custom image built from the [Containerfile](./Containerfile) in this directory. The image is published at: `quay.io/rh-aiservices-bu/rh-kb-doc-ingestion` 27 | 28 | ### Required Secret 29 | The pipeline requires a Kubernetes secret named `doc-ingestion` containing the following keys: 30 | - `EMBEDDINGS_API_KEY`: API key for embeddings service 31 | - `EMBEDDINGS_API_URL`: URL for embeddings service 32 | - `EMBEDDINGS_MODEL_NAME`: Model name for embeddings 33 | - `MILVUS_URI`: URI for Milvus database 34 | - `MILVUS_USERNAME`: Milvus username 35 | - `MILVUS_PASSWORD`: Milvus password 36 | - `MILVUS_DB`: Milvus database name 37 | - `MILVUS_BATCH_SIZE`: Batch size for Milvus operations 38 | - `CHUNK_SIZE`: Size of text chunks for processing 39 | - `CHUNK_OVERLAP`: Overlap between chunks 40 | - `DOCLING_API_URL`: URL for Docling API 41 | - `DOCLING_API_KEY`: API key for Docling service 42 | - `COLLECTIONS_PATH`: Path to collections (optional) 43 | - `COLLECTIONS_GIT_REPO_NAME`: Git repo name (optional) 44 | - `COLLECTIONS_GIT_REPO_PATH`: Git repo path (optional) 45 | - `COLLECTIONS_GIT_REPO_BRANCH`: Git repo branch (optional) 46 | 47 | ## Usage 48 | 49 | This pipeline is typically triggered by the Tekton pipeline when changes are detected in the collections directory. It can also be run manually through the Data Science Pipelines UI or API. 50 | 51 | The pipeline will: 52 | 1. Load collections from the specified source 53 | 2. Process each collection according to its directive (create, update, delete) 54 | 3. Split documents into chunks with configurable size and overlap 55 | 4. Generate embeddings for the content chunks 56 | 5. Store the indexed content in Milvus 57 | 6. Log detailed progress information 58 | 59 | ## Supported Operations 60 | 61 | - `create_or_keep`: Creates a new collection if it doesn't exist, skips if it does 62 | - `update`: Creates or replaces a collection unconditionally 63 | - `delete`: Removes a collection if it exists -------------------------------------------------------------------------------- /app/frontend/src/app/assets/bgimages/orb.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 24 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 61 | 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /collections/red_hat_openshift_ai_self-managed.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_openshift_ai_self-managed", 3 | "collection_full_name": "Red Hat OpenShift AI Self-Managed", 4 | "common_sources": [ 5 | { 6 | "ingestion_type": "docling_server", 7 | "urls": [ 8 | "https://www.redhat.com/en/technologies/cloud-computing/openshift/openshift-ai", 9 | "https://www.redhat.com/en/resources/red-hat-openshift-ai-hybrid-cloud-brief", 10 | "https://ai-on-openshift.io/getting-started/openshift/", 11 | "https://ai-on-openshift.io/getting-started/opendatahub/", 12 | "https://ai-on-openshift.io/getting-started/openshift-ai/", 13 | "https://ai-on-openshift.io/odh-rhoai/accelerator-profiles/", 14 | "https://ai-on-openshift.io/odh-rhoai/gitops/", 15 | "https://ai-on-openshift.io/odh-rhoai/configuration/", 16 | "https://ai-on-openshift.io/odh-rhoai/custom-notebooks/", 17 | "https://ai-on-openshift.io/odh-rhoai/nvidia-gpus/", 18 | "https://ai-on-openshift.io/odh-rhoai/custom-runtime-triton/", 19 | "https://ai-on-openshift.io/odh-rhoai/openshift-group-management/", 20 | "https://ai-on-openshift.io/tools-and-applications/minio/minio/", 21 | "https://ai-on-openshift.io/odh-rhoai/kueue-preemption/readme/", 22 | "https://ai-on-openshift.io/odh-rhoai/model-serving-type-modification/", 23 | "https://ai-on-openshift.io/odh-rhoai/kserve-uwm-dashboard-metrics/", 24 | "https://ai-on-openshift.io/odh-rhoai/connect-vscode-to-rhoai-wb/", 25 | "https://ai-on-openshift.io/odh-rhoai/enable-function-calling/" 26 | ] 27 | } 28 | ], 29 | "versions": [ 30 | { 31 | "version_number": "2.17", 32 | "store_directive": "update", 33 | "sources": [ 34 | { 35 | "ingestion_type": "redhat_doc", 36 | "language": "en-US" 37 | } 38 | ] 39 | }, 40 | { 41 | "version_number": "2.16", 42 | "store_directive": "update", 43 | "sources": [ 44 | { 45 | "ingestion_type": "redhat_doc", 46 | "language": "en-US" 47 | } 48 | ] 49 | }, 50 | { 51 | "version_number": "2.13", 52 | "store_directive": "update", 53 | "sources": [ 54 | { 55 | "ingestion_type": "redhat_doc", 56 | "language": "en-US" 57 | } 58 | ] 59 | }, 60 | { 61 | "version_number": "2.11", 62 | "store_directive": "delete", 63 | "sources": [ 64 | { 65 | "ingestion_type": "redhat_doc", 66 | "language": "en-US" 67 | } 68 | ] 69 | }, 70 | { 71 | "version_number": "2.10", 72 | "store_directive": "delete", 73 | "sources": [ 74 | { 75 | "ingestion_type": "redhat_doc", 76 | "language": "en-US" 77 | } 78 | ] 79 | }, 80 | { 81 | "version_number": "2.8", 82 | "store_directive": "delete", 83 | "sources": [ 84 | { 85 | "ingestion_type": "redhat_doc", 86 | "language": "en-US" 87 | } 88 | ] 89 | }, 90 | { 91 | "version_number": "2-latest", 92 | "store_directive": "delete", 93 | "sources": [ 94 | { 95 | "ingestion_type": "redhat_doc", 96 | "language": "en-US" 97 | } 98 | ] 99 | } 100 | ] 101 | } 102 | -------------------------------------------------------------------------------- /collections/red_hat_advanced_cluster_security_for_kubernetes.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_advanced_cluster_security_for_kubernetes", 3 | "collection_full_name": "Red Hat Advanced Cluster Security for Kubernetes", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "4.6", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "4.5", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "4.4", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "4.3", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "4.2", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "4.1", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "4.0", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "3.74", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | }, 86 | { 87 | "version_number": "3.73", 88 | "store_directive": "delete", 89 | "sources": [ 90 | { 91 | "ingestion_type": "redhat_doc", 92 | "language": "en-US" 93 | } 94 | ] 95 | }, 96 | { 97 | "version_number": "3.72", 98 | "store_directive": "delete", 99 | "sources": [ 100 | { 101 | "ingestion_type": "redhat_doc", 102 | "language": "en-US" 103 | } 104 | ] 105 | }, 106 | { 107 | "version_number": "3.71", 108 | "store_directive": "delete", 109 | "sources": [ 110 | { 111 | "ingestion_type": "redhat_doc", 112 | "language": "en-US" 113 | } 114 | ] 115 | }, 116 | { 117 | "version_number": "3.70", 118 | "store_directive": "delete", 119 | "sources": [ 120 | { 121 | "ingestion_type": "redhat_doc", 122 | "language": "en-US" 123 | } 124 | ] 125 | }, 126 | { 127 | "version_number": "3.69", 128 | "store_directive": "delete", 129 | "sources": [ 130 | { 131 | "ingestion_type": "redhat_doc", 132 | "language": "en-US" 133 | } 134 | ] 135 | } 136 | ] 137 | } -------------------------------------------------------------------------------- /collections/red_hat_advanced_cluster_management_for_kubernetes.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_advanced_cluster_management_for_kubernetes", 3 | "collection_full_name": "Red Hat Advanced Cluster Management for Kubernetes", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "2.12", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "2.11", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "2.10", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "2.9", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "2.8", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "2.7", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "2.6", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "2.5", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | }, 86 | { 87 | "version_number": "2.4", 88 | "store_directive": "delete", 89 | "sources": [ 90 | { 91 | "ingestion_type": "redhat_doc", 92 | "language": "en-US" 93 | } 94 | ] 95 | }, 96 | { 97 | "version_number": "2.3", 98 | "store_directive": "delete", 99 | "sources": [ 100 | { 101 | "ingestion_type": "redhat_doc", 102 | "language": "en-US" 103 | } 104 | ] 105 | }, 106 | { 107 | "version_number": "2.2", 108 | "store_directive": "delete", 109 | "sources": [ 110 | { 111 | "ingestion_type": "redhat_doc", 112 | "language": "en-US" 113 | } 114 | ] 115 | }, 116 | { 117 | "version_number": "2.1", 118 | "store_directive": "delete", 119 | "sources": [ 120 | { 121 | "ingestion_type": "redhat_doc", 122 | "language": "en-US" 123 | } 124 | ] 125 | }, 126 | { 127 | "version_number": "2.0", 128 | "store_directive": "delete", 129 | "sources": [ 130 | { 131 | "ingestion_type": "redhat_doc", 132 | "language": "en-US" 133 | } 134 | ] 135 | } 136 | ] 137 | } -------------------------------------------------------------------------------- /collections/red_hat_quay.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_quay", 3 | "collection_full_name": "Red Hat Quay", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "3.13", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "3.12", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "3.11", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "3.10", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "3.9", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "3.8", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "3.7", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "3.6", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | }, 86 | { 87 | "version_number": "3.5", 88 | "store_directive": "delete", 89 | "sources": [ 90 | { 91 | "ingestion_type": "redhat_doc", 92 | "language": "en-US" 93 | } 94 | ] 95 | }, 96 | { 97 | "version_number": "3.4", 98 | "store_directive": "delete", 99 | "sources": [ 100 | { 101 | "ingestion_type": "redhat_doc", 102 | "language": "en-US" 103 | } 104 | ] 105 | }, 106 | { 107 | "version_number": "3.3", 108 | "store_directive": "delete", 109 | "sources": [ 110 | { 111 | "ingestion_type": "redhat_doc", 112 | "language": "en-US" 113 | } 114 | ] 115 | }, 116 | { 117 | "version_number": "3.2", 118 | "store_directive": "delete", 119 | "sources": [ 120 | { 121 | "ingestion_type": "redhat_doc", 122 | "language": "en-US" 123 | } 124 | ] 125 | }, 126 | { 127 | "version_number": "3", 128 | "store_directive": "delete", 129 | "sources": [ 130 | { 131 | "ingestion_type": "redhat_doc", 132 | "language": "en-US" 133 | } 134 | ] 135 | }, 136 | { 137 | "version_number": "2.9", 138 | "store_directive": "delete", 139 | "sources": [ 140 | { 141 | "ingestion_type": "redhat_doc", 142 | "language": "en-US" 143 | } 144 | ] 145 | } 146 | ] 147 | } -------------------------------------------------------------------------------- /collections/red_hat_jboss_web_server.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_jboss_web_server", 3 | "collection_full_name": "Red Hat JBoss Web Server", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "6.0", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "5.8", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "5.7", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "5.6", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "5.5", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "5.4", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "5.3", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "5.2", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | }, 86 | { 87 | "version_number": "5.1", 88 | "store_directive": "delete", 89 | "sources": [ 90 | { 91 | "ingestion_type": "redhat_doc", 92 | "language": "en-US" 93 | } 94 | ] 95 | }, 96 | { 97 | "version_number": "5.0", 98 | "store_directive": "delete", 99 | "sources": [ 100 | { 101 | "ingestion_type": "redhat_doc", 102 | "language": "en-US" 103 | } 104 | ] 105 | }, 106 | { 107 | "version_number": "3.1", 108 | "store_directive": "delete", 109 | "sources": [ 110 | { 111 | "ingestion_type": "redhat_doc", 112 | "language": "en-US" 113 | } 114 | ] 115 | }, 116 | { 117 | "version_number": "3", 118 | "store_directive": "delete", 119 | "sources": [ 120 | { 121 | "ingestion_type": "redhat_doc", 122 | "language": "en-US" 123 | } 124 | ] 125 | }, 126 | { 127 | "version_number": "2.1", 128 | "store_directive": "delete", 129 | "sources": [ 130 | { 131 | "ingestion_type": "redhat_doc", 132 | "language": "en-US" 133 | } 134 | ] 135 | }, 136 | { 137 | "version_number": "2", 138 | "store_directive": "delete", 139 | "sources": [ 140 | { 141 | "ingestion_type": "redhat_doc", 142 | "language": "en-US" 143 | } 144 | ] 145 | }, 146 | { 147 | "version_number": "1.0", 148 | "store_directive": "delete", 149 | "sources": [ 150 | { 151 | "ingestion_type": "redhat_doc", 152 | "language": "en-US" 153 | } 154 | ] 155 | } 156 | ] 157 | } -------------------------------------------------------------------------------- /collections/red_hat_decision_manager.json: -------------------------------------------------------------------------------- 1 | { 2 | "collection_base_name": "red_hat_decision_manager", 3 | "collection_full_name": "Red Hat Decision Manager", 4 | "common_sources": [], 5 | "versions": [ 6 | { 7 | "version_number": "7.13", 8 | "store_directive": "create_or_keep", 9 | "sources": [ 10 | { 11 | "ingestion_type": "redhat_doc", 12 | "language": "en-US" 13 | } 14 | ] 15 | }, 16 | { 17 | "version_number": "7.12", 18 | "store_directive": "create_or_keep", 19 | "sources": [ 20 | { 21 | "ingestion_type": "redhat_doc", 22 | "language": "en-US" 23 | } 24 | ] 25 | }, 26 | { 27 | "version_number": "7.11", 28 | "store_directive": "delete", 29 | "sources": [ 30 | { 31 | "ingestion_type": "redhat_doc", 32 | "language": "en-US" 33 | } 34 | ] 35 | }, 36 | { 37 | "version_number": "7.10", 38 | "store_directive": "delete", 39 | "sources": [ 40 | { 41 | "ingestion_type": "redhat_doc", 42 | "language": "en-US" 43 | } 44 | ] 45 | }, 46 | { 47 | "version_number": "7.9", 48 | "store_directive": "delete", 49 | "sources": [ 50 | { 51 | "ingestion_type": "redhat_doc", 52 | "language": "en-US" 53 | } 54 | ] 55 | }, 56 | { 57 | "version_number": "7.8", 58 | "store_directive": "delete", 59 | "sources": [ 60 | { 61 | "ingestion_type": "redhat_doc", 62 | "language": "en-US" 63 | } 64 | ] 65 | }, 66 | { 67 | "version_number": "7.7", 68 | "store_directive": "delete", 69 | "sources": [ 70 | { 71 | "ingestion_type": "redhat_doc", 72 | "language": "en-US" 73 | } 74 | ] 75 | }, 76 | { 77 | "version_number": "7.6", 78 | "store_directive": "delete", 79 | "sources": [ 80 | { 81 | "ingestion_type": "redhat_doc", 82 | "language": "en-US" 83 | } 84 | ] 85 | }, 86 | { 87 | "version_number": "7.5", 88 | "store_directive": "delete", 89 | "sources": [ 90 | { 91 | "ingestion_type": "redhat_doc", 92 | "language": "en-US" 93 | } 94 | ] 95 | }, 96 | { 97 | "version_number": "7.4", 98 | "store_directive": "delete", 99 | "sources": [ 100 | { 101 | "ingestion_type": "redhat_doc", 102 | "language": "en-US" 103 | } 104 | ] 105 | }, 106 | { 107 | "version_number": "7.3", 108 | "store_directive": "delete", 109 | "sources": [ 110 | { 111 | "ingestion_type": "redhat_doc", 112 | "language": "en-US" 113 | } 114 | ] 115 | }, 116 | { 117 | "version_number": "7.2", 118 | "store_directive": "delete", 119 | "sources": [ 120 | { 121 | "ingestion_type": "redhat_doc", 122 | "language": "en-US" 123 | } 124 | ] 125 | }, 126 | { 127 | "version_number": "7.1", 128 | "store_directive": "delete", 129 | "sources": [ 130 | { 131 | "ingestion_type": "redhat_doc", 132 | "language": "en-US" 133 | } 134 | ] 135 | }, 136 | { 137 | "version_number": "7.0", 138 | "store_directive": "delete", 139 | "sources": [ 140 | { 141 | "ingestion_type": "redhat_doc", 142 | "language": "en-US" 143 | } 144 | ] 145 | }, 146 | { 147 | "version_number": "6.4", 148 | "store_directive": "delete", 149 | "sources": [ 150 | { 151 | "ingestion_type": "redhat_doc", 152 | "language": "en-US" 153 | } 154 | ] 155 | } 156 | ] 157 | } --------------------------------------------------------------------------------