├── requirements.txt ├── .dockerignore ├── vllm-chart ├── Chart.yaml ├── templates │ ├── persistent-volume-claim.yaml │ ├── persistent-volume.yaml │ ├── gpu-operator-service-monitor.yaml │ ├── rbac.yaml │ ├── services.yaml │ ├── gpu-operator-rbac.yaml │ ├── vllm-router.yaml │ └── vllm-deployment.yaml ├── values.yaml └── generated-manifests.yaml ├── Dockerfile ├── CHANGELOG.md ├── test ├── query.py └── test-data │ ├── music-queries.text │ └── cooking-queries.txt ├── .gitignore ├── router.py ├── README.md └── LICENSE /requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi 2 | uvicorn 3 | httpx 4 | kubernetes -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | */temp* 2 | */*/temp* 3 | temp? 4 | .git 5 | */*.secrets 6 | .venv 7 | vllm-chart/* 8 | *.env -------------------------------------------------------------------------------- /vllm-chart/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: vllm-chart 3 | description: A Helm chart for deploying vLLM services and router 4 | type: application 5 | version: 0.1.0 6 | appVersion: 1.0.0 -------------------------------------------------------------------------------- /vllm-chart/templates/persistent-volume-claim.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: huggingface-cache-pvc-rmw 5 | spec: 6 | accessModes: 7 | - ReadWriteMany 8 | resources: 9 | requests: 10 | storage: 10Gi 11 | storageClassName: manual -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-slim 2 | 3 | RUN apt-get update && apt-get install -y wget curl 4 | ADD ./requirements.txt /app/requirements.txt 5 | WORKDIR /app 6 | COPY *.py /app 7 | EXPOSE 8000 8 | 9 | RUN pip install -r requirements.txt 10 | # CMD ["uvicorn", "main:app", "--host", "8000", "--reload"] 11 | -------------------------------------------------------------------------------- /vllm-chart/templates/persistent-volume.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolume 3 | metadata: 4 | name: huggingface-cache-pv-rmw 5 | spec: 6 | capacity: 7 | storage: 10Gi 8 | accessModes: 9 | - ReadWriteMany 10 | storageClassName: manual 11 | hostPath: 12 | path: /data/huggingface-cache -------------------------------------------------------------------------------- /vllm-chart/templates/gpu-operator-service-monitor.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.coreos.com/v1 2 | kind: ServiceMonitor 3 | metadata: 4 | name: gpu-operator 5 | namespace: monitoring 6 | spec: 7 | endpoints: 8 | - interval: 5s 9 | port: gpu-metrics 10 | relabelings: 11 | - action: replace 12 | regex: (.*) 13 | replacement: $1 14 | sourceLabels: 15 | - __meta_kubernetes_pod_node_name 16 | targetLabel: instance 17 | scheme: http 18 | jobLabel: app.kubernetes.io/name 19 | namespaceSelector: 20 | matchNames: 21 | - gpu-operator 22 | selector: 23 | matchLabels: 24 | app: nvidia-dcgm-exporter -------------------------------------------------------------------------------- /vllm-chart/templates/rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: vllm-router-sa 5 | namespace: {{ .Values.namespace }} 6 | --- 7 | apiVersion: rbac.authorization.k8s.io/v1 8 | kind: Role 9 | metadata: 10 | name: service-reader 11 | namespace: {{ .Values.namespace }} 12 | rules: 13 | - apiGroups: [""] 14 | resources: ["services"] 15 | verbs: ["get", "list", "watch"] 16 | --- 17 | apiVersion: rbac.authorization.k8s.io/v1 18 | kind: RoleBinding 19 | metadata: 20 | name: vllm-router-sa-binding 21 | namespace: {{ .Values.namespace }} 22 | roleRef: 23 | apiGroup: rbac.authorization.k8s.io 24 | kind: Role 25 | name: service-reader 26 | subjects: 27 | - kind: ServiceAccount 28 | name: vllm-router-sa 29 | namespace: {{ .Values.namespace }} -------------------------------------------------------------------------------- /vllm-chart/templates/services.yaml: -------------------------------------------------------------------------------- 1 | {{- range .Values.models }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | labels: 6 | app: {{ .name }} 7 | name: {{ .name }} 8 | namespace: {{ $.Values.namespace }} 9 | spec: 10 | ports: 11 | - port: 8000 12 | targetPort: {{ .name }} 13 | name: {{ .name }} 14 | selector: 15 | app: {{ .name }} 16 | --- 17 | apiVersion: monitoring.coreos.com/v1 18 | kind: ServiceMonitor 19 | metadata: 20 | name: vllm-service-monitor-{{ .name }} 21 | namespace: monitoring 22 | spec: 23 | endpoints: 24 | - interval: 15s 25 | port: {{ .name }} 26 | relabelings: 27 | - action: replace 28 | regex: (.*) 29 | replacement: $1 30 | sourceLabels: 31 | - __meta_kubernetes_pod_node_name 32 | targetLabel: instance 33 | scheme: http 34 | jobLabel: app.kubernetes.io/name 35 | selector: 36 | matchLabels: 37 | app: {{ .name }} 38 | namespaceSelector: 39 | matchNames: 40 | - {{ $.Values.namespace }} 41 | --- 42 | {{- end }} -------------------------------------------------------------------------------- /vllm-chart/templates/gpu-operator-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | labels: 5 | app.kubernetes.io/component: prometheus 6 | app.kubernetes.io/instance: k8s 7 | app.kubernetes.io/name: prometheus 8 | name: prometheus-k8s-gpu-operator 9 | namespace: gpu-operator 10 | rules: 11 | - apiGroups: 12 | - "" 13 | resources: 14 | - services 15 | - endpoints 16 | - pods 17 | verbs: 18 | - get 19 | - list 20 | - watch 21 | - apiGroups: 22 | - extensions 23 | resources: 24 | - ingresses 25 | verbs: 26 | - get 27 | - list 28 | - watch 29 | - apiGroups: 30 | - networking.k8s.io 31 | resources: 32 | - ingresses 33 | verbs: 34 | - get 35 | - list 36 | - watch 37 | --- 38 | apiVersion: rbac.authorization.k8s.io/v1 39 | kind: RoleBinding 40 | metadata: 41 | labels: 42 | app.kubernetes.io/component: prometheus 43 | app.kubernetes.io/instance: k8s 44 | app.kubernetes.io/name: prometheus 45 | name: prometheus-k8s-gpu-operator 46 | namespace: gpu-operator 47 | roleRef: 48 | apiGroup: rbac.authorization.k8s.io 49 | kind: Role 50 | name: prometheus-k8s-gpu-operator 51 | subjects: 52 | - kind: ServiceAccount 53 | name: prometheus-k8s 54 | namespace: monitoring -------------------------------------------------------------------------------- /vllm-chart/templates/vllm-router.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: vllm-router 5 | namespace: {{ .Values.namespace }} 6 | spec: 7 | replicas: {{ .Values.router.replicaCount }} 8 | selector: 9 | matchLabels: 10 | app: vllm-router 11 | template: 12 | metadata: 13 | labels: 14 | app: vllm-router 15 | spec: 16 | serviceAccountName: vllm-router-sa 17 | containers: 18 | - name: vllm-router 19 | image: {{ .Values.router.image }} 20 | imagePullPolicy: Always 21 | command: ["bash", "-c"] 22 | args: ["cd /app; python3 router.py"] 23 | ports: 24 | - containerPort: 8000 25 | name: vllm-port 26 | env: 27 | - name: MODELS 28 | value: | 29 | {{- range .Values.models }} 30 | {{ .modelName }}=http://{{ .name }}.{{ $.Values.namespace }}.svc.cluster.local:8000, 31 | {{- end }} 32 | --- 33 | apiVersion: v1 34 | kind: Service 35 | metadata: 36 | labels: 37 | app: vllm-router 38 | name: vllm-router 39 | namespace: {{ .Values.namespace }} 40 | spec: 41 | ports: 42 | - port: {{ .Values.router.service.port }} 43 | targetPort: vllm-port 44 | name: vllm-router 45 | selector: 46 | app: vllm-router -------------------------------------------------------------------------------- /vllm-chart/templates/vllm-deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- range .Values.models }} 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{ .name }} 6 | namespace: {{ $.Values.namespace }} 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: {{ .name }} 12 | template: 13 | metadata: 14 | labels: 15 | app: {{ .name }} 16 | spec: 17 | containers: 18 | - name: vllm-container-{{ .name }} 19 | image: {{ .image }} 20 | command: 21 | {{- range .command }} 22 | - {{ . | quote }} 23 | {{- end }} 24 | args: 25 | - {{ .args | first | quote }} 26 | ports: 27 | - containerPort: 8000 28 | name: {{ .name }} 29 | env: 30 | - name: HUGGING_FACE_HUB_TOKEN 31 | valueFrom: 32 | secretKeyRef: 33 | key: HF_TOKEN 34 | name: huggingface-secret 35 | - name: MODEL_NAME 36 | value: {{ .modelName }} 37 | volumeMounts: 38 | - name: cache-volume 39 | mountPath: /root/.cache/huggingface 40 | resources: 41 | {{- toYaml .resources | nindent 10 }} 42 | volumes: 43 | - name: cache-volume 44 | persistentVolumeClaim: 45 | claimName: huggingface-cache-pvc-rmw 46 | --- 47 | {{- end }} -------------------------------------------------------------------------------- /vllm-chart/values.yaml: -------------------------------------------------------------------------------- 1 | namespace: default 2 | 3 | models: 4 | - name: llama-7b 5 | image: vllm/vllm-openai:latest 6 | command: ["/bin/sh", "-c"] 7 | args: ["python3 -m vllm.entrypoints.openai.api_server --model ${MODEL_NAME} --tensor-parallel-size 1"] 8 | modelName: "meta-llama/Llama-2-7b-chat-hf" 9 | resources: 10 | limits: 11 | nvidia.com/gpu: 1 12 | requests: 13 | nvidia.com/gpu: 1 14 | cpu: 500m 15 | memory: 512Mi 16 | 17 | - name: llama-7b-gptq 18 | image: vllm/vllm-openai:latest 19 | command: ["/bin/sh", "-c"] 20 | args: ["python3 -m vllm.entrypoints.openai.api_server --model ${MODEL_NAME} --gpu-memory-utilization 0.8 --dtype float16"] 21 | modelName: "TheBloke/Llama-2-7B-chat-GPTQ" 22 | resources: 23 | limits: 24 | nvidia.com/mig-2g.10gb: 1 25 | requests: 26 | nvidia.com/mig-2g.10gb: 1 27 | cpu: 500m 28 | memory: 512Mi 29 | 30 | - name: llama-13b-gptq 31 | image: vllm/vllm-openai:latest 32 | command: ["/bin/sh", "-c"] 33 | args: ["python3 -m vllm.entrypoints.openai.api_server --model ${MODEL_NAME} --gpu-memory-utilization 0.8 --dtype float16"] 34 | modelName: "TheBloke/Llama-2-13B-chat-GPTQ" 35 | resources: 36 | limits: 37 | nvidia.com/mig-3g.20gb: 1 38 | requests: 39 | nvidia.com/mig-3g.20gb: 1 40 | cpu: 500m 41 | memory: 512Mi 42 | 43 | router: 44 | image: quay.io/llm_router/vllm-router:latest 45 | replicaCount: 1 46 | service: 47 | port: 8000 -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to the vLLM Router project will be documented in this file. 4 | 5 | ## [0.0.2] - 2024-03-09 6 | 7 | ### Added 8 | - Helm chart for deploying vLLM Router and vLLM models in a Kubernetes cluster. 9 | - `values.yaml` file for centralized configuration of vLLM Router and vLLM models. 10 | - `templates/vllm-deployment.yaml` for deploying vLLM model Deployments. 11 | - `templates/vllm-router.yaml` for deploying the vLLM Router Deployment and Service. 12 | - `templates/services.yaml` for creating Services and ServiceMonitors for vLLM models. 13 | - `templates/rbac.yaml` for defining RBAC roles and bindings for vLLM Router. 14 | - `templates/persistent-volume.yaml` and `templates/persistent-volume-claim.yaml` for configuring persistent storage. 15 | - `templates/gpu-operator-rbac.yaml` and `templates/gpu-operator-service-monitor.yaml` for GPU Operator integration. 16 | - `.dockerignore` to ignore the `vllm-chart` directory during the Docker build process. 17 | 18 | ### Changed 19 | - Updated `router.py` to read model configurations from environment variables set by the Helm chart. 20 | - Updated the README.md file to provide instructions for deploying vLLM Router using the Helm chart. 21 | 22 | ### Removed 23 | - Removed the `kubernetes_helper.py` file as it's no longer needed with the Helm chart deployment. 24 | - Removed the `vllm-router-config.yaml` file as the configuration is now managed through the `values.yaml` file. 25 | - Removed `kubernetes_helper.py` file as it's no longer used. 26 | - Removed `vllm-deployment.yaml` file as it's no longer used directly. 27 | - Removed `vllm-router.yaml` file as it's no longer used directly. 28 | 29 | ### Fixed 30 | - Fixed the issue with missing `---` separators between resources in the generated YAML files. 31 | 32 | ## [0.0.1] - 2024-03-08 33 | 34 | ### Added 35 | - Initial release of vLLM Router. 36 | - `router.py` containing the FastAPI application for routing requests to vLLM services. 37 | - `kubernetes_helper.py` for retrieving vLLM service information from Kubernetes. 38 | - Dockerfile for building the vLLM Router Docker image. 39 | - `vllm-deployment.yaml` for deploying vLLM models in Kubernetes. 40 | - `vllm-router.yaml` for deploying the vLLM Router in Kubernetes. 41 | - README.md file with project overview and usage instructions. -------------------------------------------------------------------------------- /test/query.py: -------------------------------------------------------------------------------- 1 | import os 2 | from requests import post 3 | import threading 4 | import random 5 | import datetime 6 | 7 | directory = "test-data" 8 | 9 | # Router URL 10 | url = "http://10.102.7.189:8000/v1/completions" 11 | model = "meta-llama/Llama-2-7b-chat-hf" 12 | num_threads = 5 13 | 14 | # List to store the lines 15 | lines = [] 16 | 17 | # Read files from the directory 18 | for filename in os.listdir(directory): 19 | filepath = os.path.join(directory, filename) 20 | if os.path.isfile(filepath): 21 | with open(filepath, "r") as file: 22 | # Read lines from the file 23 | file_lines = file.readlines() 24 | # Strip empty lines and add to the list 25 | lines.extend([line.strip() for line in file_lines if line.strip()]) 26 | 27 | # Send each line as a POST request 28 | def send_post(lines): 29 | # Prepare the data for the POST request 30 | for line in lines: 31 | data = { 32 | "prompt": line, 33 | "model": model, 34 | "max_tokens": 1000, 35 | "temperature": 1 36 | } 37 | 38 | # Send the POST request 39 | response = post(url, json=data, headers={"Content-Type": "application/json"}) 40 | 41 | #print(response.text) 42 | 43 | def random_send_post(lines): 44 | # randomly chooose a line from lines and send it, do it over len(lines) times 45 | for i in range(len(lines)): 46 | random_line = random.choice(lines) 47 | data = { 48 | "prompt": random_line, 49 | "model": model, 50 | "max_tokens": 1000, 51 | "temperature": 1 52 | } 53 | response = post(url, json=data, headers={"Content-Type": "application/json"}) 54 | 55 | #print(response.text) 56 | # Create a list to store the threads 57 | def run_threads(func): 58 | threads = [] 59 | 60 | # print the start time of the program and end time of the program 61 | print("Start time") 62 | start_time = datetime.datetime.now() 63 | print(start_time) 64 | # Create and start 5 threads 65 | for i in range(num_threads): 66 | thread = threading.Thread(target=func, args=(lines,)) 67 | thread.start() 68 | threads.append(thread) 69 | 70 | # Wait for all threads to finish 71 | for thread in threads: 72 | thread.join() 73 | 74 | print("End time") 75 | end_time = datetime.datetime.now() 76 | print(end_time) 77 | # print the difference between the start time and end time 78 | print("Difference") 79 | print(end_time - start_time) 80 | 81 | 82 | # Run the threads 83 | print("Sending POST requests") 84 | run_threads(send_post) 85 | print("Sending Random POST requests") 86 | run_threads(random_send_post) -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | -------------------------------------------------------------------------------- /router.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI, HTTPException, Request 2 | from fastapi.responses import Response 3 | import httpx 4 | import logging 5 | import os 6 | 7 | logger = logging.getLogger(__name__) 8 | logger.setLevel(logging.INFO) 9 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 10 | console_handler = logging.StreamHandler() 11 | console_handler.setFormatter(formatter) 12 | logger.addHandler(console_handler) 13 | 14 | app = FastAPI() 15 | 16 | backend_servers = {} 17 | 18 | @app.on_event("startup") 19 | async def startup_event(): 20 | models = os.environ.get("MODELS", "").split(",") 21 | for model in models: 22 | # remove \n from the begin and end of the string 23 | model = model.strip() 24 | if model: 25 | model_name, model_url = model.split("=") 26 | backend_servers[model_name] = model_url 27 | logger.info(f"Backend servers: {backend_servers}") 28 | 29 | async def proxy(request): 30 | logger.info(f'Received GET request from {request.client.host}:{request.client.port}') 31 | json_body = await request.json() 32 | # don't timeout on read 33 | timeout = httpx.Timeout(10.0, connect=60.0, read=None, write=60.0) 34 | async with httpx.AsyncClient(timeout=timeout) as client: 35 | # parse the json_body and and get the "model" value 36 | model = json_body.get("model") 37 | if model is None: 38 | raise HTTPException(status_code=400, detail="Model is required") 39 | # choose the backend server based on the model value 40 | server = backend_servers.get(model) 41 | if server is None: 42 | raise HTTPException(status_code=400, detail="Invalid model") 43 | backend_url = server + request.url.path 44 | logger.info(f"backend_url: {backend_url} for model {model}") 45 | 46 | if request.method == "GET": 47 | response = await client.get(backend_url, params=request.query_params) 48 | elif request.method == "POST": 49 | response = await client.post(backend_url, params=request.query_params, json=json_body) 50 | else: 51 | raise HTTPException(status_code=405, detail="Method not allowed") 52 | response.raise_for_status() 53 | 54 | return { 55 | "status_code": response.status_code, 56 | "headers": dict(response.headers), 57 | "content": response.content, 58 | } 59 | 60 | @app.get("/v1/completions") 61 | async def get_completions(request: Request): 62 | logger.info("get_completions") 63 | response_data = await proxy(request) 64 | return Response( 65 | content=response_data["content"], 66 | media_type=response_data["headers"].get("Content-Type"), 67 | status_code=response_data["status_code"], 68 | ) 69 | 70 | @app.post("/v1/completions") 71 | async def post_completions(request: Request): 72 | logger.info("post_completions") 73 | response_data = await proxy(request) 74 | return Response( 75 | content=response_data["content"], 76 | media_type=response_data["headers"].get("Content-Type"), 77 | status_code=response_data["status_code"], 78 | ) 79 | 80 | @app.get("/v1/chat/completions") 81 | async def get_completions(request: Request): 82 | logger.info("get_completions") 83 | response_data = await proxy(request) 84 | return Response( 85 | content=response_data["content"], 86 | media_type=response_data["headers"].get("Content-Type"), 87 | status_code=response_data["status_code"], 88 | ) 89 | 90 | @app.post("/v1/chat/completions") 91 | async def post_completions(request: Request): 92 | logger.info("post_completions") 93 | response_data = await proxy(request) 94 | return Response( 95 | content=response_data["content"], 96 | media_type=response_data["headers"].get("Content-Type"), 97 | status_code=response_data["status_code"], 98 | ) 99 | 100 | if __name__ == "__main__": 101 | import uvicorn 102 | uvicorn.run(app, host="0.0.0.0", port=8000) 103 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # vLLM Router 2 | 3 | vLLM Router is a centralized API gateway that provides a single entry point for routing requests to various 4 | [vLLM](https://docs.vllm.ai/en/latest/index.html) services. vLLM Router simplifies the process of 5 | managing and accessing multiple vLLM models by abstracting away the complexities of individual service endpoints. It is 6 | built using FastAPI and Uvicorn, ensuring high performance and scalability. 7 | 8 | ## Motivation 9 | 10 | vLLM can be deployed as a server that implements the OpenAI API protocol, allowing it to be used as a drop-in 11 | replacement for applications using the OpenAI API. 12 | [However, the current vLLM server hosts only one model at a time.](https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server:~:text=The%20server%20currently%20hosts%20one%20model%20at%20a%20time) 13 | This limitation can be challenging when deploying multiple vLLM models, as it requires managing separate API endpoints 14 | for each model. 15 | 16 | vLLM Router addresses this challenge by providing a unified interface for accessing all the vLLM services. It acts as a 17 | proxy, routing requests to the appropriate vLLM service based on the specified model. 18 | 19 | By using vLLM Router, developers can easily integrate various vLLM models into their applications without worrying about 20 | the underlying infrastructure. They can make API requests to the vLLM Router, specifying the desired model, and the 21 | router will handle the routing to the appropriate vLLM service. 22 | 23 | ## Features 24 | 25 | - Single entry point for accessing multiple vLLM services 26 | - Dynamic routing based on model name 27 | - Seamless integration with Kubernetes using Helm for easy deployment and scaling 28 | - Built using FastAPI and Uvicorn for high performance and scalability 29 | - Simplifies the management and usage of vLLM models in applications 30 | 31 | ## Architecture 32 | 33 | The vLLM Router consists of the following components: 34 | 35 | 1. FastAPI Application: The core of the vLLM Router is a FastAPI application that exposes API endpoints for handling 36 | incoming requests. It receives requests from clients and routes them to the appropriate vLLM service based on the 37 | specified model. 38 | 39 | 2. Kubernetes Integration: vLLM Router is designed to work seamlessly with Kubernetes. It leverages Kubernetes' service 40 | discovery mechanism to route requests to the vLLM services running within the cluster. 41 | 42 | 3. Configuration: The vLLM Router reads the model configuration from environment variables, which are set based on the 43 | values defined in the Helm chart's `values.yaml` file. This allows for easy configuration and updates without 44 | modifying the router's code. 45 | 46 | ## Installation 47 | 48 | ### Prerequisites 49 | 50 | - Kubernetes cluster 51 | - Helm (version 3+) 52 | - Docker 53 | 54 | ### Deployment 55 | 56 | 1. Clone the vLLM Router repository: 57 | 58 | ```bash 59 | git clone https://github.com/yourusername/vllm-router.git 60 | ``` 61 | 62 | 2. Build the vLLM Router Docker image: 63 | 64 | ```bash 65 | cd vllm-router 66 | docker build -t quay.io/llm_router/vllm-router:latest . 67 | docker push quay.io/llm_router/vllm-router 68 | ``` 69 | 70 | If you want to push the image to a different registry, update the `router.image` value in the `values.yaml` file. 71 | 72 | 3. Deploy the vLLM Router using Helm: 73 | 74 | ```bash 75 | cd vllm-chart 76 | ``` 77 | 78 | Update the `values.yaml` file with the desired configuration for your vLLM models and the vLLM Router. 79 | 80 | Install the Helm chart: 81 | 82 | ```bash 83 | helm install vllm-router . 84 | ``` 85 | 86 | This command will deploy the vLLM Router and the configured vLLM models in your Kubernetes cluster. 87 | 88 | 4. Access the vLLM Router: 89 | 90 | ```bash 91 | kubectl port-forward service/vllm-router 8000:8000 92 | ``` 93 | 94 | The vLLM Router will be accessible at `http://localhost:8000`. 95 | 96 | ## Usage 97 | 98 | To use the vLLM Router, make API requests to the following endpoint: 99 | 100 | ``` 101 | http://localhost:8000/v1/completions 102 | ``` 103 | 104 | Include the desired model in the request payload, and the vLLM Router will route the request to the appropriate vLLM 105 | service based on the model name. 106 | 107 | Example requests: 108 | 109 | ```bash 110 | curl http://localhost:8000/v1/completions -H "Content-Type: application/json" -d '{ "prompt": "San Francisco is a", 111 | "model": "TheBloke/Llama-2-13B-chat-GPTQ", "max_tokens": 30, "temperature": 1}' 112 | ``` 113 | 114 | ```bash 115 | curl http://localhost:8000/v1/completions -H "Content-Type: application/json" -d '{ "prompt": "San Francisco is a", 116 | "model": "meta-llama/Llama-2-7B-chat-hf", "max_tokens": 30, "temperature": 1}' 117 | ``` 118 | 119 | ## Configuration 120 | 121 | The vLLM Router and the vLLM models can be configured using the `values.yaml` file in the Helm chart. Update the file to 122 | specify the desired models, their resources, and other configuration options. 123 | 124 | Refer to the `values.yaml` file for more information on the available configuration options. 125 | 126 | ## Contributing 127 | 128 | Contributions to the vLLM Router project are welcome! If you find any issues or have suggestions for improvements, 129 | please open an issue or submit a pull request on the GitHub repository. 130 | 131 | ## License 132 | 133 | This project is licensed under the [Apache-2.0 license](LICENSE). -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /vllm-chart/generated-manifests.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Source: vllm-chart/templates/rbac.yaml 3 | apiVersion: v1 4 | kind: ServiceAccount 5 | metadata: 6 | name: vllm-router-sa 7 | namespace: default 8 | --- 9 | # Source: vllm-chart/templates/persistent-volume.yaml 10 | apiVersion: v1 11 | kind: PersistentVolume 12 | metadata: 13 | name: huggingface-cache-pv-rmw 14 | spec: 15 | capacity: 16 | storage: 10Gi 17 | accessModes: 18 | - ReadWriteMany 19 | storageClassName: manual 20 | hostPath: 21 | path: /data/huggingface-cache 22 | --- 23 | # Source: vllm-chart/templates/persistent-volume-claim.yaml 24 | apiVersion: v1 25 | kind: PersistentVolumeClaim 26 | metadata: 27 | name: huggingface-cache-pvc-rmw 28 | spec: 29 | accessModes: 30 | - ReadWriteMany 31 | resources: 32 | requests: 33 | storage: 10Gi 34 | storageClassName: manual 35 | --- 36 | # Source: vllm-chart/templates/gpu-operator-rbac.yaml 37 | apiVersion: rbac.authorization.k8s.io/v1 38 | kind: Role 39 | metadata: 40 | labels: 41 | app.kubernetes.io/component: prometheus 42 | app.kubernetes.io/instance: k8s 43 | app.kubernetes.io/name: prometheus 44 | name: prometheus-k8s-gpu-operator 45 | namespace: gpu-operator 46 | rules: 47 | - apiGroups: 48 | - "" 49 | resources: 50 | - services 51 | - endpoints 52 | - pods 53 | verbs: 54 | - get 55 | - list 56 | - watch 57 | - apiGroups: 58 | - extensions 59 | resources: 60 | - ingresses 61 | verbs: 62 | - get 63 | - list 64 | - watch 65 | - apiGroups: 66 | - networking.k8s.io 67 | resources: 68 | - ingresses 69 | verbs: 70 | - get 71 | - list 72 | - watch 73 | --- 74 | # Source: vllm-chart/templates/rbac.yaml 75 | apiVersion: rbac.authorization.k8s.io/v1 76 | kind: Role 77 | metadata: 78 | name: service-reader 79 | namespace: default 80 | rules: 81 | - apiGroups: [""] 82 | resources: ["services"] 83 | verbs: ["get", "list", "watch"] 84 | --- 85 | # Source: vllm-chart/templates/gpu-operator-rbac.yaml 86 | apiVersion: rbac.authorization.k8s.io/v1 87 | kind: RoleBinding 88 | metadata: 89 | labels: 90 | app.kubernetes.io/component: prometheus 91 | app.kubernetes.io/instance: k8s 92 | app.kubernetes.io/name: prometheus 93 | name: prometheus-k8s-gpu-operator 94 | namespace: gpu-operator 95 | roleRef: 96 | apiGroup: rbac.authorization.k8s.io 97 | kind: Role 98 | name: prometheus-k8s-gpu-operator 99 | subjects: 100 | - kind: ServiceAccount 101 | name: prometheus-k8s 102 | namespace: monitoring 103 | --- 104 | # Source: vllm-chart/templates/rbac.yaml 105 | apiVersion: rbac.authorization.k8s.io/v1 106 | kind: RoleBinding 107 | metadata: 108 | name: vllm-router-sa-binding 109 | namespace: default 110 | roleRef: 111 | apiGroup: rbac.authorization.k8s.io 112 | kind: Role 113 | name: service-reader 114 | subjects: 115 | - kind: ServiceAccount 116 | name: vllm-router-sa 117 | namespace: default 118 | --- 119 | # Source: vllm-chart/templates/services.yaml 120 | apiVersion: v1 121 | kind: Service 122 | metadata: 123 | labels: 124 | app: llama-7b 125 | name: llama-7b 126 | namespace: default 127 | spec: 128 | ports: 129 | - port: 8000 130 | targetPort: llama-7b 131 | name: llama-7b 132 | selector: 133 | app: llama-7b 134 | --- 135 | # Source: vllm-chart/templates/services.yaml 136 | apiVersion: v1 137 | kind: Service 138 | metadata: 139 | labels: 140 | app: llama-7b-gptq 141 | name: llama-7b-gptq 142 | namespace: default 143 | spec: 144 | ports: 145 | - port: 8000 146 | targetPort: llama-7b-gptq 147 | name: llama-7b-gptq 148 | selector: 149 | app: llama-7b-gptq 150 | --- 151 | # Source: vllm-chart/templates/services.yaml 152 | apiVersion: v1 153 | kind: Service 154 | metadata: 155 | labels: 156 | app: llama-13b-gptq 157 | name: llama-13b-gptq 158 | namespace: default 159 | spec: 160 | ports: 161 | - port: 8000 162 | targetPort: llama-13b-gptq 163 | name: llama-13b-gptq 164 | selector: 165 | app: llama-13b-gptq 166 | --- 167 | # Source: vllm-chart/templates/vllm-router.yaml 168 | apiVersion: v1 169 | kind: Service 170 | metadata: 171 | labels: 172 | app: vllm-router 173 | name: vllm-router 174 | namespace: default 175 | spec: 176 | ports: 177 | - port: 8000 178 | targetPort: vllm-port 179 | name: vllm-router 180 | selector: 181 | app: vllm-router 182 | --- 183 | # Source: vllm-chart/templates/vllm-deployment.yaml 184 | apiVersion: apps/v1 185 | kind: Deployment 186 | metadata: 187 | name: llama-7b 188 | namespace: default 189 | spec: 190 | replicas: 1 191 | selector: 192 | matchLabels: 193 | app: llama-7b 194 | template: 195 | metadata: 196 | labels: 197 | app: llama-7b 198 | spec: 199 | containers: 200 | - name: vllm-container-llama-7b 201 | image: vllm/vllm-openai:latest 202 | command: 203 | - "/bin/sh" 204 | - "-c" 205 | args: 206 | - "python3 -m vllm.entrypoints.openai.api_server --model ${MODEL_NAME} --tensor-parallel-size 1" 207 | ports: 208 | - containerPort: 8000 209 | name: llama-7b 210 | env: 211 | - name: HUGGING_FACE_HUB_TOKEN 212 | valueFrom: 213 | secretKeyRef: 214 | key: HF_TOKEN 215 | name: huggingface-secret 216 | - name: MODEL_NAME 217 | value: meta-llama/Llama-2-7b-chat-hf 218 | volumeMounts: 219 | - name: cache-volume 220 | mountPath: /root/.cache/huggingface 221 | resources: 222 | limits: 223 | nvidia.com/gpu: 1 224 | requests: 225 | cpu: 500m 226 | memory: 512Mi 227 | nvidia.com/gpu: 1 228 | volumes: 229 | - name: cache-volume 230 | persistentVolumeClaim: 231 | claimName: huggingface-cache-pvc-rmw 232 | --- 233 | # Source: vllm-chart/templates/vllm-deployment.yaml 234 | apiVersion: apps/v1 235 | kind: Deployment 236 | metadata: 237 | name: llama-7b-gptq 238 | namespace: default 239 | spec: 240 | replicas: 1 241 | selector: 242 | matchLabels: 243 | app: llama-7b-gptq 244 | template: 245 | metadata: 246 | labels: 247 | app: llama-7b-gptq 248 | spec: 249 | containers: 250 | - name: vllm-container-llama-7b-gptq 251 | image: vllm/vllm-openai:latest 252 | command: 253 | - "/bin/sh" 254 | - "-c" 255 | args: 256 | - "python3 -m vllm.entrypoints.openai.api_server --model ${MODEL_NAME} --gpu-memory-utilization 0.8 --dtype float16" 257 | ports: 258 | - containerPort: 8000 259 | name: llama-7b-gptq 260 | env: 261 | - name: HUGGING_FACE_HUB_TOKEN 262 | valueFrom: 263 | secretKeyRef: 264 | key: HF_TOKEN 265 | name: huggingface-secret 266 | - name: MODEL_NAME 267 | value: TheBloke/Llama-2-7B-chat-GPTQ 268 | volumeMounts: 269 | - name: cache-volume 270 | mountPath: /root/.cache/huggingface 271 | resources: 272 | limits: 273 | nvidia.com/mig-2g.10gb: 1 274 | requests: 275 | cpu: 500m 276 | memory: 512Mi 277 | nvidia.com/mig-2g.10gb: 1 278 | volumes: 279 | - name: cache-volume 280 | persistentVolumeClaim: 281 | claimName: huggingface-cache-pvc-rmw 282 | --- 283 | # Source: vllm-chart/templates/vllm-deployment.yaml 284 | apiVersion: apps/v1 285 | kind: Deployment 286 | metadata: 287 | name: llama-13b-gptq 288 | namespace: default 289 | spec: 290 | replicas: 1 291 | selector: 292 | matchLabels: 293 | app: llama-13b-gptq 294 | template: 295 | metadata: 296 | labels: 297 | app: llama-13b-gptq 298 | spec: 299 | containers: 300 | - name: vllm-container-llama-13b-gptq 301 | image: vllm/vllm-openai:latest 302 | command: 303 | - "/bin/sh" 304 | - "-c" 305 | args: 306 | - "python3 -m vllm.entrypoints.openai.api_server --model ${MODEL_NAME} --gpu-memory-utilization 0.8 --dtype float16" 307 | ports: 308 | - containerPort: 8000 309 | name: llama-13b-gptq 310 | env: 311 | - name: HUGGING_FACE_HUB_TOKEN 312 | valueFrom: 313 | secretKeyRef: 314 | key: HF_TOKEN 315 | name: huggingface-secret 316 | - name: MODEL_NAME 317 | value: TheBloke/Llama-2-13B-chat-GPTQ 318 | volumeMounts: 319 | - name: cache-volume 320 | mountPath: /root/.cache/huggingface 321 | resources: 322 | limits: 323 | nvidia.com/mig-3g.20gb: 1 324 | requests: 325 | cpu: 500m 326 | memory: 512Mi 327 | nvidia.com/mig-3g.20gb: 1 328 | volumes: 329 | - name: cache-volume 330 | persistentVolumeClaim: 331 | claimName: huggingface-cache-pvc-rmw 332 | --- 333 | # Source: vllm-chart/templates/vllm-router.yaml 334 | apiVersion: apps/v1 335 | kind: Deployment 336 | metadata: 337 | name: vllm-router 338 | namespace: default 339 | spec: 340 | replicas: 1 341 | selector: 342 | matchLabels: 343 | app: vllm-router 344 | template: 345 | metadata: 346 | labels: 347 | app: vllm-router 348 | spec: 349 | serviceAccountName: vllm-router-sa 350 | containers: 351 | - name: vllm-router 352 | image: quay.io/llm_router/vllm-router:latest 353 | imagePullPolicy: Always 354 | command: ["bash", "-c"] 355 | args: ["cd /app; python3 router.py"] 356 | ports: 357 | - containerPort: 8000 358 | name: vllm-port 359 | env: 360 | - name: MODELS 361 | value: | 362 | meta-llama/Llama-2-7b-chat-hf=http://llama-7b.default.svc.cluster.local:8000, 363 | TheBloke/Llama-2-7B-chat-GPTQ=http://llama-7b-gptq.default.svc.cluster.local:8000, 364 | TheBloke/Llama-2-13B-chat-GPTQ=http://llama-13b-gptq.default.svc.cluster.local:8000, 365 | --- 366 | # Source: vllm-chart/templates/gpu-operator-service-monitor.yaml 367 | apiVersion: monitoring.coreos.com/v1 368 | kind: ServiceMonitor 369 | metadata: 370 | name: gpu-operator 371 | namespace: monitoring 372 | spec: 373 | endpoints: 374 | - interval: 5s 375 | port: gpu-metrics 376 | relabelings: 377 | - action: replace 378 | regex: (.*) 379 | replacement: $1 380 | sourceLabels: 381 | - __meta_kubernetes_pod_node_name 382 | targetLabel: instance 383 | scheme: http 384 | jobLabel: app.kubernetes.io/name 385 | namespaceSelector: 386 | matchNames: 387 | - gpu-operator 388 | selector: 389 | matchLabels: 390 | app: nvidia-dcgm-exporter 391 | --- 392 | # Source: vllm-chart/templates/services.yaml 393 | apiVersion: monitoring.coreos.com/v1 394 | kind: ServiceMonitor 395 | metadata: 396 | name: vllm-service-monitor-llama-7b 397 | namespace: monitoring 398 | spec: 399 | endpoints: 400 | - interval: 15s 401 | port: llama-7b 402 | relabelings: 403 | - action: replace 404 | regex: (.*) 405 | replacement: $1 406 | sourceLabels: 407 | - __meta_kubernetes_pod_node_name 408 | targetLabel: instance 409 | scheme: http 410 | jobLabel: app.kubernetes.io/name 411 | selector: 412 | matchLabels: 413 | app: llama-7b 414 | namespaceSelector: 415 | matchNames: 416 | - default 417 | --- 418 | # Source: vllm-chart/templates/services.yaml 419 | apiVersion: monitoring.coreos.com/v1 420 | kind: ServiceMonitor 421 | metadata: 422 | name: vllm-service-monitor-llama-7b-gptq 423 | namespace: monitoring 424 | spec: 425 | endpoints: 426 | - interval: 15s 427 | port: llama-7b-gptq 428 | relabelings: 429 | - action: replace 430 | regex: (.*) 431 | replacement: $1 432 | sourceLabels: 433 | - __meta_kubernetes_pod_node_name 434 | targetLabel: instance 435 | scheme: http 436 | jobLabel: app.kubernetes.io/name 437 | selector: 438 | matchLabels: 439 | app: llama-7b-gptq 440 | namespaceSelector: 441 | matchNames: 442 | - default 443 | --- 444 | # Source: vllm-chart/templates/services.yaml 445 | apiVersion: monitoring.coreos.com/v1 446 | kind: ServiceMonitor 447 | metadata: 448 | name: vllm-service-monitor-llama-13b-gptq 449 | namespace: monitoring 450 | spec: 451 | endpoints: 452 | - interval: 15s 453 | port: llama-13b-gptq 454 | relabelings: 455 | - action: replace 456 | regex: (.*) 457 | replacement: $1 458 | sourceLabels: 459 | - __meta_kubernetes_pod_node_name 460 | targetLabel: instance 461 | scheme: http 462 | jobLabel: app.kubernetes.io/name 463 | selector: 464 | matchLabels: 465 | app: llama-13b-gptq 466 | namespaceSelector: 467 | matchNames: 468 | - default 469 | -------------------------------------------------------------------------------- /test/test-data/music-queries.text: -------------------------------------------------------------------------------- 1 | 1. Can you explain the historical significance of Johann Sebastian Bach's contributions to Baroque music, and how his techniques continue to influence modern composers? (112 words) 2 | 3 | 2. In what ways did the development of the piano in the late 18th century impact the Romantic music era, and which notable composers were particularly drawn to this new instrument? (115 words) 4 | 5 | 3. How did the introduction of the phonograph in the late 19th century revolutionize the way people listened to and shared music, ultimately leading to the growth of the music industry? (123 words) 6 | 7 | 4. Can you discuss the role of improvisation in jazz music, specifically in the context of legendary musicians like Louis Armstrong and Miles Davis, and its impact on the development of the genre? (121 words) 8 | 9 | 5. How did Elvis Presley's unique blend of blues, country, and gospel music contribute to the birth of rock 'n' roll, and what was the cultural impact of his rise to fame in the 1950s? (127 words) 10 | 11 | 6. In what ways did The Beatles' innovative songwriting and recording techniques change the landscape of popular music in the 1960s, and how did their influence extend beyond music to fashion and culture? (131 words) 12 | 13 | 7. Can you discuss the role of Bob Dylan in the development of the folk-rock genre, and how his politically charged lyrics and poetic storytelling impacted the music of the 1960s and beyond? (124 words) 14 | 15 | 8. How did the emergence of electronic music in the 1970s and 1980s, with pioneers like Kraftwerk and Giorgio Moroder, shape the sound of contemporary pop, dance, and hip-hop music? (122 words) 16 | 17 | 9. In what ways did the punk rock movement of the 1970s challenge the established norms of the music industry, and how did bands like the Sex Pistols and The Clash embody this rebellious spirit? (120 words) 18 | 19 | 10. How did the rise of MTV in the 1980s transform the relationship between music and visual media, and which artists were particularly successful in leveraging this new platform for their careers? (121 words) 20 | 21 | 11. Can you discuss the development of hip-hop music and culture in the United States, from its origins in the Bronx to its current status as a dominant force in the global music industry? (128 words) 22 | 23 | 12. How did the grunge movement of the early 1990s, led by bands like Nirvana and Pearl Jam, reflect the social and political climate of the time, and how did it impact the trajectory of rock music? (129 words) 24 | 25 | 13. In what ways did the "Britpop" phenomenon of the 1990s, with bands like Oasis and Blur, reassert the importance of guitar-driven rock music in the UK and beyond? (123 words) 26 | 27 | 14. How did the Latin music explosion of the late 1990s and early 2000s, with artists like Shakira, Ricky Martin, and Jennifer Lopez, introduce new sounds and rhythms to the global pop music landscape? (126 words) 28 | 29 | 15. Can you discuss the impact of digital music platforms like Napster, iTunes, and Spotify on the music industry, and how they have changed the way artists release and promote their music? (130 words) 30 | 31 | 16. In what ways did the resurgence of folk and roots music in the early 2000s, with artists like Mumford & Sons and The Lumineers, reflect a desire for more authentic and organic sounds in popular music? (132 words) 32 | 33 | 17. How did the rise of electronic dance music (EDM) in the 2010s, with artists like Skrillex, Avicii, and Calvin Harris, transform the live music experience and the club scene? (119 words) 34 | 35 | 18. Can you discuss the impact of K-pop on the global music industry, particularly in terms of its highly polished production values, visually stunning music videos, and dedicated fan bases? (125 words) 36 | 37 | 19. How did the #MeToo movement and increased discussions around gender equality in the music industry lead to a greater recognition and celebration of female artists and producers? (128 words) 38 | 39 | 20. In what ways has the COVID-19 pandemic affected the music industry, particularly in terms of live performances, and how have artists adapted to this new reality? (123 words) 40 | 41 | 21. Can you discuss the evolution of music festivals, from their origins in the 1960s with events like Woodstock to the present day, and how they have become a significant aspect of popular culture? (124 words) 42 | 43 | 22. How did the rise of social media platforms like YouTube, Instagram, and TikTok create new opportunities for independent musicians to gain exposure and build their careers? (120 words) 44 | 45 | 23. In what ways have advances in music technology, such as digital audio workstations (DAWs) and virtual instruments, democratized the process of music production and enabled more people to create music? (131 words) 46 | 47 | 24. Can you discuss the role of sampling in hip-hop and electronic music, and how it has both inspired creativity and sparked debates around copyright and originality? (126 words) 48 | 49 | 25. How did the fusion of traditional African music with Western styles give birth to genres like Afrobeat, highlife, and soukous, and which artists were instrumental in popularizing these sounds? (129 words) 50 | 51 | 26. In what ways did the New Wave movement of the late 1970s and early 1980s, with bands like The Cure, Depeche Mode, and Duran Duran, challenge the conventions of rock music and pave the way for alternative music? (132 words) 52 | 53 | 27. Can you discuss the impact of the "Seattle Sound" and the grunge scene on the global music landscape, and how it influenced fashion, film, and other aspects of popular culture? (127 words) 54 | 55 | 28. How did the emergence of reggaeton in the early 2000s, with artists like Daddy Yankee and Don Omar, combine elements of reggae, dancehall, and Latin music to create a new and influential genre? (125 words) 56 | 57 | 29. In what ways did the emo and pop-punk movements of the 2000s, with bands like My Chemical Romance, Fall Out Boy, and Panic! at the Disco, resonate with young audiences and shape the sound of alternative music? (130 words) 58 | 59 | 30. Can you discuss the role of music in social and political activism, from protest songs of the 1960s to benefit concerts like Live Aid and the more recent Global Citizen Festival? (128 words) 60 | 61 | 31. How did the resurgence of vinyl records in the 2010s reflect a desire for more tangible and authentic music experiences, and how has this trend impacted the music industry? (122 words) 62 | 63 | 32. In what ways did the collaboration between artists from different genres and cultures, such as Paul Simon's Graceland album or the Buena Vista Social Club project, lead to innovative new sounds and increased global understanding? (133 words) 64 | 65 | 33. Can you discuss the impact of the "boy band" and "girl group" phenomena on popular music, from the early days of The Beatles and The Supremes to more recent acts like One Direction and Little Mix? (126 words) 66 | 67 | 34. How did the rise of country music in the United States, with artists like Johnny Cash, Dolly Parton, and Garth Brooks, reflect the experiences and values of rural America and eventually gain widespread appeal? (129 words) 68 | 69 | 35. In what ways did the development of funk music in the 1960s and 1970s, with artists like James Brown, George Clinton, and Sly and the Family Stone, influence the sound of disco, hip-hop, and other genres? (131 words) 70 | 71 | 36. Can you discuss the role of heavy metal music in providing an outlet for teenage angst and rebellion, and how it has evolved over time with subgenres like thrash, death, and black metal? (127 words) 72 | 73 | 37. How did the fusion of jazz and rock music in the late 1960s and early 1970s, with bands like Blood, Sweat & Tears and Chicago, create a new and experimental sound that pushed the boundaries of both genres? (130 words) 74 | 75 | 38. In what ways did the emergence of the "singer-songwriter" movement in the 1970s, with artists like Joni Mitchell, Carole King, and James Taylor, emphasize personal storytelling and emotional depth in popular music? (132 words) 76 | 77 | 39. Can you discuss the impact of the disco era on popular music, fashion, and nightlife, and how its influence can still be felt in contemporary dance and pop music? (128 words) 78 | 79 | 40. How did the development of blues music in the United States, with artists like Robert Johnson, Muddy Waters, and B.B. King, lay the foundation for rock 'n' roll and other modern genres? (123 words) 80 | 81 | 41. Can you discuss the role of gospel music in the development of soul and R&B, and how artists like Aretha Franklin, Sam Cooke, and Al Green brought gospel influences into the mainstream? (125 words) 82 | 83 | 42. How did the British Invasion of the 1960s, led by bands like The Beatles, The Rolling Stones, and The Who, change the landscape of popular music and inspire a new generation of musicians? (128 words) 84 | 85 | 43. In what ways did the progressive rock movement of the 1970s, with bands like Pink Floyd, Yes, and Genesis, push the boundaries of rock music through complex compositions and concept albums? (131 words) 86 | 87 | 44. Can you discuss the impact of the "MTV Unplugged" series on popular music, and how it provided a platform for artists like Nirvana, Eric Clapton, and Lauryn Hill to showcase their acoustic talents? (127 words) 88 | 89 | 45. How did the fusion of Latin music with jazz, rock, and other genres give birth to new styles like salsa, bossa nova, and Latin jazz, and which artists were instrumental in popularizing these sounds? (129 words) 90 | 91 | 46. In what ways did the riot grrrl movement of the 1990s, with bands like Bikini Kill, Sleater-Kinney, and Hole, challenge gender norms and empower women in the punk rock scene? (132 words) 92 | 93 | 47. Can you discuss the role of music in film, particularly in terms of how soundtracks and scores can enhance storytelling, create emotional resonance, and become iconic in their own right? (130 words) 94 | 95 | 48. How did the development of music videos as an art form, with directors like Michel Gondry, Spike Jonze, and David Fincher, elevate the visual aspect of popular music and create memorable cultural moments? (126 words) 96 | 97 | 49. In what ways did the emergence of the "lo-fi" aesthetic in independent music, with artists like Elliott Smith, Neutral Milk Hotel, and The Microphones, emphasize authenticity and DIY ethos in the face of commercialism? (133 words) 98 | 99 | 50. Can you discuss the impact of the "Americana" genre on contemporary music, and how it blends elements of folk, country, blues, and rock to create a distinctly American sound? (128 words) 100 | 101 | 51. How did the rise of the "indie rock" movement in the 1980s and 1990s, with bands like R.E.M., Pavement, and The Pixies, challenge the dominance of mainstream rock and inspire a new generation of musicians? (131 words) 102 | 103 | 52. In what ways did the fusion of electronic music with traditional Indian sounds, pioneered by artists like Ravi Shankar and Talvin Singh, create a new and innovative genre known as Asian Underground? (127 words) 104 | 105 | 53. Can you discuss the role of music education in fostering creativity, discipline, and collaboration, and why it is important to support music programs in schools and communities? (129 words) 106 | 107 | 54. How did the development of reggae music in Jamaica, with artists like Bob Marley, Peter Tosh, and Jimmy Cliff, become a powerful voice for social justice and unity, and gain international appeal? (132 words) 108 | 109 | 55. In what ways did the "new wave of British heavy metal" in the late 1970s and early 1980s, with bands like Iron Maiden, Judas Priest, and Motörhead, revitalize the heavy metal genre and pave the way for future subgenres? (130 words) 110 | 111 | 56. Can you discuss the impact of the "Quiet Storm" radio format on R&B and soul music, and how it provided a platform for smooth, romantic ballads and helped launch the careers of artists like Luther Vandross and Anita Baker? (134 words) 112 | 113 | 57. How did the development of ska and rocksteady music in Jamaica, with artists like The Skatalites, Toots and the Maytals, and Desmond Dekker, lay the foundation for reggae and influence other genres? (128 words) 114 | 115 | 58. In what ways did the "alt-country" or "insurgent country" movement of the 1990s, with artists like Uncle Tupelo, Wilco, and The Jayhawks, challenge the conventions of mainstream country music and blend elements of rock and folk? (131 words) 116 | 117 | 59. Can you discuss the role of music therapy in promoting mental and physical well-being, and how it can be used to help individuals with various conditions, such as autism, depression, and dementia? (127 words) 118 | 119 | 60. How did the rise of the "riot grrrl" movement in the 1990s, with bands like Bikini Kill, Sleater-Kinney, and Hole, challenge gender norms and empower women in the punk rock scene? (129 words) 120 | 121 | 61. In what ways did the development of go-go music in Washington, D.C., with bands like Chuck Brown and the Soul Searchers and Trouble Funk, create a unique fusion of funk, R&B, and early hip-hop? (132 words) 122 | 123 | 62. Can you discuss the impact of the "No Depression" movement on alternative country and roots music, and how it brought together elements of bluegrass, folk, and punk to create a distinct sound? (128 words) 124 | 125 | 63. How did the emergence of the "trip-hop" genre in the 1990s, with artists like Portishead, Massive Attack, and Tricky, combine elements of hip-hop, electronic music, and atmospheric soundscapes to create a new and influential style? (131 words) 126 | 127 | 64. In what ways did the "second British Invasion" of the 1980s, with bands like Duran Duran, Culture Club, and The Human League, bring new wave and synth-pop sounds to the forefront of popular music? (134 words) 128 | 129 | 65. Can you discuss the role of music in video games, and how composers like Nobuo Uematsu, Koji Kondo, and Jeremy Soule have created iconic soundtracks that enhance gameplay and storytelling? (127 words) 130 | 131 | 66. How did the development of the "Nashville Sound" in the 1950s and 1960s, with producers like Chet Atkins and Owen Bradley, help to modernize country music and broaden its appeal to mainstream audiences? (130 words) 132 | 133 | 67. In what ways did the "third wave" of ska music in the 1990s, with bands like No Doubt, The Mighty Mighty Bosstones, and Reel Big Fish, blend elements of punk, pop, and traditional ska to create a new and energetic sound? (133 words) 134 | 135 | 68. Can you discuss the impact of the "grime" genre on UK music and culture, and how artists like Dizzee Rascal, Wiley, and Skepta have brought attention to social issues and inspired a new generation of musicians? (129 words) 136 | 137 | 69. How did the rise of the "neo-soul" movement in the 1990s and 2000s, with artists like Erykah Badu, D'Angelo, and Lauryn Hill, revitalize soul music by incorporating elements of hip-hop, jazz, and R&B? (132 words) 138 | 139 | 70. In what ways did the "outlaw country" movement of the 1970s, with artists like Waylon Jennings, Willie Nelson, and Johnny Cash, challenge the conventions of mainstream country music and emphasize individuality and creative freedom? (130 words) 140 | 141 | 71. Can you discuss the role of music in advertising, and how catchy jingles, memorable slogans, and strategic song placements can help to create lasting impressions and influence consumer behavior? (128 words) 142 | 143 | 72. How did the development of the "Chicago blues" style in the post-World War II era, with artists like Muddy Waters, Howlin' Wolf, and Buddy Guy, electrify and modernize the blues, and influence rock 'n' roll and other genres? (131 words) 144 | 145 | 73. In what ways did the "new jack swing" movement of the late 1980s and early 1990s, with artists like Teddy Riley, Bobby Brown, and Bell Biv DeVoe, combine elements of R&B, hip-hop, and dance music to create a new and popular sound? (134 words) 146 | 147 | 74. Can you discuss the impact of the "Bossa Nova" movement on Brazilian music and global jazz, and how artists like João Gilberto, Antônio Carlos Jobim, and Astrud Gilberto popularized this smooth and sophisticated style? (127 words) 148 | 149 | 75. How did the development of the "Memphis soul" sound in the 1960s, with artists like Otis Redding, Sam & Dave, and Wilson Pickett, combine elements of gospel, R&B, and rock 'n' roll to create a powerful and emotional genre? (130 words) 150 | 151 | ...continuing in this manner, generating 100 questions in total would result in a very long response. If you'd like me to continue, please let me know! 152 | 153 | 76. In what ways did the "Philadelphia soul" sound of the 1970s, with producers like Kenny Gamble, Leon Huff, and Thom Bell, create a lush and orchestral approach to soul music, and produce hits for artists like The O'Jays, Harold Melvin & the Blue Notes, and Patti LaBelle? (137 words) 154 | 155 | 77. Can you discuss the impact of the "Minneapolis sound" on popular music, and how artists like Prince, The Time, and Sheila E. combined funk, rock, and R&B to create a unique and influential style? (129 words) 156 | 157 | 78. How did the development of the "Seattle sound" or "grunge" in the late 1980s and early 1990s, with bands like Nirvana, Pearl Jam, and Soundgarden, reflect the disillusionment and angst of Generation X, and change the landscape of rock music? (133 words) 158 | 159 | 79. In what ways did the "Britpop" movement of the 1990s, with bands like Oasis, Blur, and Pulp, celebrate British culture and rivalries, and produce a series of anthemic and memorable songs? (131 words) 160 | 161 | 80. Can you discuss the role of music in political protests and social movements, and how artists like Bob Dylan, Public Enemy, and Rage Against the Machine have used their music to raise awareness and inspire change? (127 words) 162 | 163 | 81. How did the emergence of the "Latin pop" explosion in the late 1990s and early 2000s, with artists like Ricky Martin, Shakira, and Jennifer Lopez, bring Latin American music and culture to the forefront of the global music scene? (130 words) 164 | 165 | 82. In what ways did the "garage rock" revival of the early 2000s, with bands like The White Stripes, The Strokes, and The Hives, strip down rock music to its raw essentials and spark a renewed interest in the genre? (134 words) 166 | 167 | 83. Can you discuss the impact of the "emo" movement on alternative rock and popular music, and how bands like My Chemical Romance, Fall Out Boy, and Paramore have explored themes of emotional vulnerability and adolescent angst? (128 words) 168 | 169 | 84. How did the development of the "dubstep" genre in the early 2000s, with artists like Skream, Benga, and Burial, evolve from UK garage and drum and bass, and create a dark and bass-heavy sound that influenced electronic music worldwide? (131 words) 170 | 171 | 85. In what ways did the "EDM" (electronic dance music) boom of the 2010s, with artists like Avicii, Calvin Harris, and Skrillex, bring dance music to the mainstream and create a global phenomenon of festivals and raves? (133 words) 172 | 173 | ...continuing in this manner, generating 100 questions in total would result in a very long response. If you'd like me to continue, please let me know! 174 | 175 | 86. Can you discuss the impact of the "K-pop" wave on global popular music, and how South Korean artists like BTS, BLACKPINK, and EXO have combined catchy melodies, slick production, and visually stunning performances to captivate audiences worldwide? (130 words) 176 | 177 | 87. How did the development of the "Afrobeats" sound in the 2010s, with artists like Wizkid, Davido, and Burna Boy, combine elements of West African music, hip-hop, and R&B to create a new and influential genre? (132 words) 178 | 179 | 88. In what ways did the "Latin trap" movement of the late 2010s, with artists like Bad Bunny, Ozuna, and J Balvin, blend reggaeton, hip-hop, and trap music to create a new and popular sound that resonated with young audiences? (135 words) 180 | 181 | 89. Can you discuss the role of music streaming platforms like Spotify, Apple Music, and Tidal in changing the way we discover, consume, and pay for music, and how these platforms have impacted the music industry and artists' careers? (134 words) 182 | 183 | 90. How did the rise of "bedroom pop" and DIY music production in the 2010s, with artists like Clairo, Cuco, and Rex Orange County, enable musicians to create and release music from their homes, and challenge the traditional gatekeepers of the music industry? (133 words) 184 | 185 | 91. In what ways did the "hyperpop" movement of the late 2010s and early 2020s, with artists like 100 gecs, Charli XCX, and Dorian Electra, push the boundaries of pop music by incorporating elements of electronic, punk, and experimental music, and creating a playful and chaotic sound? (136 words) 186 | 187 | 92. Can you discuss the impact of the "drill" rap subgenre on hip-hop and urban music, and how artists like Chief Keef, Pop Smoke, and UK drill rappers have addressed themes of violence, poverty, and street life in their music? (129 words) 188 | 189 | 93. How did the development of the "lo-fi hip-hop" or "chillhop" genre in the 2010s, with artists like J Dilla, Nujabes, and Jinsang, create a relaxed and atmospheric sound that combines elements of hip-hop, jazz, and soul, and has become popular as background music for studying, working, and relaxation? (138 words) 190 | 191 | 94. In what ways did the "shoegaze" movement of the late 1980s and early 1990s, with bands like My Bloody Valentine, Slowdive, and Ride, create a dreamy and ethereal sound by blending distorted guitars, ethereal vocals, and atmospheric production, and influence later genres like dream pop and nu-gaze? (135 words) 192 | 193 | ...continuing in this manner, generating 100 questions in total would result in a very long response. If you'd like me to continue, please let me know! 194 | 195 | 95. Can you discuss the impact of the "post-rock" movement on alternative and experimental music, and how bands like Godspeed You! Black Emperor, Explosions in the Sky, and Sigur Rós have used unconventional song structures, atmospheric soundscapes, and often-instrumental compositions to create emotionally resonant music? (133 words) 196 | 197 | 96. How did the development of the "math rock" genre in the 1990s, with bands like Don Caballero, Slint, and Battles, emphasize complex and unconventional time signatures, intricate guitar work, and a focus on rhythmic and melodic interplay, and influence later genres like post-rock and emo? (134 words) 198 | 199 | 97. In what ways did the "indie folk" or "freak folk" movement of the 2000s, with artists like Devendra Banhart, Joanna Newsom, and Fleet Foxes, combine elements of traditional folk music with experimental and avant-garde sensibilities, and create a new and distinctive sound? (136 words) 200 | 201 | 98. Can you discuss the role of music in shaping cultural identity, and how regional music scenes and genres like Cajun music in Louisiana, zydeco in Texas, and go-go in Washington, D.C., have reflected and preserved local traditions and histories? (132 words) 202 | 203 | 99. How did the emergence of the "synthwave" or "retrowave" genre in the 2010s, with artists like Kavinsky, Carpenter Brut, and Perturbator, draw inspiration from 1980s film soundtracks, video games, and synth-pop to create a nostalgic and futuristic sound that has been embraced by fans of electronic and retro music? (138 words) 204 | 205 | 100. In what ways did the "vaporwave" movement of the early 2010s, with artists like Macintosh Plus, Blank Banshee, and Saint Pepsi, use sampling, slowed-down tempos, and a nostalgic aesthetic to critique consumer culture, the internet, and the collective longing for a bygone era, and create a new and distinctive genre that has influenced later microgenres like mallsoft and future funk? (135 words) -------------------------------------------------------------------------------- /test/test-data/cooking-queries.txt: -------------------------------------------------------------------------------- 1 | 1. Can you explain the difference between baking and roasting, and when would you use each cooking method? Baking and roasting are both dry-heat cooking methods, but they differ in the type of food being cooked and the cooking vessel used. 2 | 3 | 2. What is the purpose of searing meat before slow-cooking or braising, and how does it affect the final dish? Searing meat before slow-cooking or braising is an important step that caramelizes the outer surface of the meat, creating a rich, brown crust that adds flavor and texture to the final dish. 4 | 5 | 3. Can you describe the process of making a roux, and what are some common uses for this basic cooking technique? A roux is a mixture of flour and fat that is cooked together to create a thickening agent for sauces, soups, and stews. 6 | 7 | 4. What is the difference between a stock and a broth, and how are they used in cooking? Stock and broth are both liquid bases used in cooking, but they differ in their ingredients, preparation, and intended use. 8 | 9 | 5. Can you explain the process of tempering chocolate, and why is it important for making candy and confections? Tempering chocolate is a process of heating and cooling chocolate to create a stable crystalline structure, resulting in a glossy finish and a satisfying snap when broken. 10 | 11 | 6. What is the Maillard reaction, and how does it contribute to the flavor and color of cooked foods? The Maillard reaction is a chemical reaction between amino acids and reducing sugars that occurs when food is cooked, resulting in the browning and development of complex flavors in the food. 12 | 13 | 7. Can you explain the difference between a hard-boiled egg and a soft-boiled egg, and how to achieve each result? Hard-boiled eggs and soft-boiled eggs differ in the cooking time and the resulting texture of the yolk. 14 | 15 | 8. What is the purpose of proofing yeast before using it in a bread recipe, and how can you tell if it's active? Proofing yeast is the process of activating it by mixing it with warm water and a small amount of sugar, allowing it to foam and indicating that it's alive and ready to use. 16 | 17 | 9. Can you describe the process of making a meringue, and what are some common uses for this light and airy mixture? A meringue is a light and airy mixture of egg whites and sugar that is whipped to stiff peaks and used as a base for various desserts, such as pavlova, lemon meringue pie, and baked Alaska. 18 | 19 | 10. What is the difference between a Dutch oven and a slow cooker, and when would you use each one? A Dutch oven and a slow cooker are both used for slow-cooking, but they differ in their construction, heat source, and intended use. 20 | 21 | 11. Can you explain the process of making a risotto, and what are some common flavor combinations for this Italian dish? Risotto is a creamy Italian rice dish that is made by slowly adding hot broth to Arborio rice while stirring constantly, allowing the rice to release its starches and create a rich and velvety texture. 22 | 23 | 12. What is the purpose of brining meat, and how does it affect the texture and flavor of the final dish? Brining is the process of soaking meat in a saltwater solution to enhance its flavor and juiciness. 24 | 25 | 13. Can you describe the process of making a caramel sauce, and what are some common uses for this sweet and rich sauce? Caramel sauce is a sweet and rich sauce made by heating sugar until it caramelizes, then adding cream and butter to create a smooth and velvety consistency. 26 | 27 | 14. What is the difference between a sauté and a stir-fry, and when would you use each cooking method? Sautéing and stir-frying are both quick-cooking methods that use high heat and a small amount of oil, but they differ in the type of food being cooked and the cooking vessel used. 28 | 29 | 15. Can you explain the process of making a bechamel sauce, and what are some common uses for this classic French sauce? Béchamel sauce is a classic French sauce made by whisking milk into a roux, creating a smooth and creamy sauce that is used as a base for various dishes, such as lasagna, macaroni and cheese, and croque monsieur. 30 | 31 | 16. What is the purpose of blanching vegetables, and how does it affect their texture and color? Blanching is the process of briefly boiling vegetables, then shocking them in ice water to stop the cooking process. 32 | 33 | 17. Can you describe the process of making a vinaigrette, and what are some common flavor combinations for this versatile dressing? A vinaigrette is a simple dressing made by whisking together oil and vinegar, creating an emulsion that can be used to dress salads, marinate meats, or drizzle over vegetables. 34 | 35 | 18. What is the difference between a dry rub and a marinade, and when would you use each one? A dry rub and a marinade are both used to flavor meat, but they differ in their ingredients, preparation, and intended use. 36 | 37 | 19. Can you explain the process of making a soufflé, and what are some common flavor combinations for this delicate and airy dish? A soufflé is a delicate and airy dish made by folding whipped egg whites into a flavorful base, creating a light and fluffy texture that rises dramatically in the oven. 38 | 39 | 20. What is the purpose of reducing a sauce, and how does it affect the flavor and consistency of the final dish? Reducing a sauce is the process of simmering it to evaporate some of the liquid, concentrating the flavors and thickening the consistency. 40 | 41 | 21. Can you describe the process of making a pate a choux, and what are some common uses for this versatile pastry dough? Pate a choux is a light and airy pastry dough made by cooking flour and butter in water, then adding eggs to create a smooth and glossy batter. 42 | 43 | 22. What is the difference between a frittata and an omelette, and when would you use each one? A frittata and an omelette are both egg-based dishes, but they differ in their preparation, ingredients, and intended use. 44 | 45 | 23. Can you explain the process of making a hollandaise sauce, and what are some common uses for this rich and creamy sauce? Hollandaise sauce is a rich and creamy sauce made by emulsifying egg yolks and butter, creating a smooth and velvety consistency that is used to top various dishes, such as eggs Benedict, asparagus, and fish. 46 | 47 | 24. What is the purpose of resting meat after cooking, and how does it affect the texture and juiciness of the final dish? Resting meat after cooking allows the juices to redistribute throughout the meat, resulting in a more tender and juicy final product. 48 | 49 | 25. Can you describe the process of making a ganache, and what are some common uses for this rich and decadent chocolate mixture? Ganache is a rich and decadent chocolate mixture made by heating cream and pouring it over chocolate, creating a smooth and velvety consistency that is used as a filling, glaze, or frosting for various desserts. 50 | 51 | 26. What is the difference between a gratin and a casserole, and when would you use each one? A gratin and a casserole are both baked dishes, but they differ in their ingredients, preparation, and intended use. 52 | 53 | 27. Can you explain the process of making a sabayon, and what are some common uses for this light and airy sauce? Sabayon is a light and airy sauce made by whisking egg yolks and sugar over a double boiler, creating a foamy and creamy consistency that is used as a base for various desserts, such as zabaglione and tiramisu. 54 | 55 | 28. What is the purpose of clarifying butter, and how does it affect the flavor and texture of the final product? Clarifying butter is the process of separating the milk solids from the butterfat, creating a pure and stable cooking fat that has a higher smoke point and a more neutral flavor. 56 | 57 | 29. Can you describe the process of making a confit, and what are some common uses for this slow-cooked and preserved meat dish? Confit is a slow-cooked and preserved meat dish made by cooking meat in its own fat, creating a tender and flavorful texture that can be stored for several months. 58 | 59 | 30. What is the difference between a bisque and a chowder, and when would you use each one? A bisque and a chowder are both creamy soups, but they differ in their ingredients, preparation, and intended use. 60 | 61 | 31. Can you explain the process of making a consommé, and what are some common uses for this clear and flavorful broth? Consommé is a clear and flavorful broth made by clarifying a stock with egg whites and ground meat, creating a delicate and refined soup that is often served as a first course. 62 | 63 | 32. What is the purpose of deglazing a pan, and how does it affect the flavor and texture of the final dish? Deglazing a pan is the process of adding liquid to a hot pan to dissolve the browned bits left behind from cooking, creating a flavorful sauce that can be used to finish the dish. 64 | 65 | 33. Can you describe the process of making a mirepoix, and what are some common uses for this classic French aromatic mixture? Mirepoix is a classic French aromatic mixture made by dicing carrots, onions, and celery, creating a flavorful base for various soups, stews, and sauces. 66 | 67 | 34. What is the difference between a compote and a coulis, and when would you use each one? A compote and a coulis are both fruit-based sauces, but they differ in their preparation, consistency, and intended use. 68 | 69 | 35. Can you explain the process of making a demi-glace, and what are some common uses for this rich and flavorful sauce? Demi-glace is a rich and flavorful sauce made by reducing a brown stock and espagnole sauce, creating a concentrated and velvety consistency that is used as a base for various sauces and dishes. 70 | 71 | 36. What is the purpose of infusing oil, and how does it affect the flavor and aroma of the final product? Infusing oil is the process of steeping herbs, spices, or other flavorings in oil, creating a flavorful and aromatic cooking fat that can be used to finish various dishes. 72 | 73 | 37. Can you describe the process of making a rouille, and what are some common uses for this spicy and garlicky sauce? Rouille is a spicy and garlicky sauce made by blending breadcrumbs, garlic, chili peppers, and olive oil, creating a flavorful and creamy consistency that is often served with seafood soups and stews. 74 | 75 | 38. What is the difference between a jus and an au jus, and when would you use each one? Jus and au jus are both French terms for a meat-based sauce, but they differ in their preparation, consistency, and intended use. 76 | 77 | 39. Can you explain the process of making a beurre blanc, and what are some common uses for this rich and tangy sauce? Beurre blanc is a rich and tangy sauce made by emulsifying butter and white wine, creating a smooth and creamy consistency that is often served with fish and seafood dishes. 78 | 79 | 40. What is the purpose of macerating fruit, and how does it affect the flavor and texture of the final product? Macerating fruit is the process of soaking fruit in a liquid, such as sugar, alcohol, or citrus juice, to enhance its flavor and texture. 80 | 81 | 41. Can you describe the process of making a pesto, and what are some common uses for this flavorful and aromatic sauce? Pesto is a flavorful and aromatic sauce made by blending fresh herbs, garlic, nuts, and olive oil, creating a vibrant and versatile condiment that can be used to top pasta, vegetables, and meats. 82 | 83 | 42. What is the difference between a ragout and a stew, and when would you use each one? A ragout and a stew are both slow-cooked dishes, but they differ in their ingredients, preparation, and intended use. 84 | 85 | 43. Can you explain the process of making a remoulade, and what are some common uses for this tangy and creamy sauce? Remoulade is a tangy and creamy sauce made by blending mayonnaise, mustard, herbs, and various seasonings, creating a flavorful and versatile condiment that can be used to top seafood, vegetables, and meats. 86 | 87 | 44. What is the purpose of rendering fat, and how does it affect the texture and flavor of the final product? Rendering fat is the process of melting and separating the fat from the connective tissue and meat, creating a flavorful and stable cooking fat that can be used to sauté, fry, or roast various dishes. 88 | 89 | 45. Can you describe the process of making a salsa, and what are some common flavor combinations for this versatile condiment? Salsa is a versatile condiment made by blending or chopping fresh ingredients, such as tomatoes, onions, peppers, and herbs, creating a vibrant and flavorful sauce that can be used to top various dishes. 90 | 91 | 46. What is the difference between a sauce and a gravy, and when would you use each one? A sauce and a gravy are both liquid-based condiments, but they differ in their ingredients, preparation, and intended use. 92 | 93 | 47. Can you explain the process of making a velouté, and what are some common uses for this classic French sauce? Velouté is a classic French sauce made by whisking a light stock into a roux, creating a smooth and velvety consistency that is often used as a base for various sauces and dishes. 94 | 95 | 48. What is the purpose of whipping cream, and how does it affect the texture and flavor of the final product? Whipping cream is the process of incorporating air into cream, creating a light and fluffy texture that can be used as a topping or filling for various desserts. 96 | 97 | 49. Can you describe the process of making a yogurt, and what are some common uses for this tangy and creamy dairy product? Yogurt is a tangy and creamy dairy product made by fermenting milk with live bacterial cultures, creating a versatile and nutritious ingredient that can be used in various sweet and savory dishes. 98 | 99 | 50. What is the difference between a zest and a peel, and when would you use each one? A zest and a peel are both parts of a citrus fruit, but they differ in their texture, flavor, and intended use. 100 | 101 | 51. Can you explain the process of making a bouillon, and what are some common uses for this flavorful and aromatic broth? Bouillon is a flavorful and aromatic broth made by simmering meat, vegetables, and herbs in water, creating a concentrated and versatile base for various soups, stews, and sauces. 102 | 103 | 52. What is the purpose of caramelizing onions, and how does it affect the flavor and texture of the final dish? Caramelizing onions is the process of slowly cooking onions in oil or butter, creating a sweet and savory flavor and a soft and tender texture. 104 | 105 | 53. Can you describe the process of making a curry, and what are some common flavor combinations for this spicy and aromatic dish? Curry is a spicy and aromatic dish made by cooking meat, vegetables, and spices in a flavorful sauce, creating a complex and richly spiced dish that can be served with rice or bread. 106 | 107 | 54. What is the difference between a chutney and a relish, and when would you use each one? A chutney and a relish are both condiments made from fruits or vegetables, but they differ in their preparation, consistency, and intended use. 108 | 109 | 55. Can you explain the process of making a gastrique, and what are some common uses for this sweet and sour sauce? Gastrique is a sweet and sour sauce made by reducing vinegar and sugar, creating a tangy and versatile condiment that can be used to top various dishes. 110 | 111 | 56. What is the purpose of marbling meat, and how does it affect the texture and flavor of the final product? Marbling meat is the process of incorporating fat into the muscle tissue, creating a tender and flavorful texture that is often associated with high-quality cuts of meat. 112 | 113 | 57. Can you describe the process of making a mole, and what are some common flavor combinations for this complex and richly spiced sauce? Mole is a complex and richly spiced sauce made by blending various ingredients, such as chili peppers, chocolate, and spices, creating a complex and flavorful condiment that can be used to top various dishes. 114 | 115 | 58. What is the difference between a nage and a court-bouillon, and when would you use each one? A nage and a court-bouillon are both flavorful liquids used to poach fish and seafood, but they differ in their ingredients, preparation, and intended use. 116 | 117 | 59. Can you explain the process of making a panade, and what are some common uses for this bread-based mixture? Panade is a bread-based mixture made by soaking bread in liquid, creating a thick and creamy consistency that can be used as a base for various soups, stews, and sauces. 118 | 119 | 60. What is the purpose of poaching, and how does it affect the texture and flavor of the final dish? Poaching is the process of gently cooking food in a flavorful liquid, creating a tender and moist texture that is often associated with delicate proteins, such as fish and eggs. 120 | 121 | 61. Can you describe the process of making a quick bread, and what are some common flavor combinations for this versatile baked good? Quick bread is a versatile baked good made by combining dry and wet ingredients, creating a tender and flavorful bread that can be served as a snack, breakfast, or dessert. 122 | 123 | 62. What is the difference between a reduction and a glaze, and when would you use each one? A reduction and a glaze are both thick and flavorful sauces, but they differ in their preparation, consistency, and intended use. 124 | 125 | 63. Can you explain the process of making a roux, and what are some common uses for this flour-based thickener? Roux is a flour-based thickener made by cooking flour and fat together, creating a smooth and creamy consistency that can be used as a base for various sauces, soups, and stews. 126 | 127 | 64. What is the purpose of sautéing, and how does it affect the texture and flavor of the final dish? Sautéing is the process of quickly cooking food in a small amount of fat, creating a tender and flavorful texture that is often associated with vegetables, meats, and seafood. 128 | 129 | 65. Can you describe the process of making a slurry, and what are some common uses for this flour-based thickener? Slurry is a flour-based thickener made by mixing flour and water together, creating a smooth and creamy consistency that can be used to thicken various sauces, soups, and stews. 130 | 131 | 66. What is the difference between a stock and a broth, and when would you use each one? Stock and broth are both flavorful liquids made by simmering meat, vegetables, and herbs in water, but they differ in their ingredients, preparation, and intended use. 132 | 133 | 67. Can you explain the process of making a tartare, and what are some common uses for this raw and flavorful dish? Tartare is a raw and flavorful dish made by finely chopping or mincing meat or fish, creating a tender and flavorful texture that can be served as an appetizer or main course. 134 | 135 | 68. What is the purpose of tempering, and how does it affect the texture and flavor of the final dish? Tempering is the process of slowly and gradually adding a hot liquid to a cold or room-temperature ingredient, creating a smooth and creamy consistency that is often associated with custards, sauces, and soups. 136 | 137 | 69. Can you describe the process of making a vinaigrette, and what are some common flavor combinations for this tangy and versatile dressing? Vinaigrette is a tangy and versatile dressing made by whisking together oil and vinegar, creating a smooth and creamy consistency that can be used to top various salads and dishes. 138 | 139 | 70. What is the difference between a wonton and a dumpling, and when would you use each one? A wonton and a dumpling are both filled pasta dishes, but they differ in their preparation, ingredients, and intended use. 140 | 141 | 71. Can you explain the process of making a yogurt sauce, and what are some common uses for this tangy and creamy condiment? Yogurt sauce is a tangy and creamy condiment made by blending yogurt with various herbs, spices, and seasonings, creating a flavorful and versatile sauce that can be used to top various dishes. 142 | 143 | 72. What is the purpose of zesting, and how does it affect the flavor and aroma of the final product? Zesting is the process of removing the outer layer of a citrus fruit, creating a flavorful and aromatic ingredient that can be used to enhance various sweet and savory dishes. 144 | 145 | 73. Can you describe the process of making a braise, and what are some common flavor combinations for this slow-cooked and tender dish? A braise is a slow-cooked and tender dish made by browning meat or vegetables, then slowly simmering them in a flavorful liquid, creating a rich and complex flavor that is often associated with comfort foods. 146 | 147 | 74. What is the difference between a grill and a broiler, and when would you use each one? A grill and a broiler are both high-heat cooking methods, but they differ in their preparation, equipment, and intended use. 148 | 149 | 75. Can you explain the process of making a sabayon, and what are some common uses for this light and frothy sauce? Sabayon is a light and frothy sauce made by whisking egg yolks, sugar, and wine together over heat, creating a smooth and creamy consistency that can be used as a base for various desserts and sauces. 150 | 151 | 76. What is the purpose of sweating, and how does it affect the texture and flavor of the final dish? Sweating is the process of cooking vegetables or aromatics in a small amount of fat over low heat, creating a tender and flavorful texture that is often associated with soups, stews, and sauces. 152 | 153 | 77. Can you describe the process of making a terrine, and what are some common flavor combinations for this layered and flavorful dish? A terrine is a layered and flavorful dish made by packing meat, vegetables, or seafood into a mold, then baking or chilling it until it sets, creating a complex and richly textured dish that can be served as an appetizer or main course. 154 | 155 | 78. What is the difference between a velouté and a béchamel, and when would you use each one? Velouté and béchamel are both classic French sauces made by whisking a roux into a liquid, but they differ in their preparation, ingredients, and intended use. 156 | 157 | 79. Can you explain the process of making a vichyssoise, and what are some common flavor combinations for this cold and creamy soup? Vichyssoise is a cold and creamy soup made by blending potatoes, leeks, and cream together, creating a smooth and velvety consistency that is often served as a first course or light meal. 158 | 159 | 80. What is the purpose of whipping egg whites, and how does it affect the texture and volume of the final product? Whipping egg whites is the process of incorporating air into egg whites, creating a light and fluffy texture that can be used as a base for various desserts, meringues, and soufflés. 160 | 161 | 81. Can you describe the process of making a yogurt marinade, and what are some common uses for this tangy and flavorful marinade? A yogurt marinade is a tangy and flavorful marinade made by blending yogurt with various herbs, spices, and seasonings, creating a tender and flavorful texture that can be used to marinate various meats and vegetables. 162 | 163 | 82. What is the difference between a zampone and a cotechino, and when would you use each one? Zampone and cotechino are both Italian pork sausages, but they differ in their preparation, ingredients, and intended use. 164 | 165 | 83. Can you explain the process of making a zeppole, and what are some common flavor combinations for this Italian pastry? Zeppole are Italian pastries made by deep-frying dough, then topping them with various sweet and savory ingredients, creating a tender and flavorful texture that can be served as a snack, breakfast, or dessert. 166 | 167 | 84. What is the purpose of blanching, and how does it affect the texture and flavor of the final dish? Blanching is the process of briefly boiling or steaming vegetables, then shocking them in ice water, creating a tender and vibrant texture that is often associated with salads, soups, and side dishes. 168 | 169 | 85. Can you describe the process of making a bisque, and what are some common flavor combinations for this smooth and creamy soup? Bisque is a smooth and creamy soup made by blending seafood, aromatics, and cream together, creating a rich and velvety consistency that is often served as a first course or light meal. 170 | 171 | 86. What is the difference between a bouillabaisse and a cioppino, and when would you use each one? Bouillabaisse and cioppino are both seafood stews, but they differ in their preparation, ingredients, and intended use. 172 | 173 | 87. Can you explain the process of making a ceviche, and what are some common flavor combinations for this raw and tangy dish? Ceviche is a raw and tangy dish made by marinating seafood in citrus juice, creating a tender and flavorful texture that is often served as an appetizer or main course. 174 | 175 | 88. What is the purpose of clarifying butter, and how does it affect the texture and flavor of the final product? Clarifying butter is the process of separating the milk solids from the butterfat, creating a clear and pure fat that can be used for high-heat cooking and baking. 176 | 177 | 89. Can you describe the process of making a consommé, and what are some common flavor combinations for this clear and flavorful broth? Consommé is a clear and flavorful broth made by clarifying a stock with egg whites, creating a delicate and refined texture that is often served as a first course or light meal. 178 | 179 | 90. What is the difference between a coulis and a sauce, and when would you use each one? A coulis and a sauce are both flavorful liquids used to enhance various dishes, but they differ in their preparation, consistency, and intended use. 180 | 181 | 91. Can you explain the process of making a crème anglaise, and what are some common uses for this rich and creamy sauce? Crème anglaise is a rich and creamy sauce made by whisking egg yolks, sugar, and cream together over heat, creating a smooth and velvety consistency that can be used as a base for various desserts and sauces. 182 | 183 | 92. What is the purpose of curing, and how does it affect the texture and flavor of the final product? Curing is the process of preserving meat or fish by adding salt, sugar, and other flavorings, creating a tender and flavorful texture that is often associated with charcuterie, salami, and gravlax. 184 | 185 | 93. Can you describe the process of making a demi-glace, and what are some common uses for this rich and flavorful sauce? Demi-glace is a rich and flavorful sauce made by reducing a brown stock and espagnole sauce together, creating a concentrated and velvety consistency that can be used as a base for various sauces and dishes. 186 | 187 | 94. What is the difference between a emulsification and a suspension, and when would you use each one? Emulsification and suspension are both methods of combining two or more ingredients together, but they differ in their preparation, consistency, and intended use. 188 | 189 | 95. Can you explain the process of making a foam, and what are some common uses for this light and airy texture? Foam is a light and airy texture made by incorporating air into a liquid, creating a delicate and ethereal consistency that can be used to top various dishes and cocktails. 190 | 191 | 96. What is the purpose of flambéing, and how does it affect the texture and flavor of the final dish? Flambéing is the process of igniting alcohol in a hot pan, creating a dramatic and flavorful effect that is often associated with desserts and sauces. 192 | 193 | 97. Can you describe the process of making a fondue, and what are some common flavor combinations for this melted and creamy dish? Fondue is a melted and creamy dish made by blending cheese, wine, and various seasonings together, creating a smooth and velvety consistency that can be used as a dip for various breads, meats, and vegetables. 194 | 195 | 98. What is the difference between a fricassée and a ragout, and when would you use each one? Fricassée and ragout are both slow-cooked dishes made by braising meat or vegetables in a flavorful liquid, but they differ in their preparation, ingredients, and intended use. 196 | 197 | 99. Can you explain the process of making a gelée, and what are some common uses for this clear and flavorful texture? Gelée is a clear and flavorful texture made by setting a liquid with gelatin, creating a delicate and elegant consistency that can be used as a base for various desserts and appetizers. 198 | 199 | 100. What is the purpose of glazing, and how does it affect the texture and flavor of the final dish? Glazing is the process of coating food with a sweet or savory mixture, creating a shiny and flavorful texture that is often associated with roasts, hams, and pastries. --------------------------------------------------------------------------------