├── docs ├── assets │ ├── logo.png │ ├── favicon.png │ ├── jaeger_traces_list.png │ ├── jaeger_traces_list_only_triton.png │ ├── jaeger_traces_list_propagation.png │ └── styles.css ├── overrides │ └── partials │ │ └── copyright.html └── reference │ ├── decorators.md │ ├── triton_config.md │ ├── triton.md │ ├── clients.md │ └── model_config.md ├── examples ├── dali_resnet101_pytorch │ ├── test_video │ │ ├── orig0.jpg │ │ ├── segm0.jpg │ │ └── sintel_trailer_short.mp4 │ ├── LICENSE │ └── __init__.py ├── tensorrt_llm │ ├── input_data.json │ ├── requirements.txt │ └── __init__.py ├── add_sub_vertex_ai │ ├── data.json │ ├── __init__.py │ ├── install.sh │ ├── health_vertex_ai_endpoint.sh │ └── infer_vertex_ai_endpoint.sh ├── perf_analyzer │ ├── input-data.json │ ├── __init__.py │ ├── install.sh │ └── client.sh ├── vllm │ ├── __init__.py │ ├── install.sh │ ├── client.sh │ └── client_streaming.sh ├── linear_cupy │ ├── __init__.py │ └── install.sh ├── add_sub_notebook │ ├── __init__.py │ └── README.md ├── add_sub_python │ ├── __init__.py │ └── install.sh ├── identity_python │ ├── __init__.py │ └── install.sh ├── huggingface_bert_jax │ ├── __init__.py │ └── install.sh ├── linear_cupy_notebook │ ├── __init__.py │ └── README.md ├── linear_random_pytorch │ └── __init__.py ├── mlp_random_tensorflow2 │ ├── __init__.py │ └── client.py ├── multiple_models_python │ ├── __init__.py │ └── install.sh ├── online_learning_mnist │ ├── __init__.py │ └── install.sh ├── huggingface_bart_pytorch │ ├── __init__.py │ ├── install.sh │ └── kubernetes │ │ ├── deployment │ │ ├── Chart.yaml │ │ ├── values.yaml │ │ └── templates │ │ │ └── service.yaml │ │ ├── test │ │ ├── Chart.yaml │ │ └── values.yaml │ │ └── Dockerfile ├── huggingface_resnet_pytorch │ ├── __init__.py │ ├── install.sh │ └── kubernetes │ │ ├── deployment │ │ ├── Chart.yaml │ │ ├── values.yaml │ │ └── templates │ │ │ └── service.yaml │ │ └── test │ │ ├── Chart.yaml │ │ └── values.yaml ├── nemo_megatron_gpt_multinode │ ├── __init__.py │ ├── install.sh │ └── kubernetes │ │ ├── multi-node │ │ ├── Chart.yaml │ │ ├── values.yaml │ │ └── templates │ │ │ └── service.yaml │ │ ├── single-node │ │ ├── Chart.yaml │ │ ├── values.yaml │ │ └── templates │ │ │ └── service.yaml │ │ ├── test │ │ ├── Chart.yaml │ │ └── values.yaml │ │ ├── persistent-volume-nfs.yaml │ │ ├── persistent-volume-claim-csi.yaml │ │ ├── persistent-volume-claim-nfs.yaml │ │ └── health.sh ├── simple_python_remote_mode │ ├── __init__.py │ └── install.sh ├── use_parameters_and_headers │ ├── __init__.py │ └── install.sh ├── add_sub_python_with_optional │ ├── __init__.py │ └── install.sh ├── huggingface_stable_diffusion │ ├── __init__.py │ ├── install.sh │ └── kubernetes │ │ ├── deployment │ │ ├── Chart.yaml │ │ ├── values.yaml │ │ └── templates │ │ │ └── service.yaml │ │ └── test │ │ ├── Chart.yaml │ │ └── values.yaml ├── multi_instance_resnet50_pytorch │ ├── __init__.py │ ├── install.sh │ └── client.sh ├── huggingface_dialogpt_streaming_pytorch │ ├── __init__.py │ └── install.sh └── huggingface_opt_multinode_jax │ ├── __init__.py │ ├── kubernetes │ ├── test │ │ ├── Chart.yaml │ │ └── values.yaml │ ├── multi-node │ │ ├── Chart.yaml │ │ ├── values.yaml │ │ └── templates │ │ │ └── service.yaml │ ├── single-node │ │ ├── Chart.yaml │ │ ├── values.yaml │ │ └── templates │ │ │ └── service.yaml │ ├── persistent-volume-nfs.yaml │ ├── persistent-volume-claim-csi.yaml │ ├── persistent-volume-claim-nfs.yaml │ └── health.sh │ ├── Dockerfile │ └── install.sh ├── CITATION.cff ├── COPYRIGHT ├── tests ├── __init__.py ├── unit │ ├── __init__.py │ ├── test_init.py │ ├── assets │ │ ├── invalid_config.pbtxt │ │ └── valid_config.pbtxt │ ├── test_utils_dataclasses.py │ └── test_tensor.py └── functional │ ├── __init__.py │ ├── common │ ├── __init__.py │ └── tests │ │ └── __init__.py │ ├── L0_decoupled_mode │ ├── __init__.py │ ├── test.py │ └── test.sh │ ├── L0_example_vllm │ ├── __init__.py │ └── test.sh │ ├── L0_network_timeouts │ ├── __init__.py │ ├── test.py │ └── test.sh │ ├── L0_pytriton_check │ ├── __init__.py │ └── test.sh │ ├── L0_response_cache │ ├── __init__.py │ └── test.sh │ ├── L0_run_in_thread │ ├── __init__.py │ └── test.sh │ ├── L1_vllm_integration │ ├── __init__.py │ ├── .gitignore │ └── test.py │ ├── L0_example_linear_cupy │ ├── __init__.py │ └── test.sh │ ├── L0_example_perf_analyzer │ ├── __init__.py │ └── test.sh │ ├── L0_fatal_error_handling │ ├── __init__.py │ └── test.sh │ ├── L0_model_error_handling │ ├── __init__.py │ └── test.sh │ ├── L0_remote_life_cycle │ ├── __init__.py │ ├── test.py │ └── test.sh │ ├── L0_run_on_python_3-10 │ └── __init__.py │ ├── L0_run_on_python_3-11 │ └── __init__.py │ ├── L0_run_on_python_3-8 │ └── __init__.py │ ├── L0_run_on_python_3-9 │ └── __init__.py │ ├── L0_tritons_cohabitation │ ├── __init__.py │ ├── test.py │ └── test.sh │ ├── L1_futures_client_stress │ ├── __init__.py │ └── test.sh │ ├── L1_hf_nlp_distilbert │ ├── __init__.py │ └── test.sh │ ├── L1_tfhub_image_detection │ ├── __init__.py │ └── test.sh │ ├── L2_hf_nlp_distilbert │ └── __init__.py │ ├── L2_tfhub_image_detection │ ├── __init__.py │ └── test.sh │ ├── L0_example_add_sub_python │ ├── __init__.py │ └── test.sh │ ├── L0_example_add_sub_vertex_ai │ ├── __init__.py │ └── test.sh │ ├── L0_example_identity_python │ ├── __init__.py │ └── test.sh │ ├── L0_example_nemo_megatron_gpt │ ├── __init__.py │ └── test.sh │ ├── L0_example_online_learning │ ├── __init__.py │ └── test.sh │ ├── L0_run_on_python_3-10_pyenv │ └── __init__.py │ ├── L0_run_on_python_3-11_pyenv │ └── __init__.py │ ├── L0_run_on_python_3-8_pyenv │ └── __init__.py │ ├── L0_run_on_python_3-9_pyenv │ └── __init__.py │ ├── L2_asyncio_client_long_test │ ├── __init__.py │ └── test.sh │ ├── L2_futures_client_long_test │ ├── __init__.py │ └── test.sh │ ├── L0_example_huggingface_bert_jax │ ├── __init__.py │ └── test.sh │ ├── L0_example_linear_random_pytorch │ ├── __init__.py │ └── test.sh │ ├── L0_example_mlp_random_tensorflow2 │ ├── __init__.py │ └── test.sh │ ├── L0_example_multiple_models_example │ ├── __init__.py │ └── test.sh │ ├── L0_inference_with_variable_io_size │ ├── __init__.py │ └── test.sh │ ├── L0_triton_close_on_parent_death │ ├── __init__.py │ └── test.sh │ ├── L0_warning_on_too_verbose_level │ ├── __init__.py │ └── test.sh │ ├── L0_example_huggingface_bart_pytorch │ ├── __init__.py │ └── test.sh │ ├── L0_example_huggingface_resnet_pytorch │ ├── __init__.py │ └── test.sh │ ├── L0_example_simple_python_remote_mode │ ├── __init__.py │ └── test.sh │ ├── L0_example_use_parameters_and_headers │ ├── __init__.py │ └── test.sh │ ├── L0_resources_released_after_signals │ ├── __init__.py │ └── test.sh │ ├── L0_example_add_sub_python_with_optionals │ ├── __init__.py │ └── test.sh │ ├── L0_example_huggingface_opt_multinode_jax │ ├── __init__.py │ └── test.sh │ ├── L0_example_multi_instance_resnet50_pytorch │ ├── __init__.py │ └── test.sh │ ├── L1_example_nemo_megatron_gpt_prompt_learning │ ├── __init__.py │ └── test.sh │ ├── L0_example_dali_resnet101_pytorch │ ├── __init__.py │ └── test.sh │ ├── L0_example_huggingface_dialogpt_streaming_pytorch │ ├── __init__.py │ └── test.sh │ └── L0_example_huggingface_stable_diffusion_pytorch │ ├── __init__.py │ └── test.sh ├── pytriton ├── check │ └── __init__.py ├── models │ └── __init__.py ├── proxy │ └── __init__.py ├── utils │ ├── __init__.py │ ├── logging.py │ └── dataclasses.py ├── server │ └── __init__.py ├── model_config │ └── __init__.py ├── client │ ├── __init__.py │ └── warnings.py ├── __init__.py └── constants.py ├── .github └── ISSUE_TEMPLATE │ └── feature_request.md └── scripts └── build_wheel.sh /docs/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/this/pytriton/main/docs/assets/logo.png -------------------------------------------------------------------------------- /docs/assets/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/this/pytriton/main/docs/assets/favicon.png -------------------------------------------------------------------------------- /docs/assets/jaeger_traces_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/this/pytriton/main/docs/assets/jaeger_traces_list.png -------------------------------------------------------------------------------- /docs/assets/jaeger_traces_list_only_triton.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/this/pytriton/main/docs/assets/jaeger_traces_list_only_triton.png -------------------------------------------------------------------------------- /docs/assets/jaeger_traces_list_propagation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/this/pytriton/main/docs/assets/jaeger_traces_list_propagation.png -------------------------------------------------------------------------------- /examples/dali_resnet101_pytorch/test_video/orig0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/this/pytriton/main/examples/dali_resnet101_pytorch/test_video/orig0.jpg -------------------------------------------------------------------------------- /examples/dali_resnet101_pytorch/test_video/segm0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/this/pytriton/main/examples/dali_resnet101_pytorch/test_video/segm0.jpg -------------------------------------------------------------------------------- /examples/dali_resnet101_pytorch/test_video/sintel_trailer_short.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/this/pytriton/main/examples/dali_resnet101_pytorch/test_video/sintel_trailer_short.mp4 -------------------------------------------------------------------------------- /examples/tensorrt_llm/input_data.json: -------------------------------------------------------------------------------- 1 | {"data": [{"text_input": {"content": ["one day I will understand"], "shape": [1]}, "max_tokens": [1], "stream": {"content": [true], "shape": [1]}}]} 2 | -------------------------------------------------------------------------------- /examples/tensorrt_llm/requirements.txt: -------------------------------------------------------------------------------- 1 | --extra-index-url https://pypi.nvidia.com 2 | hf_transfer 3 | huggingface_hub[cli] 4 | nvidia-pytriton>=0.5.2 5 | tensorrt_llm==0.8.0 6 | transformers 7 | -------------------------------------------------------------------------------- /examples/dali_resnet101_pytorch/LICENSE: -------------------------------------------------------------------------------- 1 | Test video - excerpt from Sintel https://durian.blender.org 2 | 3 | License (https://durian.blender.org/sharing/): 4 | CC BY 3.0 5 | 6 | © copyright Blender Foundation | www.sintel.org -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this software, please cite it as below." 3 | title: "PyTriton: Framework facilitating NVIDIA Triton Inference Server usage in Python environments." 4 | url: https://github.com/triton-inference-server/pytriton 5 | repository-code: https://github.com/triton-inference-server/pytriton 6 | authors: 7 | - name: "NVIDIA Corporation" -------------------------------------------------------------------------------- /examples/add_sub_vertex_ai/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "0", 3 | "inputs": [ 4 | { 5 | "name": "INPUT_1", 6 | "shape": [1, 4], 7 | "datatype": "FP32", 8 | "parameters": {}, 9 | "data": [[1,2,3,4]] 10 | }, 11 | { 12 | "name": "INPUT_2", 13 | "shape": [1, 4], 14 | "datatype": "FP32", 15 | "parameters": {}, 16 | "data": [[1,2,3,4]] 17 | } 18 | ] 19 | } 20 | 21 | -------------------------------------------------------------------------------- /examples/perf_analyzer/input-data.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": [ 3 | { 4 | "sequence": { 5 | "content": [ 6 | "one day I will see the world" 7 | ], 8 | "shape": [ 9 | 1 10 | ] 11 | }, 12 | "labels": { 13 | "content": [ 14 | "travel", 15 | "cooking", 16 | "dancing" 17 | ], 18 | "shape": [ 19 | 3 20 | ] 21 | } 22 | } 23 | ] 24 | } -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020-2022, NVIDIA CORPORATION. All rights reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/vllm/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/linear_cupy/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/add_sub_notebook/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/add_sub_python/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/add_sub_vertex_ai/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/identity_python/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/perf_analyzer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/common/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/huggingface_bert_jax/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/linear_cupy_notebook/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/linear_random_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/mlp_random_tensorflow2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/multiple_models_python/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/online_learning_mnist/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/common/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/huggingface_bart_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/huggingface_resnet_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/simple_python_remote_mode/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/tensorrt_llm/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/use_parameters_and_headers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pytriton/check/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # noqa: D104 15 | -------------------------------------------------------------------------------- /pytriton/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # noqa: D104 15 | -------------------------------------------------------------------------------- /pytriton/proxy/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # noqa: D104 15 | -------------------------------------------------------------------------------- /pytriton/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # noqa: D104 15 | -------------------------------------------------------------------------------- /tests/functional/L0_decoupled_mode/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_vllm/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_network_timeouts/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_pytriton_check/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_response_cache/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_run_in_thread/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L1_vllm_integration/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/add_sub_python_with_optional/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/huggingface_stable_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/multi_instance_resnet50_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /pytriton/server/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # noqa: D104 15 | -------------------------------------------------------------------------------- /tests/functional/L0_example_linear_cupy/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_perf_analyzer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_fatal_error_handling/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_model_error_handling/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_remote_life_cycle/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_run_on_python_3-10/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_run_on_python_3-11/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_run_on_python_3-8/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_run_on_python_3-9/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_tritons_cohabitation/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L1_futures_client_stress/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L1_hf_nlp_distilbert/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L1_tfhub_image_detection/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L2_hf_nlp_distilbert/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L2_tfhub_image_detection/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/dali_resnet101_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_add_sub_python/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_add_sub_vertex_ai/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_identity_python/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_nemo_megatron_gpt/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_online_learning/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_run_on_python_3-10_pyenv/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_run_on_python_3-11_pyenv/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_run_on_python_3-8_pyenv/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_run_on_python_3-9_pyenv/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L2_asyncio_client_long_test/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L2_futures_client_long_test/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /examples/huggingface_dialogpt_streaming_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_huggingface_bert_jax/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_linear_random_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_mlp_random_tensorflow2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_multiple_models_example/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_inference_with_variable_io_size/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_triton_close_on_parent_death/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_warning_on_too_verbose_level/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_huggingface_bart_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_huggingface_resnet_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_simple_python_remote_mode/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_use_parameters_and_headers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_resources_released_after_signals/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /docs/overrides/partials/copyright.html: -------------------------------------------------------------------------------- 1 | 16 | Copyright © 2022 NVIDIA Corporation -------------------------------------------------------------------------------- /tests/functional/L0_example_add_sub_python_with_optionals/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_huggingface_opt_multinode_jax/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_multi_instance_resnet50_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L1_example_nemo_megatron_gpt_prompt_learning/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L1_vllm_integration/.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | *.json 15 | *.log 16 | -------------------------------------------------------------------------------- /tests/functional/L0_example_dali_resnet101_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_huggingface_dialogpt_streaming_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/functional/L0_example_huggingface_stable_diffusion_pytorch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /docs/reference/decorators.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # Decorators 18 | 19 | ::: pytriton.decorators 20 | -------------------------------------------------------------------------------- /examples/simple_python_remote_mode/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | pip install numpy -------------------------------------------------------------------------------- /examples/add_sub_python/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | pip install numpy -------------------------------------------------------------------------------- /examples/identity_python/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | pip install numpy -------------------------------------------------------------------------------- /examples/use_parameters_and_headers/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | pip install numpy -------------------------------------------------------------------------------- /examples/add_sub_vertex_ai/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | pip install numpy -------------------------------------------------------------------------------- /examples/vllm/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | pip install "vllm>=0.2.3" -------------------------------------------------------------------------------- /examples/multiple_models_python/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | pip install numpy -------------------------------------------------------------------------------- /examples/online_learning_mnist/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | pip install numpy -------------------------------------------------------------------------------- /docs/reference/triton_config.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # TritonConfig 18 | 19 | ::: pytriton.triton.TritonConfig 20 | 21 | -------------------------------------------------------------------------------- /examples/add_sub_python_with_optional/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | pip install numpy -------------------------------------------------------------------------------- /docs/assets/styles.css: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | .md-header__button.md-logo :is(img,svg) { 18 | height: 1.8rem; !important; 19 | } 20 | -------------------------------------------------------------------------------- /examples/huggingface_bart_pytorch/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | pip install transformers 18 | -------------------------------------------------------------------------------- /examples/huggingface_dialogpt_streaming_pytorch/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | pip install transformers 18 | -------------------------------------------------------------------------------- /examples/huggingface_resnet_pytorch/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | pip install transformers datasets 18 | -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | pip3 install "huggingface-hub>=0.13" "numpy==1.22" 17 | -------------------------------------------------------------------------------- /examples/huggingface_stable_diffusion/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | pip install transformers accelerate diffusers Pillow filelock -------------------------------------------------------------------------------- /docs/reference/triton.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # Triton 18 | 19 | ::: pytriton.triton.Triton 20 | 21 | ::: pytriton.triton.RemoteTriton 22 | 23 | ::: pytriton.proxy.types.Request 24 | -------------------------------------------------------------------------------- /examples/linear_cupy/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | pip install numpy 18 | pip install cupy-cuda12x --extra-index-url=https://pypi.ngc.nvidia.com 19 | -------------------------------------------------------------------------------- /tests/functional/L1_vllm_integration/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | METADATA = { 17 | "image_name": "nvcr.io/nvidia/pytorch:{TEST_CONTAINER_VERSION}-py3", 18 | } 19 | -------------------------------------------------------------------------------- /examples/add_sub_vertex_ai/health_vertex_ai_endpoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | infer_out=$(curl -s -IX GET http://localhost:8080/v2/health/ready) 18 | echo "$infer_out" 19 | 20 | -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/kubernetes/test/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v2 15 | description: JAX Example Test 16 | name: jax-example-test 17 | version: 0.0.1 18 | appVersion: 0.0.1 19 | -------------------------------------------------------------------------------- /examples/multi_instance_resnet50_pytorch/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | apt-get update 18 | apt-get install -y libb64-dev 19 | pip install transformers nvidia-pytriton 20 | -------------------------------------------------------------------------------- /examples/huggingface_bart_pytorch/kubernetes/deployment/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v2 15 | description: BART PyTriton Example 16 | name: bart-pytorch-example 17 | version: 0.0.1 18 | appVersion: 0.0.1 19 | -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/kubernetes/multi-node/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v2 15 | description: JAX Multi Node Example 16 | name: jax-example 17 | version: 0.0.1 18 | appVersion: 0.0.1 19 | -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/kubernetes/single-node/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v2 15 | description: JAX Single Node Example 16 | name: jax-example 17 | version: 0.0.1 18 | appVersion: 0.0.1 19 | -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/kubernetes/multi-node/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v2 15 | description: NeMo Multi Node Example 16 | name: nemo-example 17 | version: 0.0.1 18 | appVersion: 0.0.1 19 | -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/kubernetes/single-node/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v2 15 | description: NeMo Single Node Example 16 | name: nemo-example 17 | version: 0.0.1 18 | appVersion: 0.0.1 19 | -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/kubernetes/test/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v2 15 | description: BART PyTriton Example Test 16 | name: nemo-example-test 17 | version: 0.0.1 18 | appVersion: 0.0.1 19 | -------------------------------------------------------------------------------- /examples/huggingface_bart_pytorch/kubernetes/test/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v2 15 | description: BART PyTriton Example Test 16 | name: bart-pytorch-example-test 17 | version: 0.0.1 18 | appVersion: 0.0.1 19 | -------------------------------------------------------------------------------- /examples/huggingface_resnet_pytorch/kubernetes/deployment/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v2 15 | description: ResNet PyTriton Example 16 | name: resnet-pytorch-example 17 | version: 0.0.1 18 | appVersion: 0.0.1 19 | -------------------------------------------------------------------------------- /tests/functional/L0_remote_life_cycle/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Test metadata""" 16 | 17 | METADATA = { 18 | "image_name": "nvcr.io/nvidia/pytorch:{TEST_CONTAINER_VERSION}-py3", 19 | } 20 | -------------------------------------------------------------------------------- /examples/huggingface_resnet_pytorch/kubernetes/test/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v2 15 | description: ResNet PyTriton Example Test 16 | name: resnet-pytorch-example-test 17 | version: 0.0.1 18 | appVersion: 0.0.1 19 | -------------------------------------------------------------------------------- /tests/unit/test_init.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # noqa: D104 15 | from importlib.metadata import version 16 | 17 | import pytriton 18 | 19 | 20 | def test_version(): 21 | assert pytriton.__version__ == version("nvidia-pytriton") 22 | -------------------------------------------------------------------------------- /examples/perf_analyzer/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | echo "Installing libb64-dev required by Perf Analyzer" 18 | apt-get update 19 | apt-get install -y libb64-dev 20 | pip install transformers 21 | -------------------------------------------------------------------------------- /examples/huggingface_stable_diffusion/kubernetes/deployment/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v2 15 | description: Stable Diffusion 1.5 PyTriton Example 16 | name: stable-diffusion-example 17 | version: 0.0.1 18 | appVersion: 0.0.1 19 | -------------------------------------------------------------------------------- /examples/huggingface_stable_diffusion/kubernetes/test/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v2 15 | description: Stable Diffusion PyTriton Example Test 16 | name: stable-diffusion-example-test 17 | version: 0.0.1 18 | appVersion: 0.0.1 19 | -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/kubernetes/test/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | imagePullSecret: null 15 | pullPolicy: Always 16 | restartPolicy: OnFailure 17 | backoffLimit: 4 18 | image: null 19 | serviceUrl: "http://jax-example-service:8000" -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/kubernetes/test/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | imagePullSecret: null 15 | pullPolicy: Always 16 | restartPolicy: OnFailure 17 | backoffLimit: 4 18 | image: null 19 | serviceUrl: "http://nemo-example-service:8000" -------------------------------------------------------------------------------- /examples/huggingface_bart_pytorch/kubernetes/test/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | imagePullSecret: null 15 | pullPolicy: Always 16 | restartPolicy: OnFailure 17 | backoffLimit: 4 18 | image: null 19 | serviceUrl: "http://bart-pytorch-example-service:8000" -------------------------------------------------------------------------------- /examples/huggingface_resnet_pytorch/kubernetes/test/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | imagePullSecret: null 15 | pullPolicy: Always 16 | restartPolicy: OnFailure 17 | backoffLimit: 4 18 | image: null 19 | serviceUrl: "http://resnet-pytorch-example-service:8000" -------------------------------------------------------------------------------- /examples/huggingface_stable_diffusion/kubernetes/test/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | imagePullSecret: null 15 | pullPolicy: Always 16 | restartPolicy: OnFailure 17 | backoffLimit: 4 18 | image: null 19 | serviceUrl: "http://stable-diffusion-example-service:8000" -------------------------------------------------------------------------------- /pytriton/model_config/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # noqa: D104 15 | from .common import DeviceKind, DynamicBatcher, QueuePolicy, TimeoutAction # noqa: F401 16 | from .model_config import ModelConfig # noqa: F401 17 | from .tensor import Tensor # noqa: F401 18 | -------------------------------------------------------------------------------- /tests/functional/L0_decoupled_mode/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Test metadata""" 16 | 17 | METADATA = { 18 | "image_name": "nvcr.io/nvidia/pytorch:{TEST_CONTAINER_VERSION}-py3", 19 | "platforms": ["amd64", "arm64"], 20 | } 21 | -------------------------------------------------------------------------------- /tests/functional/L0_network_timeouts/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Test metadata""" 16 | 17 | METADATA = { 18 | "image_name": "nvcr.io/nvidia/pytorch:{TEST_CONTAINER_VERSION}-py3", 19 | "platforms": ["amd64", "arm64"], 20 | } 21 | -------------------------------------------------------------------------------- /tests/functional/L0_tritons_cohabitation/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Test metadata""" 16 | 17 | METADATA = { 18 | "image_name": "nvcr.io/nvidia/pytorch:{TEST_CONTAINER_VERSION}-py3", 19 | "platforms": ["amd64", "arm64"], 20 | } 21 | -------------------------------------------------------------------------------- /examples/huggingface_bart_pytorch/kubernetes/deployment/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | imagePullSecret: null 15 | pullPolicy: Always 16 | restartPolicy: Always 17 | replicaCount: 1 18 | deployment: 19 | image: null 20 | numOfGPUs: 1 21 | service: 22 | type: ClusterIP 23 | -------------------------------------------------------------------------------- /examples/huggingface_resnet_pytorch/kubernetes/deployment/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | imagePullSecret: null 15 | pullPolicy: Always 16 | restartPolicy: Always 17 | replicaCount: 1 18 | deployment: 19 | image: null 20 | numOfGPUs: 1 21 | service: 22 | type: ClusterIP 23 | -------------------------------------------------------------------------------- /examples/huggingface_stable_diffusion/kubernetes/deployment/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | imagePullSecret: null 15 | pullPolicy: Always 16 | restartPolicy: Always 17 | replicaCount: 1 18 | deployment: 19 | image: null 20 | numOfGPUs: 1 21 | service: 22 | type: ClusterIP 23 | -------------------------------------------------------------------------------- /tests/functional/L0_example_vllm/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /examples/add_sub_vertex_ai/infer_vertex_ai_endpoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | infer_out=$(curl -s -X POST -H "Content-Type: application/json" -d @examples/add_sub_vertex_ai/data.json http://localhost:8080/v2/models/AddSub/infer) 18 | echo "$infer_out" 19 | 20 | -------------------------------------------------------------------------------- /examples/multi_instance_resnet50_pytorch/client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | perf_analyzer -u 127.0.0.1:8000 \ 16 | -i http \ 17 | -m ResNet50 \ 18 | --measurement-mode count_windows \ 19 | --measurement-request-count 100 \ 20 | --concurrency-range 64:64:64 \ 21 | -v 22 | -------------------------------------------------------------------------------- /tests/functional/L0_pytriton_check/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_add_sub_python/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_linear_cupy/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_perf_analyzer/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_add_sub_vertex_ai/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_identity_python/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_online_learning/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 600 -------------------------------------------------------------------------------- /tests/functional/L0_example_huggingface_bart_pytorch/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_linear_random_pytorch/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_multiple_models_example/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_add_sub_python_with_optionals/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_huggingface_bert_jax/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 23 | -------------------------------------------------------------------------------- /tests/functional/L0_example_huggingface_resnet_pytorch/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_mlp_random_tensorflow2/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 23 | -------------------------------------------------------------------------------- /tests/functional/L0_example_simple_python_remote_mode/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_use_parameters_and_headers/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_multi_instance_resnet50_pytorch/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /docs/reference/clients.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # Model Clients 18 | 19 | ::: pytriton.client.ModelClient 20 | 21 | ::: pytriton.client.AsyncioModelClient 22 | 23 | ::: pytriton.client.DecoupledModelClient 24 | 25 | ::: pytriton.client.AsyncioDecoupledModelClient 26 | 27 | ::: pytriton.client.FuturesModelClient 28 | 29 | -------------------------------------------------------------------------------- /examples/huggingface_bert_jax/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | # 0.4.14 raises error 18 | pip install --upgrade "jax[cuda12]!=0.4.14" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html 19 | # torch is required for checkpoint loading 20 | pip install transformers flax torch -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/kubernetes/persistent-volume-nfs.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: PersistentVolume 16 | metadata: 17 | name: llm-cache 18 | spec: 19 | capacity: 20 | storage: 2.5T 21 | accessModes: 22 | - ReadWriteMany 23 | nfs: 24 | path: /llm 25 | server: {IP} 26 | -------------------------------------------------------------------------------- /pytriton/client/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # noqa: D104 15 | 16 | from .client import ( 17 | AsyncioDecoupledModelClient, # noqa: F401 18 | AsyncioModelClient, # noqa: F401 19 | DecoupledModelClient, # noqa: F401 20 | FuturesModelClient, # noqa: F401 21 | ModelClient, # noqa: F401 22 | ) 23 | -------------------------------------------------------------------------------- /tests/functional/L0_example_dali_resnet101_pytorch/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_huggingface_opt_multinode_jax/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 23 | -------------------------------------------------------------------------------- /tests/functional/L0_example_huggingface_stable_diffusion_pytorch/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 -------------------------------------------------------------------------------- /tests/functional/L0_example_huggingface_dialogpt_streaming_pytorch/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 300 23 | -------------------------------------------------------------------------------- /tests/functional/L1_example_nemo_megatron_gpt_prompt_learning/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --timeout-s 3000 23 | -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/kubernetes/persistent-volume-nfs.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: PersistentVolume 16 | metadata: 17 | name: llm-cache 18 | spec: 19 | capacity: 20 | storage: 2.5T 21 | accessModes: 22 | - ReadWriteMany 23 | nfs: 24 | path: /llm 25 | server: {IP} 26 | -------------------------------------------------------------------------------- /tests/functional/L0_model_error_handling/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --init-timeout-s 300 \ 23 | --verbose -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/kubernetes/persistent-volume-claim-csi.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: PersistentVolumeClaim 16 | metadata: 17 | name: llm-cache-pvc 18 | spec: 19 | accessModes: 20 | - ReadWriteOnce 21 | resources: 22 | requests: 23 | storage: 50Gi 24 | storageClassName: csi-hostpath-sc -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | ARG FROM_IMAGE_NAME=nvcr.io/nvidia/tensorflow:24.08-tf2-py3 15 | FROM ${FROM_IMAGE_NAME} 16 | 17 | ENV XLA_PYTHON_CLIENT_PREALLOCATE=false 18 | ENV NCCL_LAUNCH_MODE="PARALLEL" 19 | 20 | WORKDIR /workdir 21 | 22 | COPY install.sh . 23 | RUN ./install.sh 24 | 25 | COPY . . 26 | -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/kubernetes/persistent-volume-claim-csi.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: PersistentVolumeClaim 16 | metadata: 17 | name: llm-cache-pvc 18 | spec: 19 | accessModes: 20 | - ReadWriteOnce 21 | resources: 22 | requests: 23 | storage: 50Gi 24 | storageClassName: csi-hostpath-sc 25 | -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/kubernetes/persistent-volume-claim-nfs.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: PersistentVolumeClaim 16 | metadata: 17 | name: llm-cache-pvc 18 | spec: 19 | accessModes: 20 | - ReadWriteMany 21 | storageClassName: "" 22 | volumeName: llm-cache 23 | resources: 24 | requests: 25 | storage: 2.5T -------------------------------------------------------------------------------- /docs/reference/model_config.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # Model Config 18 | 19 | ::: pytriton.model_config.ModelConfig 20 | 21 | ::: pytriton.model_config.Tensor 22 | 23 | ::: pytriton.model_config.DeviceKind 24 | 25 | ::: pytriton.model_config.DynamicBatcher 26 | 27 | ::: pytriton.model_config.QueuePolicy 28 | 29 | ::: pytriton.model_config.TimeoutAction 30 | -------------------------------------------------------------------------------- /tests/functional/L0_response_cache/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --init-timeout-s 300 \ 23 | --shutdown-timeout-s 60 \ 24 | --verbose -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | set -xe 16 | 17 | # Use 0.4.14 raises error 18 | pip install --upgrade "jax[cuda12_pip]!=0.4.14" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html 19 | pip install --upgrade flax omegaconf sacrebleu SentencePiece tokenizers "transformers>=4.26" 20 | -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/kubernetes/persistent-volume-claim-nfs.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: PersistentVolumeClaim 16 | metadata: 17 | name: llm-cache-pvc 18 | spec: 19 | accessModes: 20 | - ReadWriteMany 21 | storageClassName: "" 22 | volumeName: llm-cache 23 | resources: 24 | requests: 25 | storage: 2.5T -------------------------------------------------------------------------------- /tests/functional/L0_fatal_error_handling/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --init-timeout-s 300 \ 23 | --shutdown-timeout-s 60 \ 24 | --verbose -------------------------------------------------------------------------------- /pytriton/client/warnings.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Warnings for pytriton module.""" 15 | 16 | 17 | class PyTritonWarning(UserWarning): 18 | """Base warning for pytriton module.""" 19 | 20 | pass 21 | 22 | 23 | class NotSupportedTimeoutWarning(PyTritonWarning): 24 | """A warning for client, which doesn't support timeout configuration.""" 25 | 26 | pass 27 | -------------------------------------------------------------------------------- /scripts/build_wheel.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | DOCKER_PLATFORM=${1} 19 | 20 | WHEEL_ARCH=$(echo ${DOCKER_PLATFORM} | sed -e 's/^linux\/amd64$/linux_x86_64/g' -e 's/^linux\/arm64$/linux_aarch64/g') 21 | python3 -m build --wheel -C="--build-option=--plat-name" -C="--build-option=${WHEEL_ARCH}" . 22 | python3 -m build --sdist . 23 | -------------------------------------------------------------------------------- /tests/functional/L0_warning_on_too_verbose_level/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --init-timeout-s 300 \ 23 | --shutdown-timeout-s 60 \ 24 | --verbose -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/kubernetes/single-node/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | imagePullSecret: null 15 | pullPolicy: Always 16 | restartPolicy: Always 17 | replicaCount: 1 18 | jaxPort: 12345 19 | socketPort: 65432 20 | deployment: 21 | image: null 22 | numOfGPUs: 1 23 | modelId: facebook/opt-1.3b 24 | initialDelaySeconds: 180 25 | service: 26 | type: LoadBalancer 27 | -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/kubernetes/single-node/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | imagePullSecret: null 15 | pullPolicy: Always 16 | restartPolicy: Always 17 | replicaCount: 1 18 | communicationPort: 12345 19 | deployment: 20 | image: null 21 | numOfGPUs: 1 22 | modelId: nvidia/nemo-megatron-gpt-1.3B 23 | initialDelaySeconds: 180 24 | service: 25 | type: LoadBalancer 26 | -------------------------------------------------------------------------------- /tests/functional/L0_triton_close_on_parent_death/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | 21 | pip install numpy 22 | python -m"${TEST_MODULE}" \ 23 | --timeout-s 60 \ 24 | --seed 2022110811 \ 25 | --verbose 26 | -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/kubernetes/health.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | #!/bin/bash 16 | set -xe 17 | 18 | RANK=${HOSTNAME##*-} 19 | 20 | if [[ "${RANK}" == "0" ]]; 21 | then 22 | # For head node - validate if Triton Server is running 23 | curl --fail localhost:8000/v2/health/live 24 | else 25 | # For workers - validate the process is running 26 | cat /tmp/health 27 | fi -------------------------------------------------------------------------------- /tests/functional/L0_inference_with_variable_io_size/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | 21 | pip install numpy 22 | python -m"${TEST_MODULE}" \ 23 | --init-timeout-s 300 \ 24 | --seed 2022101915 \ 25 | --verbose 26 | -------------------------------------------------------------------------------- /tests/functional/L1_futures_client_stress/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --test-time-s 30 \ 23 | --init-timeout-s 200 \ 24 | --batch-size 16 \ 25 | --seed 20221019 \ 26 | -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/kubernetes/health.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | #!/bin/bash 16 | set -xe 17 | 18 | RANK=${HOSTNAME##*-} 19 | 20 | if [[ "${RANK}" == "0" ]]; 21 | then 22 | # For head node - validate if Triton Server is running 23 | curl --fail localhost:8000/v2/health/live 24 | else 25 | # For workers - validate the process is running 26 | cat /tmp/health 27 | fi -------------------------------------------------------------------------------- /pytriton/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # noqa: D104 15 | from importlib.metadata import PackageNotFoundError, version 16 | 17 | try: 18 | __version__ = version("nvidia-pytriton") 19 | except PackageNotFoundError: 20 | # package is not installed 21 | pass 22 | 23 | from pytriton import ( 24 | client, # noqa: F401 25 | model_config, # noqa: F401 26 | triton, # noqa: F401 27 | ) 28 | -------------------------------------------------------------------------------- /tests/functional/L1_tfhub_image_detection/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --test-time-s 3000 \ 23 | --init-timeout-s 200 \ 24 | --batch-size 128 \ 25 | --seed 20221019 \ 26 | --verbose 27 | -------------------------------------------------------------------------------- /tests/functional/L2_tfhub_image_detection/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | 21 | python -m"${TEST_MODULE}" \ 22 | --test-time-s 36000 \ 23 | --init-timeout-s 300 \ 24 | --batch-size 128 \ 25 | --seed 20221019 \ 26 | --verbose 27 | -------------------------------------------------------------------------------- /examples/perf_analyzer/client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 16 | INPUT_DATA="${THIS_SCRIPT_DIR}/input-data.json" 17 | 18 | perf_analyzer -u 127.0.0.1:8001 \ 19 | -i grpc \ 20 | -m BART \ 21 | --measurement-mode count_windows \ 22 | --measurement-request-count 100 \ 23 | --input-data ${INPUT_DATA} \ 24 | --concurrency-range 4:16:4 \ 25 | -v 26 | -------------------------------------------------------------------------------- /tests/functional/L0_resources_released_after_signals/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | 21 | pip install numpy 22 | python -m"${TEST_MODULE}" \ 23 | --test-timeout-s 300 \ 24 | --init-timeout-s 300 \ 25 | --seed 2022112315 \ 26 | --verbose 27 | -------------------------------------------------------------------------------- /tests/functional/L0_run_in_thread/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | 21 | pip install numpy 22 | python -m"${TEST_MODULE}" \ 23 | --timeout-s 300 \ 24 | --init-timeout-s 300 \ 25 | --seed 2022101915 \ 26 | --batch-size 32 \ 27 | --verbose 28 | -------------------------------------------------------------------------------- /tests/functional/L0_decoupled_mode/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | 20 | pip install pytest-timeout numpy 21 | pytest -svvv \ 22 | --log-cli-level=DEBUG \ 23 | --log-cli-format='%(asctime)s [%(levelname)s] [%(process)d:%(thread)d] %(name)s:%(lineno)d: %(message)s' \ 24 | --timeout=25 \ 25 | ${THIS_SCRIPT_DIR}/test_pytest.py -------------------------------------------------------------------------------- /tests/functional/L0_remote_life_cycle/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | 20 | pip install pytest-timeout numpy 21 | pytest -svvv \ 22 | --log-cli-level=DEBUG \ 23 | --log-cli-format='%(asctime)s [%(levelname)s] [%(process)d:%(thread)d] %(name)s:%(lineno)d: %(message)s' \ 24 | --timeout=60 \ 25 | ${THIS_SCRIPT_DIR}/test_pytest.py -------------------------------------------------------------------------------- /tests/functional/L0_network_timeouts/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | 20 | pip install pytest-timeout numpy 21 | pytest -svvv \ 22 | --log-cli-level=DEBUG \ 23 | --log-cli-format='%(asctime)s [%(levelname)s] [%(process)d:%(thread)d] %(name)s:%(lineno)d: %(message)s' \ 24 | --timeout=120 \ 25 | ${THIS_SCRIPT_DIR}/test_pytest.py 26 | -------------------------------------------------------------------------------- /tests/functional/L0_tritons_cohabitation/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | 20 | pip install pytest-timeout numpy 21 | pytest -svvv \ 22 | --log-cli-level=DEBUG \ 23 | --log-cli-format='%(asctime)s [%(levelname)s] [%(process)d:%(thread)d] %(name)s:%(lineno)d: %(message)s' \ 24 | --timeout=60 \ 25 | ${THIS_SCRIPT_DIR}/test_pytest.py 26 | -------------------------------------------------------------------------------- /pytriton/utils/logging.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Module with logging related utils.""" 15 | 16 | import logging 17 | 18 | 19 | def silence_3rd_party_loggers(): 20 | """Silence 3rd party libraries which adds enormous number of log lines on DEBUG level.""" 21 | logging.getLogger("sh.command").setLevel(logging.WARNING) 22 | logging.getLogger("sh.stream_bufferer").setLevel(logging.WARNING) 23 | logging.getLogger("sh.streamreader").setLevel(logging.WARNING) 24 | -------------------------------------------------------------------------------- /tests/functional/L0_example_nemo_megatron_gpt/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_DIR="$(realpath --relative-to="${PWD}" "$(dirname "$0")")" 19 | TEST_MODULE="${THIS_SCRIPT_DIR//\//.}.test" 20 | 21 | # This is necessary to avoid error in ONNX used my NeMo toolkit for version of protobuf never than 3.7 22 | export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python 23 | 24 | python -m"${TEST_MODULE}" \ 25 | --timeout-s 300 26 | -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/kubernetes/multi-node/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | imagePullSecret: null 15 | pullPolicy: Always 16 | restartPolicy: Always 17 | replicaCount: 1 18 | communicationPort: 12345 19 | statefulset: 20 | image: null 21 | numOfGPUs: 1 22 | numOfNodes: 3 23 | persistentVolumeClaim: llm-cache-pvc 24 | modelId: nvidia/nemo-megatron-gpt-1.3B 25 | clusterName: default.svc.cluster.local 26 | initialDelaySeconds: 180 27 | service: 28 | type: LoadBalancer 29 | -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/kubernetes/multi-node/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | imagePullSecret: null 15 | pullPolicy: Always 16 | restartPolicy: Always 17 | replicaCount: 1 18 | jaxPort: 12345 19 | socketPort: 65432 20 | statefulset: 21 | image: null 22 | numOfGPUs: 1 23 | numOfNodes: 3 24 | persistentVolumeClaim: llm-cache-pvc 25 | modelId: facebook/opt-1.3b 26 | clusterName: default.svc.cluster.local 27 | initialDelaySeconds: 180 28 | service: 29 | type: LoadBalancer 30 | -------------------------------------------------------------------------------- /tests/functional/L1_hf_nlp_distilbert/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | 21 | pip install transformers datasets --upgrade 22 | 23 | python -m"${TEST_MODULE}" \ 24 | --test-time-s 3000 \ 25 | --init-timeout-s 200 \ 26 | --batch-size 16 \ 27 | --sequence-length 128 \ 28 | --seed 20221019 \ 29 | --verbose 30 | -------------------------------------------------------------------------------- /tests/unit/assets/invalid_config.pbtxt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | name: "simple" 15 | platform: "tensorflow_graphdef" 16 | max_batch_size: 8 17 | input [ 18 | { 19 | name: "INPUT0" 20 | data_type: TYPE_INT32 21 | dims: [ ] 22 | }, 23 | { 24 | name: "INPUT1" 25 | data_type: TYPE_INT32 26 | dims: [ 16 ] 27 | } 28 | ] 29 | output [ 30 | { 31 | name: "OUTPUT0" 32 | data_type: TYPE_INT32 33 | dims: [ 16 ] 34 | }, 35 | { 36 | name: "OUTPUT1" 37 | data_type: TYPE_INT32 38 | dims: [ 16 ] 39 | } 40 | ] -------------------------------------------------------------------------------- /tests/unit/assets/valid_config.pbtxt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | name: "simple" 15 | platform: "tensorflow_graphdef" 16 | max_batch_size: 8 17 | input [ 18 | { 19 | name: "INPUT0" 20 | data_type: TYPE_INT32 21 | dims: [ 16 ] 22 | }, 23 | { 24 | name: "INPUT1" 25 | data_type: TYPE_INT32 26 | dims: [ 16 ] 27 | } 28 | ] 29 | output [ 30 | { 31 | name: "OUTPUT0" 32 | data_type: TYPE_INT32 33 | dims: [ 16 ] 34 | }, 35 | { 36 | name: "OUTPUT1" 37 | data_type: TYPE_INT32 38 | dims: [ 16 ] 39 | } 40 | ] -------------------------------------------------------------------------------- /tests/unit/test_utils_dataclasses.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import dataclasses 15 | 16 | import pytest 17 | 18 | from pytriton.utils.dataclasses import kwonly_dataclass 19 | 20 | 21 | def test_kwonly_dataclasses(): 22 | @kwonly_dataclass 23 | @dataclasses.dataclass 24 | class A: 25 | a: int 26 | b: int = 0 27 | c: str = "hello" 28 | 29 | a = A(a=1, b=2, c="world") 30 | with pytest.raises(TypeError): 31 | a = A(1, 2, "world") 32 | 33 | assert isinstance(a, A) 34 | assert dataclasses.is_dataclass(a) 35 | -------------------------------------------------------------------------------- /examples/linear_cupy_notebook/README.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # Linear CuPy Model in Jupyter Notebook 18 | 19 | ## Overview 20 | 21 | The example presents a simple Linear model in Jupyter Notebook using RAPIDS/CuPy library. 22 | 23 | Example consists of following scripts: 24 | 25 | - `linear.ipynb` - Jupyter Notebook file 26 | 27 | ## Quick Start 28 | 29 | The step-by-step guide: 30 | 31 | 1. Install PyTriton following the [installation instruction](../../README.md#installation) 32 | 2. In current terminal run jupyter notebook and open `linear.ipynb` file 33 | 34 | ```shell 35 | jupyter notebook 36 | ``` 37 | -------------------------------------------------------------------------------- /tests/unit/test_tensor.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import dataclasses 15 | 16 | import numpy as np 17 | import pytest 18 | 19 | from pytriton.model_config import Tensor 20 | 21 | 22 | def test_tensor_set_correct_dtype_when_np_dtype_passed(): 23 | tensor = Tensor(name="variable", dtype=np.float32().dtype, shape=(2, 1)) 24 | 25 | assert tensor.dtype == np.float32().dtype.type 26 | 27 | 28 | def test_tensor_raise_error_when_mutate_field(): 29 | tensor = Tensor(name="variable", dtype=np.float32, shape=(2, 1)) 30 | 31 | with pytest.raises(dataclasses.FrozenInstanceError): 32 | tensor.dtype = np.int32().dtype 33 | -------------------------------------------------------------------------------- /examples/vllm/client.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | if [ -z "$1" ]; then 17 | echo "Provide model name as the first argument" 18 | exit 1 19 | fi 20 | 21 | # replace - and / to _ as Triton doesn't allow them in model names 22 | # and are replaced in the server script 23 | MODEL_NAME=${1//-/_} 24 | MODEL_NAME=${MODEL_NAME//\//_} 25 | 26 | curl http://localhost:8000/v2/models/${MODEL_NAME}/generate \ 27 | -H "Content-Type: application/json" \ 28 | -sS \ 29 | -w "\n" \ 30 | -d '{ 31 | "prompt": "San Francisco is a", 32 | "max_tokens": 128, 33 | "temperature": 0 34 | }' -------------------------------------------------------------------------------- /examples/vllm/client_streaming.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | if [ -z "$1" ]; then 17 | echo "Provide model name as the first argument" 18 | exit 1 19 | fi 20 | 21 | # replace - and / to _ as Triton doesn't allow them in model names 22 | # and are replaced in the server script 23 | MODEL_NAME=${1//-/_} 24 | MODEL_NAME=${MODEL_NAME//\//_} 25 | 26 | curl http://localhost:8000/v2/models/${MODEL_NAME}/generate_stream \ 27 | -H "Content-Type: application/json" \ 28 | -sS \ 29 | -w "\n" \ 30 | -d '{ 31 | "prompt": "San Francisco is a", 32 | "max_tokens": 128, 33 | "temperature": 0, 34 | "stream": true 35 | }' -------------------------------------------------------------------------------- /examples/add_sub_notebook/README.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # Add-Sub Python Model in Jupyter Notebook 18 | 19 | ## Overview 20 | 21 | The example presents a simple Add-Sub model which perform an addition and subtraction operations 22 | on passed input data and it also shows how to redefine inference callable and reload the model. 23 | 24 | Example consists of following scripts: 25 | 26 | - `add_sub.ipynb` - Jupyter Notebook file 27 | 28 | ## Quick Start 29 | 30 | The step-by-step guide: 31 | 32 | 1. Install PyTriton following the [installation instruction](../../README.md#installation) 33 | 2. In current terminal run jupyter notebook and open add_sub.ipynb file 34 | 35 | ```shell 36 | jupyter notebook 37 | ``` 38 | -------------------------------------------------------------------------------- /tests/functional/L2_futures_client_long_test/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | LOGS_DIR=${LOGS_DIR:-$PWD} 21 | # Create the logs folder if it does not exist 22 | mkdir -p "$LOGS_DIR" 23 | 24 | # Set the log path with the date and time 25 | LOG_PATH="$LOGS_DIR/log_$(date '+%Y-%m-%d_%H-%M-%S').txt" 26 | 27 | python -m"${TEST_MODULE}" \ 28 | --test-time-s 36000 \ 29 | --init-timeout-s 200 \ 30 | --batch-size 16 \ 31 | --seed 20221019 \ 32 | --enable-fault-handler \ 33 | --process-monitoring-interval 600 \ 34 | --log-path "${LOG_PATH}" \ 35 | --compress-logs -------------------------------------------------------------------------------- /pytriton/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # noqa: D104 15 | """Constants for pytriton.""" 16 | 17 | import os 18 | import pathlib 19 | 20 | DEFAULT_HTTP_PORT = 8000 21 | DEFAULT_GRPC_PORT = 8001 22 | DEFAULT_METRICS_PORT = 8002 23 | TRITON_LOCAL_IP = "127.0.0.1" 24 | TRITON_CONTEXT_FIELD_NAME = "triton_context" 25 | TRITON_PYTHON_BACKEND_INTERPRETER_DIRNAME = "python_backend_interpreter" 26 | DEFAULT_TRITON_STARTUP_TIMEOUT_S = 120 27 | CREATE_TRITON_CLIENT_TIMEOUT_S = 10 28 | 29 | __DEFAULT_PYTRITON_HOME = os.path.join(os.getenv("XDG_CACHE_HOME", "$HOME/.cache"), "pytriton") 30 | __PYTRITON_HOME = os.path.expanduser(os.path.expandvars(os.getenv("PYTRITON_HOME", __DEFAULT_PYTRITON_HOME))) 31 | PYTRITON_HOME = pathlib.Path(__PYTRITON_HOME).resolve().absolute() 32 | -------------------------------------------------------------------------------- /tests/functional/L2_asyncio_client_long_test/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -xe 17 | 18 | THIS_SCRIPT_PATH="$(realpath --relative-to="$(pwd)" "$0")" 19 | TEST_MODULE="$(dirname "${THIS_SCRIPT_PATH}"|sed 's/\//./g').test" 20 | LOGS_DIR=${LOGS_DIR:-$PWD} 21 | # Create the logs folder if it does not exist 22 | mkdir -p "$LOGS_DIR" 23 | 24 | # Set the log path with the date and time 25 | LOG_PATH="$LOGS_DIR/log_$(date '+%Y-%m-%d_%H-%M-%S').txt" 26 | 27 | python -m"${TEST_MODULE}" \ 28 | --test-time-s 36000 \ 29 | --init-timeout-s 200 \ 30 | --batch-size 16 \ 31 | --seed 20221019 \ 32 | --enable-fault-handler \ 33 | --process-monitoring-interval 600 \ 34 | --log-path "${LOG_PATH}" \ 35 | --compress-logs 36 | -------------------------------------------------------------------------------- /examples/huggingface_bart_pytorch/kubernetes/deployment/templates/service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: Service 16 | metadata: 17 | name: {{ template "selector.fullname" . }}-service 18 | namespace: {{ .Release.Namespace }} 19 | labels: 20 | app: {{ template "selector.name" . }} 21 | chart: {{template "selector.chart" . }} 22 | release: {{ .Release.Name }} 23 | heritage: {{ .Release.Service }} 24 | spec: 25 | type: {{ .Values.service.type }} 26 | ports: 27 | - port: 8000 28 | targetPort: 8000 29 | name: http 30 | - port: 8001 31 | targetPort: 8001 32 | name: grpc 33 | - port: 8002 34 | targetPort: 8002 35 | name: metrics 36 | selector: 37 | app: {{ template "selector.name" . }} 38 | release: {{ .Release.Name }} 39 | -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/kubernetes/multi-node/templates/service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: Service 16 | metadata: 17 | name: {{ template "selector.fullname" . }}-service 18 | namespace: {{ .Release.Namespace }} 19 | labels: 20 | app: {{ template "selector.name" . }} 21 | chart: {{template "selector.chart" . }} 22 | release: {{ .Release.Name }} 23 | heritage: {{ .Release.Service }} 24 | spec: 25 | type: {{ .Values.service.type }} 26 | selector: 27 | statefulset.kubernetes.io/pod-name: {{ template "selector.name" . }}-0 28 | ports: 29 | - port: 8000 30 | targetPort: 8000 31 | name: http 32 | - port: 8001 33 | targetPort: 8001 34 | name: grpc 35 | - port: 8002 36 | targetPort: 8002 37 | name: metrics 38 | -------------------------------------------------------------------------------- /pytriton/utils/dataclasses.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Collection of utils for dataclasses.""" 15 | 16 | import wrapt 17 | 18 | 19 | @wrapt.decorator 20 | def kwonly_dataclass(wrapped, instance, args, kwargs): 21 | """Poor dataclass wrapper to have init method keyword-only. 22 | 23 | Dataclass keyword-only arguments are available since Python 3.10. 24 | 25 | Example usage: 26 | 27 | @kwonly_dataclass 28 | @dataclass.dataclasses 29 | class MyDataClass: 30 | a: int 31 | b: str 32 | 33 | my_dataclass = MyDataClass(a=1, b="hello") 34 | MyDataClass(1, "hello") # raises TypeError 35 | """ 36 | if args: 37 | raise TypeError(f"{wrapped.__name__} initialization can't be used with positional arguments") 38 | return wrapped(**kwargs) 39 | -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/kubernetes/multi-node/templates/service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: Service 16 | metadata: 17 | name: {{ template "selector.fullname" . }}-service 18 | namespace: {{ .Release.Namespace }} 19 | labels: 20 | app: {{ template "selector.name" . }} 21 | chart: {{template "selector.chart" . }} 22 | release: {{ .Release.Name }} 23 | heritage: {{ .Release.Service }} 24 | spec: 25 | type: {{ .Values.service.type }} 26 | selector: 27 | statefulset.kubernetes.io/pod-name: {{ template "selector.name" . }}-0 28 | ports: 29 | - port: 8000 30 | targetPort: 8000 31 | name: http 32 | - port: 8001 33 | targetPort: 8001 34 | name: grpc 35 | - port: 8002 36 | targetPort: 8002 37 | name: metrics 38 | -------------------------------------------------------------------------------- /examples/huggingface_resnet_pytorch/kubernetes/deployment/templates/service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: Service 16 | metadata: 17 | name: {{ template "selector.fullname" . }}-service 18 | namespace: {{ .Release.Namespace }} 19 | labels: 20 | app: {{ template "selector.name" . }} 21 | chart: {{template "selector.chart" . }} 22 | release: {{ .Release.Name }} 23 | heritage: {{ .Release.Service }} 24 | spec: 25 | type: {{ .Values.service.type }} 26 | ports: 27 | - port: 8000 28 | targetPort: 8000 29 | name: http 30 | - port: 8001 31 | targetPort: 8001 32 | name: grpc 33 | - port: 8002 34 | targetPort: 8002 35 | name: metrics 36 | selector: 37 | app: {{ template "selector.name" . }} 38 | release: {{ .Release.Name }} 39 | -------------------------------------------------------------------------------- /examples/huggingface_stable_diffusion/kubernetes/deployment/templates/service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: Service 16 | metadata: 17 | name: {{ template "selector.fullname" . }}-service 18 | namespace: {{ .Release.Namespace }} 19 | labels: 20 | app: {{ template "selector.name" . }} 21 | chart: {{template "selector.chart" . }} 22 | release: {{ .Release.Name }} 23 | heritage: {{ .Release.Service }} 24 | spec: 25 | type: {{ .Values.service.type }} 26 | ports: 27 | - port: 8000 28 | targetPort: 8000 29 | name: http 30 | - port: 8001 31 | targetPort: 8001 32 | name: grpc 33 | - port: 8002 34 | targetPort: 8002 35 | name: metrics 36 | selector: 37 | app: {{ template "selector.name" . }} 38 | release: {{ .Release.Name }} 39 | -------------------------------------------------------------------------------- /examples/nemo_megatron_gpt_multinode/kubernetes/single-node/templates/service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: Service 16 | metadata: 17 | name: {{ template "selector.fullname" . }}-service 18 | namespace: {{ .Release.Namespace }} 19 | labels: 20 | app: {{ template "selector.name" . }} 21 | chart: {{template "selector.chart" . }} 22 | release: {{ .Release.Name }} 23 | heritage: {{ .Release.Service }} 24 | spec: 25 | type: {{ .Values.service.type }} 26 | ports: 27 | - port: 8000 28 | targetPort: 8000 29 | name: http 30 | - port: 8001 31 | targetPort: 8001 32 | name: grpc 33 | - port: 8002 34 | targetPort: 8002 35 | name: metrics 36 | selector: 37 | app: {{ template "selector.name" . }} 38 | release: {{ .Release.Name }} 39 | -------------------------------------------------------------------------------- /examples/huggingface_opt_multinode_jax/kubernetes/single-node/templates/service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: Service 16 | metadata: 17 | name: {{ template "selector.fullname" . }}-service 18 | namespace: {{ .Release.Namespace }} 19 | labels: 20 | app: {{ template "selector.name" . }} 21 | chart: {{template "selector.chart" . }} 22 | release: {{ .Release.Name }} 23 | heritage: {{ .Release.Service }} 24 | spec: 25 | type: {{ .Values.service.type }} 26 | ports: 27 | - port: 8000 28 | targetPort: 8000 29 | name: http 30 | - port: 8001 31 | targetPort: 8001 32 | name: grpc 33 | - port: 8002 34 | targetPort: 8002 35 | name: metrics 36 | selector: 37 | app: {{ template "selector.name" . }} 38 | release: {{ .Release.Name }} 39 | -------------------------------------------------------------------------------- /examples/mlp_random_tensorflow2/client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Client for mlp_random example.""" 16 | 17 | import logging 18 | 19 | import numpy as np 20 | 21 | from pytriton.client import ModelClient 22 | 23 | logger = logging.getLogger("examples.mlp_random_tensorflow2.client") 24 | logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(name)s: %(message)s") 25 | 26 | batch_size = 16 27 | image_size = (224, 224, 3) 28 | images_batch = np.random.uniform(size=(batch_size,) + image_size).astype(np.float32) 29 | 30 | logger.info(f"Input: {images_batch}") 31 | 32 | with ModelClient("localhost", "MLP") as client: 33 | logger.info("Sending request") 34 | result_dict = client.infer_batch(images_batch) 35 | 36 | logger.info(f"results: {result_dict}") 37 | -------------------------------------------------------------------------------- /examples/huggingface_bart_pytorch/kubernetes/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:24.08-py3 15 | ARG BUILD_FROM 16 | 17 | FROM ${FROM_IMAGE_NAME} as base 18 | WORKDIR /opt/app 19 | 20 | # Use when build PyTriton from source 21 | FROM base as install-from-dist 22 | COPY dist/*.whl /opt/app 23 | RUN pip install /opt/app/*.whl 24 | 25 | # Install from pypi 26 | FROM base as install-from-pypi 27 | RUN pip install -U nvidia-pytriton 28 | 29 | FROM install-from-${BUILD_FROM} AS image 30 | 31 | ENV PYTHONUNBUFFERED=1 32 | 33 | WORKDIR /opt/app 34 | 35 | COPY examples/huggingface_bart_pytorch/install.sh /opt/app 36 | RUN /opt/app/install.sh 37 | 38 | COPY examples/huggingface_bart_pytorch/client.py /opt/app 39 | COPY examples/huggingface_bart_pytorch/server.py /opt/app 40 | 41 | ENTRYPOINT [] --------------------------------------------------------------------------------