├── .clang-format ├── .github └── workflows │ └── pre-commit.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CMakeLists.txt ├── LICENSE ├── README.md ├── cmake ├── TritonBackendConfig.cmake.in └── define.cuda_architectures.cmake ├── docs ├── backend_platform_support_matrix.md └── python_based_backends.md ├── examples ├── README.md ├── backends │ ├── bls │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ ├── cmake │ │ │ └── TritonBLSBackendConfig.cmake.in │ │ └── src │ │ │ ├── backend.cc │ │ │ ├── bls.cc │ │ │ ├── bls.h │ │ │ ├── bls_utils.cc │ │ │ ├── bls_utils.h │ │ │ └── libtriton_bls.ldscript │ ├── minimal │ │ ├── CMakeLists.txt │ │ ├── cmake │ │ │ └── TutorialMinimalBackendConfig.cmake.in │ │ └── src │ │ │ ├── libtriton_minimal.ldscript │ │ │ └── minimal.cc │ └── recommended │ │ ├── CMakeLists.txt │ │ ├── cmake │ │ └── TutorialRecommendedBackendConfig.cmake.in │ │ └── src │ │ ├── libtriton_recommended.ldscript │ │ └── recommended.cc ├── batching_strategies │ ├── single_batching │ │ ├── CMakeLists.txt │ │ ├── cmake │ │ │ └── triton-single-batching.cmake.in │ │ └── src │ │ │ ├── libtriton_singlebatching.ldscript │ │ │ └── single_batching.cc │ └── volume_batching │ │ ├── CMakeLists.txt │ │ ├── cmake │ │ └── triton-volume-batching.cmake.in │ │ └── src │ │ ├── libtriton_volumebatching.ldscript │ │ └── volume_batching.cc ├── clients │ ├── bls_client │ ├── minimal_client │ └── recommended_client └── model_repos │ ├── bls_models │ ├── addsub_onnx │ │ ├── 1 │ │ │ └── model.onnx │ │ └── config.pbtxt │ ├── addsub_python │ │ ├── 1 │ │ │ └── model.py │ │ └── config.pbtxt │ └── bls_fp32 │ │ └── config.pbtxt │ ├── minimal_models │ ├── batching │ │ ├── 1 │ │ │ └── .gitkeep │ │ └── config.pbtxt │ └── nonbatching │ │ ├── 1 │ │ └── .gitkeep │ │ └── config.pbtxt │ └── recommended_models │ └── batching │ ├── 1 │ └── .gitkeep │ └── config.pbtxt ├── include └── triton │ └── backend │ ├── backend_common.h │ ├── backend_input_collector.h │ ├── backend_memory.h │ ├── backend_model.h │ ├── backend_model_instance.h │ ├── backend_output_responder.h │ └── device_memory_tracker.h ├── pyproject.toml └── src ├── backend_common.cc ├── backend_input_collector.cc ├── backend_memory.cc ├── backend_model.cc ├── backend_model_instance.cc ├── backend_output_responder.cc ├── device_memory_tracker.cc ├── kernel.cu └── kernel.h /.clang-format: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/.clang-format -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/.github/workflows/pre-commit.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | /.vscode 3 | *.so 4 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/CMakeLists.txt -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/README.md -------------------------------------------------------------------------------- /cmake/TritonBackendConfig.cmake.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/cmake/TritonBackendConfig.cmake.in -------------------------------------------------------------------------------- /cmake/define.cuda_architectures.cmake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/cmake/define.cuda_architectures.cmake -------------------------------------------------------------------------------- /docs/backend_platform_support_matrix.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/docs/backend_platform_support_matrix.md -------------------------------------------------------------------------------- /docs/python_based_backends.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/docs/python_based_backends.md -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/README.md -------------------------------------------------------------------------------- /examples/backends/bls/CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/bls/CMakeLists.txt -------------------------------------------------------------------------------- /examples/backends/bls/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/bls/README.md -------------------------------------------------------------------------------- /examples/backends/bls/cmake/TritonBLSBackendConfig.cmake.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/bls/cmake/TritonBLSBackendConfig.cmake.in -------------------------------------------------------------------------------- /examples/backends/bls/src/backend.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/bls/src/backend.cc -------------------------------------------------------------------------------- /examples/backends/bls/src/bls.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/bls/src/bls.cc -------------------------------------------------------------------------------- /examples/backends/bls/src/bls.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/bls/src/bls.h -------------------------------------------------------------------------------- /examples/backends/bls/src/bls_utils.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/bls/src/bls_utils.cc -------------------------------------------------------------------------------- /examples/backends/bls/src/bls_utils.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/bls/src/bls_utils.h -------------------------------------------------------------------------------- /examples/backends/bls/src/libtriton_bls.ldscript: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/bls/src/libtriton_bls.ldscript -------------------------------------------------------------------------------- /examples/backends/minimal/CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/minimal/CMakeLists.txt -------------------------------------------------------------------------------- /examples/backends/minimal/cmake/TutorialMinimalBackendConfig.cmake.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/minimal/cmake/TutorialMinimalBackendConfig.cmake.in -------------------------------------------------------------------------------- /examples/backends/minimal/src/libtriton_minimal.ldscript: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/minimal/src/libtriton_minimal.ldscript -------------------------------------------------------------------------------- /examples/backends/minimal/src/minimal.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/minimal/src/minimal.cc -------------------------------------------------------------------------------- /examples/backends/recommended/CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/recommended/CMakeLists.txt -------------------------------------------------------------------------------- /examples/backends/recommended/cmake/TutorialRecommendedBackendConfig.cmake.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/recommended/cmake/TutorialRecommendedBackendConfig.cmake.in -------------------------------------------------------------------------------- /examples/backends/recommended/src/libtriton_recommended.ldscript: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/recommended/src/libtriton_recommended.ldscript -------------------------------------------------------------------------------- /examples/backends/recommended/src/recommended.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/backends/recommended/src/recommended.cc -------------------------------------------------------------------------------- /examples/batching_strategies/single_batching/CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/batching_strategies/single_batching/CMakeLists.txt -------------------------------------------------------------------------------- /examples/batching_strategies/single_batching/cmake/triton-single-batching.cmake.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/batching_strategies/single_batching/cmake/triton-single-batching.cmake.in -------------------------------------------------------------------------------- /examples/batching_strategies/single_batching/src/libtriton_singlebatching.ldscript: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/batching_strategies/single_batching/src/libtriton_singlebatching.ldscript -------------------------------------------------------------------------------- /examples/batching_strategies/single_batching/src/single_batching.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/batching_strategies/single_batching/src/single_batching.cc -------------------------------------------------------------------------------- /examples/batching_strategies/volume_batching/CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/batching_strategies/volume_batching/CMakeLists.txt -------------------------------------------------------------------------------- /examples/batching_strategies/volume_batching/cmake/triton-volume-batching.cmake.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/batching_strategies/volume_batching/cmake/triton-volume-batching.cmake.in -------------------------------------------------------------------------------- /examples/batching_strategies/volume_batching/src/libtriton_volumebatching.ldscript: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/batching_strategies/volume_batching/src/libtriton_volumebatching.ldscript -------------------------------------------------------------------------------- /examples/batching_strategies/volume_batching/src/volume_batching.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/batching_strategies/volume_batching/src/volume_batching.cc -------------------------------------------------------------------------------- /examples/clients/bls_client: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/clients/bls_client -------------------------------------------------------------------------------- /examples/clients/minimal_client: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/clients/minimal_client -------------------------------------------------------------------------------- /examples/clients/recommended_client: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/clients/recommended_client -------------------------------------------------------------------------------- /examples/model_repos/bls_models/addsub_onnx/1/model.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/model_repos/bls_models/addsub_onnx/1/model.onnx -------------------------------------------------------------------------------- /examples/model_repos/bls_models/addsub_onnx/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/model_repos/bls_models/addsub_onnx/config.pbtxt -------------------------------------------------------------------------------- /examples/model_repos/bls_models/addsub_python/1/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/model_repos/bls_models/addsub_python/1/model.py -------------------------------------------------------------------------------- /examples/model_repos/bls_models/addsub_python/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/model_repos/bls_models/addsub_python/config.pbtxt -------------------------------------------------------------------------------- /examples/model_repos/bls_models/bls_fp32/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/model_repos/bls_models/bls_fp32/config.pbtxt -------------------------------------------------------------------------------- /examples/model_repos/minimal_models/batching/1/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/model_repos/minimal_models/batching/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/model_repos/minimal_models/batching/config.pbtxt -------------------------------------------------------------------------------- /examples/model_repos/minimal_models/nonbatching/1/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/model_repos/minimal_models/nonbatching/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/model_repos/minimal_models/nonbatching/config.pbtxt -------------------------------------------------------------------------------- /examples/model_repos/recommended_models/batching/1/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/model_repos/recommended_models/batching/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/examples/model_repos/recommended_models/batching/config.pbtxt -------------------------------------------------------------------------------- /include/triton/backend/backend_common.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/include/triton/backend/backend_common.h -------------------------------------------------------------------------------- /include/triton/backend/backend_input_collector.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/include/triton/backend/backend_input_collector.h -------------------------------------------------------------------------------- /include/triton/backend/backend_memory.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/include/triton/backend/backend_memory.h -------------------------------------------------------------------------------- /include/triton/backend/backend_model.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/include/triton/backend/backend_model.h -------------------------------------------------------------------------------- /include/triton/backend/backend_model_instance.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/include/triton/backend/backend_model_instance.h -------------------------------------------------------------------------------- /include/triton/backend/backend_output_responder.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/include/triton/backend/backend_output_responder.h -------------------------------------------------------------------------------- /include/triton/backend/device_memory_tracker.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/include/triton/backend/device_memory_tracker.h -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/pyproject.toml -------------------------------------------------------------------------------- /src/backend_common.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/src/backend_common.cc -------------------------------------------------------------------------------- /src/backend_input_collector.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/src/backend_input_collector.cc -------------------------------------------------------------------------------- /src/backend_memory.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/src/backend_memory.cc -------------------------------------------------------------------------------- /src/backend_model.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/src/backend_model.cc -------------------------------------------------------------------------------- /src/backend_model_instance.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/src/backend_model_instance.cc -------------------------------------------------------------------------------- /src/backend_output_responder.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/src/backend_output_responder.cc -------------------------------------------------------------------------------- /src/device_memory_tracker.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/src/device_memory_tracker.cc -------------------------------------------------------------------------------- /src/kernel.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/src/kernel.cu -------------------------------------------------------------------------------- /src/kernel.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/backend/HEAD/src/kernel.h --------------------------------------------------------------------------------