├── .github └── workflows │ ├── pre-commit.yml │ ├── python-package.yaml │ └── trigger_ci.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CODEOWNERS ├── README.md ├── pyproject.toml ├── src └── triton_cli │ ├── .gitignore │ ├── __init__.py │ ├── client │ └── client.py │ ├── common.py │ ├── docker │ └── Dockerfile │ ├── main.py │ ├── metrics.py │ ├── parser.py │ ├── profile.py │ ├── repository.py │ ├── server │ ├── __init__.py │ ├── server.py │ ├── server_config.py │ ├── server_docker.py │ ├── server_factory.py │ ├── server_local.py │ └── server_utils.py │ ├── templates │ └── trt_llm │ │ ├── postprocessing │ │ ├── 1 │ │ │ └── model.py │ │ └── config.pbtxt │ │ ├── preprocessing │ │ ├── 1 │ │ │ └── model.py │ │ └── config.pbtxt │ │ ├── tensorrt_llm │ │ ├── 1 │ │ │ ├── .gitkeep │ │ │ └── model.py │ │ └── config.pbtxt │ │ └── tensorrt_llm_bls │ │ ├── 1 │ │ ├── lib │ │ │ ├── decode.py │ │ │ └── triton_decoder.py │ │ └── model.py │ │ └── config.pbtxt │ └── trt_llm │ └── engine_config_parser.py └── tests ├── conftest.py ├── test_cli.py ├── test_e2e.py ├── test_library.py ├── test_model_repository.py ├── test_models ├── add_sub │ ├── 1 │ │ └── model.py │ └── config.pbtxt └── mock_llm │ ├── 1 │ └── model.py │ └── config.pbtxt ├── trtllm-requirements.txt └── utils.py /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/.github/workflows/pre-commit.yml -------------------------------------------------------------------------------- /.github/workflows/python-package.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/.github/workflows/python-package.yaml -------------------------------------------------------------------------------- /.github/workflows/trigger_ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/.github/workflows/trigger_ci.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/CODEOWNERS -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/README.md -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/pyproject.toml -------------------------------------------------------------------------------- /src/triton_cli/.gitignore: -------------------------------------------------------------------------------- 1 | *.json 2 | *.cache 3 | -------------------------------------------------------------------------------- /src/triton_cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/__init__.py -------------------------------------------------------------------------------- /src/triton_cli/client/client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/client/client.py -------------------------------------------------------------------------------- /src/triton_cli/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/common.py -------------------------------------------------------------------------------- /src/triton_cli/docker/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/docker/Dockerfile -------------------------------------------------------------------------------- /src/triton_cli/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/main.py -------------------------------------------------------------------------------- /src/triton_cli/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/metrics.py -------------------------------------------------------------------------------- /src/triton_cli/parser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/parser.py -------------------------------------------------------------------------------- /src/triton_cli/profile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/profile.py -------------------------------------------------------------------------------- /src/triton_cli/repository.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/repository.py -------------------------------------------------------------------------------- /src/triton_cli/server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/server/__init__.py -------------------------------------------------------------------------------- /src/triton_cli/server/server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/server/server.py -------------------------------------------------------------------------------- /src/triton_cli/server/server_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/server/server_config.py -------------------------------------------------------------------------------- /src/triton_cli/server/server_docker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/server/server_docker.py -------------------------------------------------------------------------------- /src/triton_cli/server/server_factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/server/server_factory.py -------------------------------------------------------------------------------- /src/triton_cli/server/server_local.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/server/server_local.py -------------------------------------------------------------------------------- /src/triton_cli/server/server_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/server/server_utils.py -------------------------------------------------------------------------------- /src/triton_cli/templates/trt_llm/postprocessing/1/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/templates/trt_llm/postprocessing/1/model.py -------------------------------------------------------------------------------- /src/triton_cli/templates/trt_llm/postprocessing/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/templates/trt_llm/postprocessing/config.pbtxt -------------------------------------------------------------------------------- /src/triton_cli/templates/trt_llm/preprocessing/1/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/templates/trt_llm/preprocessing/1/model.py -------------------------------------------------------------------------------- /src/triton_cli/templates/trt_llm/preprocessing/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/templates/trt_llm/preprocessing/config.pbtxt -------------------------------------------------------------------------------- /src/triton_cli/templates/trt_llm/tensorrt_llm/1/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/triton_cli/templates/trt_llm/tensorrt_llm/1/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/templates/trt_llm/tensorrt_llm/1/model.py -------------------------------------------------------------------------------- /src/triton_cli/templates/trt_llm/tensorrt_llm/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/templates/trt_llm/tensorrt_llm/config.pbtxt -------------------------------------------------------------------------------- /src/triton_cli/templates/trt_llm/tensorrt_llm_bls/1/lib/decode.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/templates/trt_llm/tensorrt_llm_bls/1/lib/decode.py -------------------------------------------------------------------------------- /src/triton_cli/templates/trt_llm/tensorrt_llm_bls/1/lib/triton_decoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/templates/trt_llm/tensorrt_llm_bls/1/lib/triton_decoder.py -------------------------------------------------------------------------------- /src/triton_cli/templates/trt_llm/tensorrt_llm_bls/1/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/templates/trt_llm/tensorrt_llm_bls/1/model.py -------------------------------------------------------------------------------- /src/triton_cli/templates/trt_llm/tensorrt_llm_bls/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/templates/trt_llm/tensorrt_llm_bls/config.pbtxt -------------------------------------------------------------------------------- /src/triton_cli/trt_llm/engine_config_parser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/src/triton_cli/trt_llm/engine_config_parser.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/tests/conftest.py -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/tests/test_cli.py -------------------------------------------------------------------------------- /tests/test_e2e.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/tests/test_e2e.py -------------------------------------------------------------------------------- /tests/test_library.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/tests/test_library.py -------------------------------------------------------------------------------- /tests/test_model_repository.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/tests/test_model_repository.py -------------------------------------------------------------------------------- /tests/test_models/add_sub/1/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/tests/test_models/add_sub/1/model.py -------------------------------------------------------------------------------- /tests/test_models/add_sub/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/tests/test_models/add_sub/config.pbtxt -------------------------------------------------------------------------------- /tests/test_models/mock_llm/1/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/tests/test_models/mock_llm/1/model.py -------------------------------------------------------------------------------- /tests/test_models/mock_llm/config.pbtxt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/tests/test_models/mock_llm/config.pbtxt -------------------------------------------------------------------------------- /tests/trtllm-requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/tests/trtllm-requirements.txt -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/triton_cli/HEAD/tests/utils.py --------------------------------------------------------------------------------