├── .github └── workflows │ └── publish_docs.yml ├── .gitignore ├── .markdownlint.yaml ├── .pipelines └── build-package.yml ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── SECURITY.md ├── docs ├── Makefile ├── README.rst ├── batcher │ ├── bucket_seq_batcher.rst │ ├── bucket_seq_batcher_example.py │ ├── concat_batcher.rst │ ├── concat_batcher_example.py │ ├── customized_batcher.py │ ├── seq_batcher.rst │ ├── seq_batcher_example.py │ └── what_is_batcher.rst ├── conf.py ├── examples │ ├── benchmark.py │ ├── bert_embedding.py │ ├── bert_embedding.rst │ ├── bert_embedding_benchmark.py │ ├── fully_connected.py │ ├── gpt2_baseline.py │ ├── gpt2_completion.py │ ├── gpt2_completion_benchmark.py │ └── gpt_completion.rst ├── figures │ ├── batching_overview.png │ ├── model_host.png │ ├── model_host_class.png │ └── remote_model_host.png ├── index.rst ├── make.bat ├── model_host.rst └── remote_model_host.rst ├── isort.cfg ├── pyproject.toml ├── src ├── __init__.py └── batch_inference │ ├── __init__.py │ ├── aio │ ├── __init__.py │ ├── batch_context.py │ ├── model_host.py │ └── remote_model_host │ │ ├── __init__.py │ │ ├── model_host.proto │ │ ├── model_host_client.py │ │ ├── model_host_pb2.py │ │ ├── model_host_pb2.pyi │ │ ├── model_host_pb2_grpc.py │ │ ├── model_host_server.py │ │ ├── msgpack_serialization.py │ │ ├── readme.txt │ │ └── remote_model_host.py │ ├── batch_context.py │ ├── batcher │ ├── __init__.py │ ├── batcher.py │ ├── bucket_seq_batcher.py │ ├── concat_batcher.py │ ├── multi_batcher.py │ ├── seq_batcher.py │ ├── tensor_ops_np.py │ └── tensor_ops_pt.py │ ├── decorators.py │ ├── logger.py │ ├── model_host.py │ └── remote_model_host.py └── tests ├── gen_test_model.py ├── matmul.onnx ├── test_bucket_seq_batcher.py ├── test_decorator.py ├── test_model_host.py ├── test_model_host_aio.py ├── test_ort_model.py ├── test_remote_model_host.py ├── test_start_twice.py ├── test_torch_model.py └── test_torch_rnn_batcher.py /.github/workflows/publish_docs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/.github/workflows/publish_docs.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/.gitignore -------------------------------------------------------------------------------- /.markdownlint.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/.markdownlint.yaml -------------------------------------------------------------------------------- /.pipelines/build-package.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/.pipelines/build-package.yml -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/.readthedocs.yaml -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/README.md -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/SECURITY.md -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/README.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/README.rst -------------------------------------------------------------------------------- /docs/batcher/bucket_seq_batcher.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/batcher/bucket_seq_batcher.rst -------------------------------------------------------------------------------- /docs/batcher/bucket_seq_batcher_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/batcher/bucket_seq_batcher_example.py -------------------------------------------------------------------------------- /docs/batcher/concat_batcher.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/batcher/concat_batcher.rst -------------------------------------------------------------------------------- /docs/batcher/concat_batcher_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/batcher/concat_batcher_example.py -------------------------------------------------------------------------------- /docs/batcher/customized_batcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/batcher/customized_batcher.py -------------------------------------------------------------------------------- /docs/batcher/seq_batcher.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/batcher/seq_batcher.rst -------------------------------------------------------------------------------- /docs/batcher/seq_batcher_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/batcher/seq_batcher_example.py -------------------------------------------------------------------------------- /docs/batcher/what_is_batcher.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/batcher/what_is_batcher.rst -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/conf.py -------------------------------------------------------------------------------- /docs/examples/benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/examples/benchmark.py -------------------------------------------------------------------------------- /docs/examples/bert_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/examples/bert_embedding.py -------------------------------------------------------------------------------- /docs/examples/bert_embedding.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/examples/bert_embedding.rst -------------------------------------------------------------------------------- /docs/examples/bert_embedding_benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/examples/bert_embedding_benchmark.py -------------------------------------------------------------------------------- /docs/examples/fully_connected.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/examples/fully_connected.py -------------------------------------------------------------------------------- /docs/examples/gpt2_baseline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/examples/gpt2_baseline.py -------------------------------------------------------------------------------- /docs/examples/gpt2_completion.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/examples/gpt2_completion.py -------------------------------------------------------------------------------- /docs/examples/gpt2_completion_benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/examples/gpt2_completion_benchmark.py -------------------------------------------------------------------------------- /docs/examples/gpt_completion.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/examples/gpt_completion.rst -------------------------------------------------------------------------------- /docs/figures/batching_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/figures/batching_overview.png -------------------------------------------------------------------------------- /docs/figures/model_host.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/figures/model_host.png -------------------------------------------------------------------------------- /docs/figures/model_host_class.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/figures/model_host_class.png -------------------------------------------------------------------------------- /docs/figures/remote_model_host.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/figures/remote_model_host.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/index.rst -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/make.bat -------------------------------------------------------------------------------- /docs/model_host.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/model_host.rst -------------------------------------------------------------------------------- /docs/remote_model_host.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/docs/remote_model_host.rst -------------------------------------------------------------------------------- /isort.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/isort.cfg -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/pyproject.toml -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/__init__.py -------------------------------------------------------------------------------- /src/batch_inference/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/__init__.py -------------------------------------------------------------------------------- /src/batch_inference/aio/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/aio/__init__.py -------------------------------------------------------------------------------- /src/batch_inference/aio/batch_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/aio/batch_context.py -------------------------------------------------------------------------------- /src/batch_inference/aio/model_host.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/aio/model_host.py -------------------------------------------------------------------------------- /src/batch_inference/aio/remote_model_host/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/aio/remote_model_host/__init__.py -------------------------------------------------------------------------------- /src/batch_inference/aio/remote_model_host/model_host.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/aio/remote_model_host/model_host.proto -------------------------------------------------------------------------------- /src/batch_inference/aio/remote_model_host/model_host_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/aio/remote_model_host/model_host_client.py -------------------------------------------------------------------------------- /src/batch_inference/aio/remote_model_host/model_host_pb2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/aio/remote_model_host/model_host_pb2.py -------------------------------------------------------------------------------- /src/batch_inference/aio/remote_model_host/model_host_pb2.pyi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/aio/remote_model_host/model_host_pb2.pyi -------------------------------------------------------------------------------- /src/batch_inference/aio/remote_model_host/model_host_pb2_grpc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/aio/remote_model_host/model_host_pb2_grpc.py -------------------------------------------------------------------------------- /src/batch_inference/aio/remote_model_host/model_host_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/aio/remote_model_host/model_host_server.py -------------------------------------------------------------------------------- /src/batch_inference/aio/remote_model_host/msgpack_serialization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/aio/remote_model_host/msgpack_serialization.py -------------------------------------------------------------------------------- /src/batch_inference/aio/remote_model_host/readme.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/aio/remote_model_host/readme.txt -------------------------------------------------------------------------------- /src/batch_inference/aio/remote_model_host/remote_model_host.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/aio/remote_model_host/remote_model_host.py -------------------------------------------------------------------------------- /src/batch_inference/batch_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/batch_context.py -------------------------------------------------------------------------------- /src/batch_inference/batcher/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/batcher/__init__.py -------------------------------------------------------------------------------- /src/batch_inference/batcher/batcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/batcher/batcher.py -------------------------------------------------------------------------------- /src/batch_inference/batcher/bucket_seq_batcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/batcher/bucket_seq_batcher.py -------------------------------------------------------------------------------- /src/batch_inference/batcher/concat_batcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/batcher/concat_batcher.py -------------------------------------------------------------------------------- /src/batch_inference/batcher/multi_batcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/batcher/multi_batcher.py -------------------------------------------------------------------------------- /src/batch_inference/batcher/seq_batcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/batcher/seq_batcher.py -------------------------------------------------------------------------------- /src/batch_inference/batcher/tensor_ops_np.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/batcher/tensor_ops_np.py -------------------------------------------------------------------------------- /src/batch_inference/batcher/tensor_ops_pt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/batcher/tensor_ops_pt.py -------------------------------------------------------------------------------- /src/batch_inference/decorators.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/decorators.py -------------------------------------------------------------------------------- /src/batch_inference/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/logger.py -------------------------------------------------------------------------------- /src/batch_inference/model_host.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/model_host.py -------------------------------------------------------------------------------- /src/batch_inference/remote_model_host.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/src/batch_inference/remote_model_host.py -------------------------------------------------------------------------------- /tests/gen_test_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/tests/gen_test_model.py -------------------------------------------------------------------------------- /tests/matmul.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/tests/matmul.onnx -------------------------------------------------------------------------------- /tests/test_bucket_seq_batcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/tests/test_bucket_seq_batcher.py -------------------------------------------------------------------------------- /tests/test_decorator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/tests/test_decorator.py -------------------------------------------------------------------------------- /tests/test_model_host.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/tests/test_model_host.py -------------------------------------------------------------------------------- /tests/test_model_host_aio.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/tests/test_model_host_aio.py -------------------------------------------------------------------------------- /tests/test_ort_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/tests/test_ort_model.py -------------------------------------------------------------------------------- /tests/test_remote_model_host.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/tests/test_remote_model_host.py -------------------------------------------------------------------------------- /tests/test_start_twice.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/tests/test_start_twice.py -------------------------------------------------------------------------------- /tests/test_torch_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/tests/test_torch_model.py -------------------------------------------------------------------------------- /tests/test_torch_rnn_batcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/batch-inference/HEAD/tests/test_torch_rnn_batcher.py --------------------------------------------------------------------------------