├── .dockerignore ├── .github └── workflows │ ├── build-container.yaml │ ├── docker-build-action.yaml │ ├── integration-test-action.yaml │ ├── integration-test.yaml │ ├── quality.yaml │ └── unit-test.yaml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── dockerfiles └── pytorch │ ├── Dockerfile │ └── Dockerfile.inf2 ├── pyproject.toml ├── scripts ├── entrypoint.sh ├── inf2_entrypoint.sh └── inf2_env.py ├── setup.cfg ├── setup.py ├── src └── huggingface_inference_toolkit │ ├── __init__.py │ ├── async_utils.py │ ├── const.py │ ├── diffusers_utils.py │ ├── env_utils.py │ ├── handler.py │ ├── logging.py │ ├── optimum_utils.py │ ├── sentence_transformers_utils.py │ ├── serialization │ ├── __init__.py │ ├── audio_utils.py │ ├── base.py │ ├── image_utils.py │ └── json_utils.py │ ├── utils.py │ ├── vertex_ai_utils.py │ └── webservice_starlette.py └── tests ├── __init__.py ├── integ ├── __init__.py ├── config.py ├── conftest.py ├── helpers.py ├── test_pytorch_local_cpu.py ├── test_pytorch_local_gpu.py ├── test_pytorch_local_inf2.py ├── test_pytorch_remote_cpu.py ├── test_pytorch_remote_gpu.py └── utils.py ├── resources ├── audio │ ├── long_sample.mp3 │ ├── sample.amr │ ├── sample.m4a │ ├── sample1.flac │ ├── sample1.mp3 │ ├── sample1.ogg │ ├── sample1.wav │ └── sample1.webm ├── custom_handler │ ├── custom_utils.py │ └── pipeline.py └── image │ ├── tiger.bmp │ ├── tiger.gif │ ├── tiger.jpeg │ ├── tiger.png │ ├── tiger.tiff │ └── tiger.webp └── unit ├── __init__.py ├── conftest.py ├── test_const.py ├── test_diffusers.py ├── test_handler.py ├── test_optimum_utils.py ├── test_sentence_transformers.py ├── test_serializer.py ├── test_utils.py └── test_vertex_ai_utils.py /.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/.dockerignore -------------------------------------------------------------------------------- /.github/workflows/build-container.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/.github/workflows/build-container.yaml -------------------------------------------------------------------------------- /.github/workflows/docker-build-action.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/.github/workflows/docker-build-action.yaml -------------------------------------------------------------------------------- /.github/workflows/integration-test-action.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/.github/workflows/integration-test-action.yaml -------------------------------------------------------------------------------- /.github/workflows/integration-test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/.github/workflows/integration-test.yaml -------------------------------------------------------------------------------- /.github/workflows/quality.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/.github/workflows/quality.yaml -------------------------------------------------------------------------------- /.github/workflows/unit-test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/.github/workflows/unit-test.yaml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/README.md -------------------------------------------------------------------------------- /dockerfiles/pytorch/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/dockerfiles/pytorch/Dockerfile -------------------------------------------------------------------------------- /dockerfiles/pytorch/Dockerfile.inf2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/dockerfiles/pytorch/Dockerfile.inf2 -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/entrypoint.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/scripts/entrypoint.sh -------------------------------------------------------------------------------- /scripts/inf2_entrypoint.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/scripts/inf2_entrypoint.sh -------------------------------------------------------------------------------- /scripts/inf2_env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/scripts/inf2_env.py -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/setup.cfg -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/setup.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/async_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/async_utils.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/const.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/const.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/diffusers_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/diffusers_utils.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/env_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/env_utils.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/handler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/handler.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/logging.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/optimum_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/optimum_utils.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/sentence_transformers_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/sentence_transformers_utils.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/serialization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/serialization/__init__.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/serialization/audio_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/serialization/audio_utils.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/serialization/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/serialization/base.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/serialization/image_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/serialization/image_utils.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/serialization/json_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/serialization/json_utils.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/utils.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/vertex_ai_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/vertex_ai_utils.py -------------------------------------------------------------------------------- /src/huggingface_inference_toolkit/webservice_starlette.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/src/huggingface_inference_toolkit/webservice_starlette.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/integ/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/integ/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/integ/config.py -------------------------------------------------------------------------------- /tests/integ/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/integ/conftest.py -------------------------------------------------------------------------------- /tests/integ/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/integ/helpers.py -------------------------------------------------------------------------------- /tests/integ/test_pytorch_local_cpu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/integ/test_pytorch_local_cpu.py -------------------------------------------------------------------------------- /tests/integ/test_pytorch_local_gpu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/integ/test_pytorch_local_gpu.py -------------------------------------------------------------------------------- /tests/integ/test_pytorch_local_inf2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/integ/test_pytorch_local_inf2.py -------------------------------------------------------------------------------- /tests/integ/test_pytorch_remote_cpu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/integ/test_pytorch_remote_cpu.py -------------------------------------------------------------------------------- /tests/integ/test_pytorch_remote_gpu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/integ/test_pytorch_remote_gpu.py -------------------------------------------------------------------------------- /tests/integ/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/integ/utils.py -------------------------------------------------------------------------------- /tests/resources/audio/long_sample.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/audio/long_sample.mp3 -------------------------------------------------------------------------------- /tests/resources/audio/sample.amr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/audio/sample.amr -------------------------------------------------------------------------------- /tests/resources/audio/sample.m4a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/audio/sample.m4a -------------------------------------------------------------------------------- /tests/resources/audio/sample1.flac: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/audio/sample1.flac -------------------------------------------------------------------------------- /tests/resources/audio/sample1.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/audio/sample1.mp3 -------------------------------------------------------------------------------- /tests/resources/audio/sample1.ogg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/audio/sample1.ogg -------------------------------------------------------------------------------- /tests/resources/audio/sample1.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/audio/sample1.wav -------------------------------------------------------------------------------- /tests/resources/audio/sample1.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/audio/sample1.webm -------------------------------------------------------------------------------- /tests/resources/custom_handler/custom_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/custom_handler/custom_utils.py -------------------------------------------------------------------------------- /tests/resources/custom_handler/pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/custom_handler/pipeline.py -------------------------------------------------------------------------------- /tests/resources/image/tiger.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/image/tiger.bmp -------------------------------------------------------------------------------- /tests/resources/image/tiger.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/image/tiger.gif -------------------------------------------------------------------------------- /tests/resources/image/tiger.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/image/tiger.jpeg -------------------------------------------------------------------------------- /tests/resources/image/tiger.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/image/tiger.png -------------------------------------------------------------------------------- /tests/resources/image/tiger.tiff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/image/tiger.tiff -------------------------------------------------------------------------------- /tests/resources/image/tiger.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/resources/image/tiger.webp -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/unit/conftest.py -------------------------------------------------------------------------------- /tests/unit/test_const.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/unit/test_const.py -------------------------------------------------------------------------------- /tests/unit/test_diffusers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/unit/test_diffusers.py -------------------------------------------------------------------------------- /tests/unit/test_handler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/unit/test_handler.py -------------------------------------------------------------------------------- /tests/unit/test_optimum_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/unit/test_optimum_utils.py -------------------------------------------------------------------------------- /tests/unit/test_sentence_transformers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/unit/test_sentence_transformers.py -------------------------------------------------------------------------------- /tests/unit/test_serializer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/unit/test_serializer.py -------------------------------------------------------------------------------- /tests/unit/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/unit/test_utils.py -------------------------------------------------------------------------------- /tests/unit/test_vertex_ai_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huggingface/huggingface-inference-toolkit/HEAD/tests/unit/test_vertex_ai_utils.py --------------------------------------------------------------------------------