├── .gitignore ├── 00_load_and_infer.py ├── 01_pytorch_latency_benchmark.py ├── 02_convert_to_onnx.py ├── 03_onnx_cpu_inference.py ├── 04_onnx_cuda_inference.py ├── 05_onnx_trt_inference.py ├── 06_export_preprocessing_onnx.py ├── 07_onnx_compose_merge.py ├── 08_inference_merged_model.py ├── 09_video_inference.py ├── 10_gradio_app.py ├── README.md ├── beignets-task-guide.png ├── builtin_prepost_onnxruntime_extensions.py ├── imagenet_classes.py ├── make_preprocessing_onnx.py ├── merge_preprocessing_to_model.py ├── notebooks ├── 00_benchmark_timm.ipynb ├── 01_convert_to_onnx.ipynb ├── 02_benchmark_onnxruntime.ipynb ├── 03_benchmark_onnxruntime_trt.ipynb ├── 04_onnx_preprocessing_input.ipynb ├── export_onnx_prepostproc.ipynb └── onnxruntime_extension_prepost.ipynb ├── output.mp4 ├── requirements.txt ├── run_trt.py └── sample.mp4 /.gitignore: -------------------------------------------------------------------------------- 1 | *.onnx 2 | trt_cache/ 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | share/python-wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .nox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | *.py,cover 53 | .hypothesis/ 54 | .pytest_cache/ 55 | cover/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | .pybuilder/ 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | # For a library or package, you might want to ignore these files since the code is 90 | # intended to run in multiple environments; otherwise, check them in: 91 | # .python-version 92 | 93 | # pipenv 94 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 95 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 96 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 97 | # install all needed dependencies. 98 | #Pipfile.lock 99 | 100 | # poetry 101 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 102 | # This is especially recommended for binary packages to ensure reproducibility, and is more 103 | # commonly ignored for libraries. 104 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 105 | #poetry.lock 106 | 107 | # pdm 108 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 109 | #pdm.lock 110 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 111 | # in version control. 112 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 113 | .pdm.toml 114 | .pdm-python 115 | .pdm-build/ 116 | 117 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 118 | __pypackages__/ 119 | 120 | # Celery stuff 121 | celerybeat-schedule 122 | celerybeat.pid 123 | 124 | # SageMath parsed files 125 | *.sage.py 126 | 127 | # Environments 128 | .env 129 | .venv 130 | env/ 131 | venv/ 132 | ENV/ 133 | env.bak/ 134 | venv.bak/ 135 | 136 | # Spyder project settings 137 | .spyderproject 138 | .spyproject 139 | 140 | # Rope project settings 141 | .ropeproject 142 | 143 | # mkdocs documentation 144 | /site 145 | 146 | # mypy 147 | .mypy_cache/ 148 | .dmypy.json 149 | dmypy.json 150 | 151 | # Pyre type checker 152 | .pyre/ 153 | 154 | # pytype static type analyzer 155 | .pytype/ 156 | 157 | # Cython debug symbols 158 | cython_debug/ 159 | 160 | # PyCharm 161 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 162 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 163 | # and can be added to the global gitignore or merged into this file. For a more nuclear 164 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 165 | #.idea/ -------------------------------------------------------------------------------- /00_load_and_infer.py: -------------------------------------------------------------------------------- 1 | from urllib.request import urlopen 2 | 3 | import timm 4 | import torch 5 | from PIL import Image 6 | 7 | from imagenet_classes import IMAGENET2012_CLASSES 8 | 9 | if __name__ == "__main__": 10 | model_name = "eva02_large_patch14_448.mim_m38m_ft_in22k_in1k" 11 | model = timm.create_model(model_name, pretrained=True).eval() 12 | 13 | data_config = timm.data.resolve_model_data_config(model) 14 | transforms = timm.data.create_transform(**data_config, is_training=False) 15 | 16 | img = Image.open( 17 | urlopen( 18 | "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png" 19 | ) 20 | ) 21 | 22 | with torch.inference_mode(): 23 | output = model(transforms(img).unsqueeze(0)) 24 | 25 | top5_probabilities, top5_class_indices = torch.topk( 26 | output.softmax(dim=1) * 100, k=5 27 | ) 28 | im_classes = list(IMAGENET2012_CLASSES.values()) 29 | class_names = [im_classes[i] for i in top5_class_indices[0]] 30 | 31 | print("Top 5 predictions:") 32 | for name, prob in zip(class_names, top5_probabilities[0]): 33 | print(f" {name}: {prob:.2f}%") 34 | -------------------------------------------------------------------------------- /01_pytorch_latency_benchmark.py: -------------------------------------------------------------------------------- 1 | import time 2 | from urllib.request import urlopen 3 | 4 | import timm 5 | import torch 6 | from PIL import Image 7 | 8 | model_name = "eva02_large_patch14_448.mim_m38m_ft_in22k_in1k" 9 | model = timm.create_model(model_name, pretrained=True).eval() 10 | 11 | data_config = timm.data.resolve_model_data_config(model) 12 | transforms = timm.data.create_transform(**data_config, is_training=False) 13 | 14 | img = Image.open( 15 | urlopen( 16 | "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png" 17 | ) 18 | ) 19 | 20 | 21 | def run_benchmark(model, device, num_images=10): 22 | model = model.to(device) 23 | 24 | with torch.inference_mode(): 25 | start = time.perf_counter() 26 | for _ in range(num_images): 27 | input_tensor = transforms(img).unsqueeze(0).to(device) 28 | model(input_tensor) 29 | end = time.perf_counter() 30 | 31 | ms_per_image = (end - start) / num_images * 1000 32 | fps = num_images / (end - start) 33 | 34 | print(f"PyTorch model on {device}: {ms_per_image:.3f} ms per image, FPS: {fps:.2f}") 35 | 36 | 37 | if __name__ == "__main__": 38 | run_benchmark(model, "cpu") 39 | run_benchmark(model, "cuda") 40 | -------------------------------------------------------------------------------- /02_convert_to_onnx.py: -------------------------------------------------------------------------------- 1 | import timm 2 | import torch 3 | 4 | model = timm.create_model( 5 | "eva02_large_patch14_448.mim_m38m_ft_in22k_in1k", pretrained=True 6 | ).eval() 7 | 8 | onnx_filename = "eva02_large_patch14_448.onnx" 9 | torch.onnx.export( 10 | model, 11 | torch.randn(1, 3, 448, 448), 12 | onnx_filename, 13 | export_params=True, 14 | opset_version=20, 15 | do_constant_folding=True, 16 | input_names=["input"], 17 | output_names=["output"], 18 | dynamic_axes={"input": {0: "batch_size"}, "output": {0: "batch_size"}}, 19 | ) 20 | -------------------------------------------------------------------------------- /03_onnx_cpu_inference.py: -------------------------------------------------------------------------------- 1 | import time 2 | from urllib.request import urlopen 3 | 4 | import numpy as np 5 | import onnxruntime as ort 6 | import torch 7 | from PIL import Image 8 | 9 | from imagenet_classes import IMAGENET2012_CLASSES 10 | 11 | img = Image.open( 12 | urlopen( 13 | "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png" 14 | ) 15 | ) 16 | 17 | 18 | def transforms_numpy(image: Image.Image): 19 | image = image.convert("RGB") 20 | image = image.resize((448, 448), Image.BICUBIC) 21 | img_numpy = np.array(image).astype(np.float32) / 255.0 22 | img_numpy = img_numpy.transpose(2, 0, 1) 23 | mean = np.array([0.4815, 0.4578, 0.4082]).reshape(-1, 1, 1) 24 | std = np.array([0.2686, 0.2613, 0.2758]).reshape(-1, 1, 1) 25 | img_numpy = (img_numpy - mean) / std 26 | img_numpy = np.expand_dims(img_numpy, axis=0) 27 | img_numpy = img_numpy.astype(np.float32) 28 | return img_numpy 29 | 30 | 31 | # Create ONNX Runtime session with CPU provider 32 | onnx_filename = "eva02_large_patch14_448.onnx" 33 | session = ort.InferenceSession(onnx_filename, providers=["CPUExecutionProvider"]) 34 | 35 | # Get input and output names 36 | input_name = session.get_inputs()[0].name 37 | output_name = session.get_outputs()[0].name 38 | 39 | # Run inference 40 | output = session.run([output_name], {input_name: transforms_numpy(img)})[0] 41 | 42 | # Check the output 43 | output = torch.from_numpy(output) 44 | top5_probabilities, top5_class_indices = torch.topk(output.softmax(dim=1) * 100, k=5) 45 | 46 | im_classes = list(IMAGENET2012_CLASSES.values()) 47 | class_names = [im_classes[i] for i in top5_class_indices[0]] 48 | 49 | # Print class names and probabilities 50 | for name, prob in zip(class_names, top5_probabilities[0]): 51 | print(f"{name}: {prob:.2f}%") 52 | 53 | # Run benchmark 54 | num_images = 10 55 | start = time.perf_counter() 56 | for i in range(num_images): 57 | output = session.run([output_name], {input_name: transforms_numpy(img)})[0] 58 | end = time.perf_counter() 59 | time_taken = end - start 60 | 61 | ms_per_image = time_taken / num_images * 1000 62 | fps = num_images / time_taken 63 | 64 | print(f"Onnxruntime CPU: {ms_per_image:.3f} ms per image, FPS: {fps:.2f}") 65 | -------------------------------------------------------------------------------- /04_onnx_cuda_inference.py: -------------------------------------------------------------------------------- 1 | import time 2 | from urllib.request import urlopen 3 | 4 | import cupy as cp 5 | import numpy as np 6 | import onnxruntime as ort 7 | import torch 8 | from PIL import Image 9 | 10 | from imagenet_classes import IMAGENET2012_CLASSES 11 | 12 | img = Image.open( 13 | urlopen( 14 | "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png" 15 | ) 16 | ) 17 | 18 | 19 | def transforms_numpy(image: Image.Image): 20 | image = image.convert("RGB") 21 | image = image.resize((448, 448), Image.BICUBIC) 22 | img_numpy = np.array(image).astype(np.float32) / 255.0 23 | img_numpy = img_numpy.transpose(2, 0, 1) 24 | mean = np.array([0.4815, 0.4578, 0.4082]).reshape(-1, 1, 1) 25 | std = np.array([0.2686, 0.2613, 0.2758]).reshape(-1, 1, 1) 26 | img_numpy = (img_numpy - mean) / std 27 | img_numpy = np.expand_dims(img_numpy, axis=0) 28 | img_numpy = img_numpy.astype(np.float32) 29 | return img_numpy 30 | 31 | 32 | def transforms_cupy(image: Image.Image): 33 | # Convert image to RGB and resize 34 | image = image.convert("RGB") 35 | image = image.resize((448, 448), Image.BICUBIC) 36 | 37 | # Convert to CuPy array and normalize 38 | img_cupy = cp.array(image, dtype=cp.float32) / 255.0 39 | img_cupy = img_cupy.transpose(2, 0, 1) 40 | 41 | # Apply mean and std normalization 42 | mean = cp.array([0.4815, 0.4578, 0.4082], dtype=cp.float32).reshape(-1, 1, 1) 43 | std = cp.array([0.2686, 0.2613, 0.2758], dtype=cp.float32).reshape(-1, 1, 1) 44 | img_cupy = (img_cupy - mean) / std 45 | 46 | # Add batch dimension 47 | img_cupy = cp.expand_dims(img_cupy, axis=0) 48 | 49 | return img_cupy 50 | 51 | 52 | # Create ONNX Runtime session with CPU provider 53 | onnx_filename = "eva02_large_patch14_448.onnx" 54 | session = ort.InferenceSession(onnx_filename, providers=["CUDAExecutionProvider"]) 55 | 56 | # Get input and output names 57 | input_name = session.get_inputs()[0].name 58 | output_name = session.get_outputs()[0].name 59 | 60 | # Run inference 61 | output = session.run([output_name], {input_name: transforms_numpy(img)})[0] 62 | 63 | 64 | # Check the output 65 | output = torch.from_numpy(output) 66 | top5_probabilities, top5_class_indices = torch.topk(output.softmax(dim=1) * 100, k=5) 67 | 68 | im_classes = list(IMAGENET2012_CLASSES.values()) 69 | class_names = [im_classes[i] for i in top5_class_indices[0]] 70 | 71 | # Print class names and probabilities 72 | for name, prob in zip(class_names, top5_probabilities[0]): 73 | print(f"{name}: {prob:.2f}%") 74 | 75 | # Run benchmark numpy 76 | num_images = 100 77 | start = time.perf_counter() 78 | for i in range(num_images): 79 | output = session.run([output_name], {input_name: transforms_numpy(img)})[0] 80 | end = time.perf_counter() 81 | time_taken = end - start 82 | 83 | ms_per_image = time_taken / num_images * 1000 84 | fps = num_images / time_taken 85 | 86 | print( 87 | f"Onnxruntime CUDA numpy transforms: {ms_per_image:.3f} ms per image, FPS: {fps:.2f}" 88 | ) 89 | 90 | 91 | # Run benchmark cupy 92 | num_images = 100 93 | start = time.perf_counter() 94 | for i in range(num_images): 95 | img_cupy = transforms_cupy(img) 96 | output = session.run([output_name], {input_name: cp.asnumpy(img_cupy)})[0] 97 | end = time.perf_counter() 98 | time_taken = end - start 99 | 100 | ms_per_image = time_taken / num_images * 1000 101 | fps = num_images / time_taken 102 | 103 | print( 104 | f"Onnxruntime CUDA cupy transforms: {ms_per_image:.3f} ms per image, FPS: {fps:.2f}" 105 | ) 106 | -------------------------------------------------------------------------------- /05_onnx_trt_inference.py: -------------------------------------------------------------------------------- 1 | import time 2 | from urllib.request import urlopen 3 | 4 | import cupy as cp 5 | import numpy as np 6 | import onnxruntime as ort 7 | from PIL import Image 8 | 9 | img = Image.open( 10 | urlopen( 11 | "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png" 12 | ) 13 | ) 14 | 15 | 16 | def transforms_numpy(image: Image.Image): 17 | image = image.convert("RGB") 18 | image = image.resize((448, 448), Image.BICUBIC) 19 | img_numpy = np.array(image).astype(np.float32) / 255.0 20 | img_numpy = img_numpy.transpose(2, 0, 1) 21 | mean = np.array([0.4815, 0.4578, 0.4082]).reshape(-1, 1, 1) 22 | std = np.array([0.2686, 0.2613, 0.2758]).reshape(-1, 1, 1) 23 | img_numpy = (img_numpy - mean) / std 24 | img_numpy = np.expand_dims(img_numpy, axis=0) 25 | img_numpy = img_numpy.astype(np.float32) 26 | return img_numpy 27 | 28 | 29 | def transforms_cupy(image: Image.Image): 30 | # Convert image to RGB and resize 31 | image = image.convert("RGB") 32 | image = image.resize((448, 448), Image.BICUBIC) 33 | 34 | # Convert to CuPy array and normalize 35 | img_cupy = cp.array(image, dtype=cp.float32) / 255.0 36 | img_cupy = img_cupy.transpose(2, 0, 1) 37 | 38 | # Apply mean and std normalization 39 | mean = cp.array([0.4815, 0.4578, 0.4082], dtype=cp.float32).reshape(-1, 1, 1) 40 | std = cp.array([0.2686, 0.2613, 0.2758], dtype=cp.float32).reshape(-1, 1, 1) 41 | img_cupy = (img_cupy - mean) / std 42 | 43 | # Add batch dimension 44 | img_cupy = cp.expand_dims(img_cupy, axis=0) 45 | 46 | return img_cupy 47 | 48 | 49 | # Create ONNX Runtime session with CPU provider 50 | onnx_filename = "eva02_large_patch14_448.onnx" 51 | providers = [ 52 | ( 53 | "TensorrtExecutionProvider", 54 | { 55 | "device_id": 0, 56 | "trt_max_workspace_size": 8589934592, 57 | "trt_fp16_enable": True, 58 | "trt_engine_cache_enable": True, 59 | "trt_engine_cache_path": "./trt_cache", 60 | "trt_force_sequential_engine_build": False, 61 | "trt_max_partition_iterations": 10000, 62 | "trt_min_subgraph_size": 1, 63 | "trt_builder_optimization_level": 5, 64 | "trt_timing_cache_enable": True, 65 | }, 66 | ), 67 | ] 68 | session = ort.InferenceSession(onnx_filename, providers=providers) 69 | 70 | # Get input and output names 71 | input_name = session.get_inputs()[0].name 72 | output_name = session.get_outputs()[0].name 73 | 74 | # Run inference 75 | output = session.run([output_name], {input_name: transforms_numpy(img)})[0] 76 | 77 | # Run benchmark numpy 78 | num_images = 100 79 | start = time.perf_counter() 80 | for i in range(num_images): 81 | output = session.run([output_name], {input_name: transforms_numpy(img)})[0] 82 | end = time.perf_counter() 83 | time_taken = end - start 84 | 85 | ms_per_image = time_taken / num_images * 1000 86 | fps = num_images / time_taken 87 | 88 | print(f"TensorRT + numpy: {ms_per_image:.3f} ms per image, FPS: {fps:.2f}") 89 | 90 | 91 | # Run benchmark cupy 92 | num_images = 100 93 | start = time.perf_counter() 94 | for i in range(num_images): 95 | img_cupy = transforms_cupy(img) 96 | output = session.run([output_name], {input_name: cp.asnumpy(img_cupy)})[0] 97 | end = time.perf_counter() 98 | time_taken = end - start 99 | 100 | ms_per_image = time_taken / num_images * 1000 101 | fps = num_images / time_taken 102 | 103 | print(f"TensorRT + cupy : {ms_per_image:.3f} ms per image, FPS: {fps:.2f}") 104 | -------------------------------------------------------------------------------- /06_export_preprocessing_onnx.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import onnx 4 | import torch 5 | import torch.nn as nn 6 | from onnxsim import simplify 7 | 8 | 9 | class Preprocess(nn.Module): 10 | def __init__(self, input_shape: List[int]): 11 | super(Preprocess, self).__init__() 12 | self.input_shape = tuple(input_shape) 13 | self.mean = torch.tensor([0.4815, 0.4578, 0.4082]).view(1, 3, 1, 1) 14 | self.std = torch.tensor([0.2686, 0.2613, 0.2758]).view(1, 3, 1, 1) 15 | 16 | def forward(self, x: torch.Tensor): 17 | x = torch.nn.functional.interpolate( 18 | input=x, 19 | size=self.input_shape[2:], 20 | ) 21 | x = x / 255.0 22 | x = (x - self.mean) / self.std 23 | 24 | return x 25 | 26 | 27 | if __name__ == "__main__": 28 | input_shape = [1, 3, 448, 448] 29 | output_onnx_file = "preprocessing.onnx" 30 | model = Preprocess(input_shape=input_shape) 31 | 32 | torch.onnx.export( 33 | model, 34 | torch.randn(input_shape), 35 | output_onnx_file, 36 | opset_version=20, 37 | input_names=["input_rgb"], 38 | output_names=["output_preprocessing"], 39 | dynamic_axes={ 40 | "input_rgb": { 41 | 0: "batch_size", 42 | 2: "height", 43 | 3: "width", 44 | }, 45 | }, 46 | ) 47 | 48 | model_onnx = onnx.load(output_onnx_file) 49 | model_simplified, _ = simplify(model_onnx) 50 | onnx.save(model_simplified, output_onnx_file) 51 | -------------------------------------------------------------------------------- /07_onnx_compose_merge.py: -------------------------------------------------------------------------------- 1 | import onnx 2 | from onnx import compose 3 | 4 | # Load the models 5 | model1 = onnx.load("preprocessing.onnx") 6 | model2 = onnx.load("eva02_large_patch14_448.onnx") 7 | 8 | # Merge the models 9 | merged_model = compose.merge_models( 10 | model1, 11 | model2, 12 | io_map=[("output_preprocessing", "input")], 13 | prefix1="preprocessing_", 14 | prefix2="model_", 15 | doc_string="Merged preprocessing and eva02_large_patch14_448 model", 16 | producer_name="dickson.neoh@gmail.com using onnx compose", 17 | ) 18 | 19 | # Save the merged model 20 | onnx.save(merged_model, "merged_model_compose.onnx") 21 | -------------------------------------------------------------------------------- /08_inference_merged_model.py: -------------------------------------------------------------------------------- 1 | import time 2 | from urllib.request import urlopen 3 | 4 | import cupy as cp 5 | import numpy as np 6 | import onnxruntime as ort 7 | import torch 8 | from PIL import Image 9 | 10 | from imagenet_classes import IMAGENET2012_CLASSES 11 | 12 | img = Image.open( 13 | urlopen( 14 | "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png" 15 | ) 16 | ) 17 | 18 | 19 | def read_image(image: Image.Image): 20 | image = image.convert("RGB") 21 | img_numpy = np.array(image).astype(np.float32) 22 | img_numpy = img_numpy.transpose(2, 0, 1) 23 | img_numpy = np.expand_dims(img_numpy, axis=0) 24 | return img_numpy 25 | 26 | 27 | providers = [ 28 | ( 29 | "TensorrtExecutionProvider", 30 | { 31 | "device_id": 0, 32 | "trt_max_workspace_size": 8589934592, 33 | "trt_fp16_enable": True, 34 | "trt_engine_cache_enable": True, 35 | "trt_engine_cache_path": "./trt_cache", 36 | "trt_force_sequential_engine_build": False, 37 | "trt_max_partition_iterations": 10000, 38 | "trt_min_subgraph_size": 1, 39 | "trt_builder_optimization_level": 5, 40 | "trt_timing_cache_enable": True, 41 | }, 42 | ), 43 | ] 44 | 45 | session = ort.InferenceSession("merged_model_compose.onnx", providers=providers) 46 | 47 | input_name = session.get_inputs()[0].name 48 | output_name = session.get_outputs()[0].name 49 | 50 | output = session.run([output_name], {input_name: read_image(img)}) 51 | 52 | # print(output[0]) 53 | 54 | # Check the output 55 | output = torch.from_numpy(output[0]) 56 | print(output.shape) 57 | 58 | 59 | top5_probabilities, top5_class_indices = torch.topk(output.softmax(dim=1) * 100, k=5) 60 | 61 | im_classes = list(IMAGENET2012_CLASSES.values()) 62 | class_names = [im_classes[i] for i in top5_class_indices[0]] 63 | 64 | # Print class names and probabilities 65 | for name, prob in zip(class_names, top5_probabilities[0]): 66 | print(f"{name}: {prob:.2f}%") 67 | 68 | num_images = 1000 69 | start = time.perf_counter() 70 | for i in range(num_images): 71 | output = session.run([output_name], {input_name: read_image(img)}) 72 | end = time.perf_counter() 73 | time_taken = end - start 74 | 75 | ms_per_image = time_taken / num_images * 1000 76 | fps = num_images / time_taken 77 | 78 | print(f"Onnxruntime TensorRT: {ms_per_image:.3f} ms per image, FPS: {fps:.2f}") 79 | -------------------------------------------------------------------------------- /09_video_inference.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import time 3 | 4 | import cv2 5 | import numpy as np 6 | import onnxruntime as ort 7 | 8 | from imagenet_classes import IMAGENET2012_CLASSES 9 | 10 | 11 | def parse_arguments(): 12 | parser = argparse.ArgumentParser(description="Video inference with TensorRT") 13 | parser.add_argument("--output_video", type=str, help="Path to output video file") 14 | parser.add_argument("--input_video", type=str, help="Path to input video file") 15 | parser.add_argument("--webcam", action="store_true", help="Use webcam as input") 16 | parser.add_argument( 17 | "--live", action="store_true", help="View video live during inference" 18 | ) 19 | return parser.parse_args() 20 | 21 | 22 | def get_ort_session(model_path): 23 | providers = [ 24 | ( 25 | "TensorrtExecutionProvider", 26 | { 27 | "device_id": 0, 28 | "trt_max_workspace_size": 8589934592, 29 | "trt_fp16_enable": True, 30 | "trt_engine_cache_enable": True, 31 | "trt_engine_cache_path": "./trt_cache", 32 | "trt_force_sequential_engine_build": False, 33 | "trt_max_partition_iterations": 10000, 34 | "trt_min_subgraph_size": 1, 35 | "trt_builder_optimization_level": 5, 36 | "trt_timing_cache_enable": True, 37 | }, 38 | ), 39 | ] 40 | return ort.InferenceSession(model_path, providers=providers) 41 | 42 | 43 | def preprocess_frame(frame): 44 | # Use cv2 for resizing instead of PIL for better performance 45 | resized = cv2.resize(frame, (448, 448), interpolation=cv2.INTER_LINEAR) 46 | img_numpy = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB).astype(np.float32) 47 | img_numpy = img_numpy.transpose(2, 0, 1) 48 | img_numpy = np.expand_dims(img_numpy, axis=0) 49 | return img_numpy 50 | 51 | 52 | def get_top_predictions(output, top_k=5): 53 | # Apply softmax 54 | exp_output = np.exp(output - np.max(output, axis=1, keepdims=True)) 55 | probabilities = exp_output / np.sum(exp_output, axis=1, keepdims=True) 56 | 57 | # Get top k indices and probabilities 58 | top_indices = np.argsort(probabilities[0])[-top_k:][::-1] 59 | top_probs = probabilities[0][top_indices] * 100 60 | 61 | im_classes = list(IMAGENET2012_CLASSES.values()) 62 | class_names = [im_classes[i] for i in top_indices] 63 | 64 | return list(zip(class_names, top_probs.tolist())) 65 | 66 | 67 | def draw_predictions(frame, predictions, fps): 68 | # Draw FPS in the top right corner with dark blue background 69 | fps_text = f"FPS: {fps:.2f}" 70 | (text_width, text_height), _ = cv2.getTextSize( 71 | fps_text, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2 72 | ) 73 | text_offset_x = frame.shape[1] - text_width - 10 74 | text_offset_y = 30 75 | box_coords = ( 76 | (text_offset_x - 5, text_offset_y + 5), 77 | (text_offset_x + text_width + 5, text_offset_y - text_height - 5), 78 | ) 79 | cv2.rectangle( 80 | frame, box_coords[0], box_coords[1], (139, 0, 0), cv2.FILLED 81 | ) # Dark blue background 82 | cv2.putText( 83 | frame, 84 | fps_text, 85 | (text_offset_x, text_offset_y), 86 | cv2.FONT_HERSHEY_SIMPLEX, 87 | 0.7, 88 | (255, 255, 255), # White text 89 | 2, 90 | ) 91 | 92 | # Draw predictions 93 | for i, (name, prob) in enumerate(predictions): 94 | text = f"{name}: {prob:.2f}%" 95 | cv2.putText( 96 | frame, 97 | text, 98 | (10, 30 + i * 30), 99 | cv2.FONT_HERSHEY_SIMPLEX, 100 | 0.7, 101 | (0, 255, 0), 102 | 2, 103 | ) 104 | 105 | # Draw model name at the bottom of the frame with red background 106 | model_name = "Model: eva02_large_patch14_448" 107 | (text_width, text_height), _ = cv2.getTextSize( 108 | model_name, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2 109 | ) 110 | text_x = (frame.shape[1] - text_width) // 2 111 | text_y = frame.shape[0] - 20 112 | box_coords = ( 113 | (text_x - 5, text_y + 5), 114 | (text_x + text_width + 5, text_y - text_height - 5), 115 | ) 116 | cv2.rectangle( 117 | frame, box_coords[0], box_coords[1], (0, 0, 255), cv2.FILLED 118 | ) # Red background 119 | cv2.putText( 120 | frame, 121 | model_name, 122 | (text_x, text_y), 123 | cv2.FONT_HERSHEY_SIMPLEX, 124 | 0.7, 125 | (255, 255, 255), # White text 126 | 2, 127 | ) 128 | 129 | return frame 130 | 131 | 132 | def process_video(input_path, output_path, session, live_view=False, use_webcam=False): 133 | if use_webcam: 134 | cap = cv2.VideoCapture(0) 135 | else: 136 | cap = cv2.VideoCapture(input_path) 137 | 138 | width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) 139 | height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) 140 | fps = int(cap.get(cv2.CAP_PROP_FPS)) 141 | 142 | out = None 143 | if output_path: 144 | fourcc = cv2.VideoWriter_fourcc(*"mp4v") 145 | out = cv2.VideoWriter(output_path, fourcc, fps, (width, height)) 146 | 147 | input_name = session.get_inputs()[0].name 148 | output_name = session.get_outputs()[0].name 149 | 150 | frame_count = 0 151 | total_time = 0 152 | current_fps = 0 153 | 154 | while cap.isOpened(): 155 | ret, frame = cap.read() 156 | if not ret: 157 | break 158 | 159 | start_time = time.time() 160 | 161 | preprocessed = preprocess_frame(frame) 162 | output = session.run([output_name], {input_name: preprocessed}) 163 | predictions = get_top_predictions(output[0]) 164 | 165 | end_time = time.time() 166 | frame_time = end_time - start_time 167 | current_fps = 1 / frame_time 168 | 169 | frame_with_predictions = draw_predictions(frame, predictions, current_fps) 170 | 171 | if out: 172 | out.write(frame_with_predictions) 173 | 174 | if live_view: 175 | cv2.imshow("Inference", frame_with_predictions) 176 | if cv2.waitKey(1) & 0xFF == ord("q"): 177 | break 178 | 179 | total_time += frame_time 180 | frame_count += 1 181 | 182 | print( 183 | f"Processed frame {frame_count}, Time: {frame_time:.3f}s, FPS: {current_fps:.2f}" 184 | ) 185 | 186 | cap.release() 187 | if out: 188 | out.release() 189 | cv2.destroyAllWindows() 190 | 191 | avg_time = total_time / frame_count 192 | print(f"Average processing time per frame: {avg_time:.3f}s") 193 | print(f"Average FPS: {1/avg_time:.2f}") 194 | 195 | 196 | def main(): 197 | args = parse_arguments() 198 | session = get_ort_session("merged_model_compose.onnx") 199 | 200 | if args.webcam: 201 | process_video(None, args.output_video, session, args.live, use_webcam=True) 202 | elif args.input_video: 203 | process_video(args.input_video, args.output_video, session, args.live) 204 | else: 205 | print("Error: Please specify either --input_video or --webcam") 206 | return 207 | 208 | 209 | if __name__ == "__main__": 210 | main() 211 | -------------------------------------------------------------------------------- /10_gradio_app.py: -------------------------------------------------------------------------------- 1 | import time 2 | from urllib.request import urlopen 3 | 4 | import gradio as gr 5 | import numpy as np 6 | import onnxruntime as ort 7 | import torch 8 | from PIL import Image 9 | 10 | from imagenet_classes import IMAGENET2012_CLASSES 11 | 12 | 13 | def read_image(image: Image.Image): 14 | image = image.convert("RGB") 15 | img_numpy = np.array(image).astype(np.float32) 16 | img_numpy = img_numpy.transpose(2, 0, 1) 17 | img_numpy = np.expand_dims(img_numpy, axis=0) 18 | return img_numpy 19 | 20 | 21 | providers = ["CPUExecutionProvider"] 22 | 23 | session = ort.InferenceSession("merged_model_compose.onnx", providers=providers) 24 | 25 | input_name = session.get_inputs()[0].name 26 | output_name = session.get_outputs()[0].name 27 | 28 | 29 | def predict(img): 30 | output = session.run([output_name], {input_name: read_image(img)}) 31 | output = torch.from_numpy(output[0]) 32 | 33 | top5_probabilities, top5_class_indices = torch.topk(output.softmax(dim=1), k=5) 34 | 35 | im_classes = list(IMAGENET2012_CLASSES.values()) 36 | class_names = [im_classes[i] for i in top5_class_indices[0]] 37 | 38 | results = { 39 | name: float(prob) for name, prob in zip(class_names, top5_probabilities[0]) 40 | } 41 | return results 42 | 43 | 44 | # Add an example image 45 | example_image = "beignets-task-guide.png" 46 | 47 | iface = gr.Interface( 48 | fn=predict, 49 | inputs=gr.Image(type="pil"), 50 | outputs=gr.Label(num_top_classes=5), 51 | title="Image Classification with ONNX using EVA02 model", 52 | description="Blog post: https://dicksonneoh.com/portfolio/supercharge_your_pytorch_image_models/", 53 | examples=[example_image], # Add the example image to the interface 54 | ) 55 | 56 | if __name__ == "__main__": 57 | iface.launch() 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![PyTorch to ONNX-TensorRT](https://dicksonneoh.com/images/portfolio/supercharge_your_pytorch_image_models/post_image.png) 2 | 3 | This repository contains code to optimize PyTorch image models using ONNX Runtime and TensorRT, achieving up to 8x faster inference speeds. Read the full blog post [here](https://dicksonneoh.com/portfolio/supercharge_your_pytorch_image_models/). 4 | 5 | 6 | ## Installation 7 | Create and activate a conda environment: 8 | 9 | ```bash 10 | conda create -n supercharge_timm_tensorrt python=3.11 11 | conda activate supercharge_timm_tensorrt 12 | ``` 13 | Install required packages: 14 | 15 | 16 | ```bash 17 | pip install timm 18 | pip install onnx 19 | pip install onnxruntime-gpu==1.19.2 20 | pip install cupy-cuda12x 21 | pip install tensorrt==10.1.0 tensorrt-cu12==10.1.0 tensorrt-cu12-bindings==10.1.0 tensorrt-cu12-libs==10.1.0 22 | ``` 23 | 24 | Install CUDA dependencies: 25 | ```bash 26 | conda install -c nvidia cuda=12.2.2 cuda-tools=12.2.2 cuda-toolkit=12.2.2 cuda-version=12.2 cuda-command-line-tools=12.2.2 cuda-compiler=12.2.2 cuda-runtime=12.2.2 27 | ``` 28 | 29 | Install cuDNN: 30 | ```bash 31 | conda install cudnn==9.2.1.18 32 | ``` 33 | 34 | Set up library paths: 35 | ```bash 36 | export LD_LIBRARY_PATH="/home/dnth/mambaforge-pypy3/envs/supercharge_timm_tensorrt/lib:$LD_LIBRARY_PATH" 37 | export LD_LIBRARY_PATH="/home/dnth/mambaforge-pypy3/envs/supercharge_timm_tensorrt/lib/python3.11/site-packages/tensorrt_libs:$LD_LIBRARY_PATH" 38 | ``` 39 | 40 | ## Running the code 41 | 42 | The following codes correspond to the steps in the blog post. 43 | 44 | ### Load timm model and run inference: 45 | ```bash 46 | python 00_load_and_infer.py 47 | ``` 48 | Read more [here](https://dicksonneoh.com/portfolio/supercharge_your_pytorch_image_models/#-load-and-infer) 49 | 50 | ### PyTorch latency benchmark: 51 | ```bash 52 | python 01_pytorch_latency_benchmark.py 53 | ``` 54 | Read more [here](https://dicksonneoh.com/portfolio/supercharge_your_pytorch_image_models//#-baseline-latency) 55 | 56 | ### Convert model to ONNX: 57 | ```bash 58 | python 02_convert_to_onnx.py 59 | ``` 60 | Read more [here](https://dicksonneoh.com/portfolio/supercharge_your_pytorch_image_models//#-convert-to-onnx) 61 | 62 | ### ONNX Runtime CPU inference: 63 | ```bash 64 | python 03_onnx_cpu_inference.py 65 | ``` 66 | Read more [here](https://dicksonneoh.com/portfolio/supercharge_your_pytorch_image_models//#-onnx-runtime-on-cpu) 67 | 68 | ### ONNX Runtime CUDA inference: 69 | ```bash 70 | python 04_onnx_cuda_inference.py 71 | ``` 72 | Read more [here](https://dicksonneoh.com/portfolio/supercharge_your_pytorch_image_models//#-onnx-runtime-on-cuda) 73 | 74 | ### ONNX Runtime TensorRT inference: 75 | ```bash 76 | python 05_onnx_trt_inference.py 77 | ``` 78 | Read more [here](https://dicksonneoh.com/portfolio/supercharge_your_pytorch_image_models//#-onnx-runtime-on-tensorrt) 79 | 80 | ### Export preprocessing to ONNX: 81 | ```bash 82 | python 06_export_preprocessing_onnx.py 83 | ``` 84 | Read more [here](https://dicksonneoh.com/portfolio/supercharge_your_pytorch_image_models//#-bake-pre-processing-into-onnx) 85 | 86 | ### Merge preprocessing and model ONNX: 87 | ```bash 88 | python 07_onnx_compose_merge.py 89 | ``` 90 | Read more [here](https://dicksonneoh.com/portfolio/supercharge_your_pytorch_image_models//#-bake-pre-processing-into-onnx) 91 | 92 | ### Run inference on merged model: 93 | ```bash 94 | python 08_inference_merged_model.py 95 | ``` 96 | Read more [here](https://dicksonneoh.com/portfolio/supercharge_your_pytorch_image_models//#-bake-pre-processing-into-onnx) 97 | 98 | ### Run inference on video: 99 | ```bash 100 | python 09_video_inference.py sample.mp4 output.mp4 --live 101 | ``` 102 | 103 | 104 | 105 | https://github.com/user-attachments/assets/1a25dd6e-3512-475a-9541-29e836022bb5 106 | 107 | 108 | 109 | 110 | 111 | 174 | -------------------------------------------------------------------------------- /beignets-task-guide.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dnth/supercharge-your-pytorch-image-models-blogpost/55253d09dd72c530c4bd2ff550e277a96914930c/beignets-task-guide.png -------------------------------------------------------------------------------- /builtin_prepost_onnxruntime_extensions.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import onnxruntime as ort 3 | from PIL import Image 4 | 5 | 6 | def read_image(filename: str): 7 | img = Image.open(filename) 8 | img = img.resize((448, 448)) 9 | img = np.array(img) 10 | img = img.astype(np.float32) 11 | return img 12 | 13 | 14 | onnx_filename = "eva02_large_patch14_448_prepost.onnx" 15 | providers = [ 16 | ( 17 | "TensorrtExecutionProvider", 18 | { 19 | "device_id": 0, 20 | "trt_max_workspace_size": 8589934592, 21 | "trt_fp16_enable": True, 22 | "trt_engine_cache_enable": True, 23 | "trt_engine_cache_path": "./trt_cache", 24 | "trt_force_sequential_engine_build": False, 25 | "trt_max_partition_iterations": 10000, 26 | "trt_min_subgraph_size": 1, 27 | "trt_builder_optimization_level": 5, 28 | "trt_timing_cache_enable": True, 29 | }, 30 | ), 31 | ] 32 | session = ort.InferenceSession(onnx_filename, providers=providers) 33 | 34 | input_name = session.get_inputs()[0].name 35 | output_name = session.get_outputs()[0].name 36 | 37 | output = session.run( 38 | [output_name], {input_name: read_image("beignets-task-guide.png")} 39 | )[0] 40 | 41 | print(output.shape) 42 | print(output) 43 | 44 | import time 45 | 46 | # Run benchmark 47 | num_images = 100 48 | start = time.perf_counter() 49 | for i in range(num_images): 50 | output = session.run( 51 | [output_name], {input_name: read_image("beignets-task-guide.png")} 52 | )[0] 53 | end = time.perf_counter() 54 | time_taken = end - start 55 | 56 | ms_per_image = time_taken / num_images * 1000 57 | fps = num_images / time_taken 58 | 59 | print( 60 | f"Onnxruntime builtin transforms: {ms_per_image:.3f} ms per image, FPS: {fps:.2f}" 61 | ) 62 | -------------------------------------------------------------------------------- /imagenet_classes.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2022 the HuggingFace Datasets Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from collections import OrderedDict 17 | 18 | 19 | IMAGENET2012_CLASSES = OrderedDict( 20 | { 21 | "n01440764": "tench, Tinca tinca", 22 | "n01443537": "goldfish, Carassius auratus", 23 | "n01484850": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", 24 | "n01491361": "tiger shark, Galeocerdo cuvieri", 25 | "n01494475": "hammerhead, hammerhead shark", 26 | "n01496331": "electric ray, crampfish, numbfish, torpedo", 27 | "n01498041": "stingray", 28 | "n01514668": "cock", 29 | "n01514859": "hen", 30 | "n01518878": "ostrich, Struthio camelus", 31 | "n01530575": "brambling, Fringilla montifringilla", 32 | "n01531178": "goldfinch, Carduelis carduelis", 33 | "n01532829": "house finch, linnet, Carpodacus mexicanus", 34 | "n01534433": "junco, snowbird", 35 | "n01537544": "indigo bunting, indigo finch, indigo bird, Passerina cyanea", 36 | "n01558993": "robin, American robin, Turdus migratorius", 37 | "n01560419": "bulbul", 38 | "n01580077": "jay", 39 | "n01582220": "magpie", 40 | "n01592084": "chickadee", 41 | "n01601694": "water ouzel, dipper", 42 | "n01608432": "kite", 43 | "n01614925": "bald eagle, American eagle, Haliaeetus leucocephalus", 44 | "n01616318": "vulture", 45 | "n01622779": "great grey owl, great gray owl, Strix nebulosa", 46 | "n01629819": "European fire salamander, Salamandra salamandra", 47 | "n01630670": "common newt, Triturus vulgaris", 48 | "n01631663": "eft", 49 | "n01632458": "spotted salamander, Ambystoma maculatum", 50 | "n01632777": "axolotl, mud puppy, Ambystoma mexicanum", 51 | "n01641577": "bullfrog, Rana catesbeiana", 52 | "n01644373": "tree frog, tree-frog", 53 | "n01644900": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", 54 | "n01664065": "loggerhead, loggerhead turtle, Caretta caretta", 55 | "n01665541": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", 56 | "n01667114": "mud turtle", 57 | "n01667778": "terrapin", 58 | "n01669191": "box turtle, box tortoise", 59 | "n01675722": "banded gecko", 60 | "n01677366": "common iguana, iguana, Iguana iguana", 61 | "n01682714": "American chameleon, anole, Anolis carolinensis", 62 | "n01685808": "whiptail, whiptail lizard", 63 | "n01687978": "agama", 64 | "n01688243": "frilled lizard, Chlamydosaurus kingi", 65 | "n01689811": "alligator lizard", 66 | "n01692333": "Gila monster, Heloderma suspectum", 67 | "n01693334": "green lizard, Lacerta viridis", 68 | "n01694178": "African chameleon, Chamaeleo chamaeleon", 69 | "n01695060": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", 70 | "n01697457": "African crocodile, Nile crocodile, Crocodylus niloticus", 71 | "n01698640": "American alligator, Alligator mississipiensis", 72 | "n01704323": "triceratops", 73 | "n01728572": "thunder snake, worm snake, Carphophis amoenus", 74 | "n01728920": "ringneck snake, ring-necked snake, ring snake", 75 | "n01729322": "hognose snake, puff adder, sand viper", 76 | "n01729977": "green snake, grass snake", 77 | "n01734418": "king snake, kingsnake", 78 | "n01735189": "garter snake, grass snake", 79 | "n01737021": "water snake", 80 | "n01739381": "vine snake", 81 | "n01740131": "night snake, Hypsiglena torquata", 82 | "n01742172": "boa constrictor, Constrictor constrictor", 83 | "n01744401": "rock python, rock snake, Python sebae", 84 | "n01748264": "Indian cobra, Naja naja", 85 | "n01749939": "green mamba", 86 | "n01751748": "sea snake", 87 | "n01753488": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", 88 | "n01755581": "diamondback, diamondback rattlesnake, Crotalus adamanteus", 89 | "n01756291": "sidewinder, horned rattlesnake, Crotalus cerastes", 90 | "n01768244": "trilobite", 91 | "n01770081": "harvestman, daddy longlegs, Phalangium opilio", 92 | "n01770393": "scorpion", 93 | "n01773157": "black and gold garden spider, Argiope aurantia", 94 | "n01773549": "barn spider, Araneus cavaticus", 95 | "n01773797": "garden spider, Aranea diademata", 96 | "n01774384": "black widow, Latrodectus mactans", 97 | "n01774750": "tarantula", 98 | "n01775062": "wolf spider, hunting spider", 99 | "n01776313": "tick", 100 | "n01784675": "centipede", 101 | "n01795545": "black grouse", 102 | "n01796340": "ptarmigan", 103 | "n01797886": "ruffed grouse, partridge, Bonasa umbellus", 104 | "n01798484": "prairie chicken, prairie grouse, prairie fowl", 105 | "n01806143": "peacock", 106 | "n01806567": "quail", 107 | "n01807496": "partridge", 108 | "n01817953": "African grey, African gray, Psittacus erithacus", 109 | "n01818515": "macaw", 110 | "n01819313": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", 111 | "n01820546": "lorikeet", 112 | "n01824575": "coucal", 113 | "n01828970": "bee eater", 114 | "n01829413": "hornbill", 115 | "n01833805": "hummingbird", 116 | "n01843065": "jacamar", 117 | "n01843383": "toucan", 118 | "n01847000": "drake", 119 | "n01855032": "red-breasted merganser, Mergus serrator", 120 | "n01855672": "goose", 121 | "n01860187": "black swan, Cygnus atratus", 122 | "n01871265": "tusker", 123 | "n01872401": "echidna, spiny anteater, anteater", 124 | "n01873310": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", 125 | "n01877812": "wallaby, brush kangaroo", 126 | "n01882714": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", 127 | "n01883070": "wombat", 128 | "n01910747": "jellyfish", 129 | "n01914609": "sea anemone, anemone", 130 | "n01917289": "brain coral", 131 | "n01924916": "flatworm, platyhelminth", 132 | "n01930112": "nematode, nematode worm, roundworm", 133 | "n01943899": "conch", 134 | "n01944390": "snail", 135 | "n01945685": "slug", 136 | "n01950731": "sea slug, nudibranch", 137 | "n01955084": "chiton, coat-of-mail shell, sea cradle, polyplacophore", 138 | "n01968897": "chambered nautilus, pearly nautilus, nautilus", 139 | "n01978287": "Dungeness crab, Cancer magister", 140 | "n01978455": "rock crab, Cancer irroratus", 141 | "n01980166": "fiddler crab", 142 | "n01981276": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", 143 | "n01983481": "American lobster, Northern lobster, Maine lobster, Homarus americanus", 144 | "n01984695": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", 145 | "n01985128": "crayfish, crawfish, crawdad, crawdaddy", 146 | "n01986214": "hermit crab", 147 | "n01990800": "isopod", 148 | "n02002556": "white stork, Ciconia ciconia", 149 | "n02002724": "black stork, Ciconia nigra", 150 | "n02006656": "spoonbill", 151 | "n02007558": "flamingo", 152 | "n02009229": "little blue heron, Egretta caerulea", 153 | "n02009912": "American egret, great white heron, Egretta albus", 154 | "n02011460": "bittern", 155 | "n02012849": "crane", 156 | "n02013706": "limpkin, Aramus pictus", 157 | "n02017213": "European gallinule, Porphyrio porphyrio", 158 | "n02018207": "American coot, marsh hen, mud hen, water hen, Fulica americana", 159 | "n02018795": "bustard", 160 | "n02025239": "ruddy turnstone, Arenaria interpres", 161 | "n02027492": "red-backed sandpiper, dunlin, Erolia alpina", 162 | "n02028035": "redshank, Tringa totanus", 163 | "n02033041": "dowitcher", 164 | "n02037110": "oystercatcher, oyster catcher", 165 | "n02051845": "pelican", 166 | "n02056570": "king penguin, Aptenodytes patagonica", 167 | "n02058221": "albatross, mollymawk", 168 | "n02066245": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", 169 | "n02071294": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", 170 | "n02074367": "dugong, Dugong dugon", 171 | "n02077923": "sea lion", 172 | "n02085620": "Chihuahua", 173 | "n02085782": "Japanese spaniel", 174 | "n02085936": "Maltese dog, Maltese terrier, Maltese", 175 | "n02086079": "Pekinese, Pekingese, Peke", 176 | "n02086240": "Shih-Tzu", 177 | "n02086646": "Blenheim spaniel", 178 | "n02086910": "papillon", 179 | "n02087046": "toy terrier", 180 | "n02087394": "Rhodesian ridgeback", 181 | "n02088094": "Afghan hound, Afghan", 182 | "n02088238": "basset, basset hound", 183 | "n02088364": "beagle", 184 | "n02088466": "bloodhound, sleuthhound", 185 | "n02088632": "bluetick", 186 | "n02089078": "black-and-tan coonhound", 187 | "n02089867": "Walker hound, Walker foxhound", 188 | "n02089973": "English foxhound", 189 | "n02090379": "redbone", 190 | "n02090622": "borzoi, Russian wolfhound", 191 | "n02090721": "Irish wolfhound", 192 | "n02091032": "Italian greyhound", 193 | "n02091134": "whippet", 194 | "n02091244": "Ibizan hound, Ibizan Podenco", 195 | "n02091467": "Norwegian elkhound, elkhound", 196 | "n02091635": "otterhound, otter hound", 197 | "n02091831": "Saluki, gazelle hound", 198 | "n02092002": "Scottish deerhound, deerhound", 199 | "n02092339": "Weimaraner", 200 | "n02093256": "Staffordshire bullterrier, Staffordshire bull terrier", 201 | "n02093428": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", 202 | "n02093647": "Bedlington terrier", 203 | "n02093754": "Border terrier", 204 | "n02093859": "Kerry blue terrier", 205 | "n02093991": "Irish terrier", 206 | "n02094114": "Norfolk terrier", 207 | "n02094258": "Norwich terrier", 208 | "n02094433": "Yorkshire terrier", 209 | "n02095314": "wire-haired fox terrier", 210 | "n02095570": "Lakeland terrier", 211 | "n02095889": "Sealyham terrier, Sealyham", 212 | "n02096051": "Airedale, Airedale terrier", 213 | "n02096177": "cairn, cairn terrier", 214 | "n02096294": "Australian terrier", 215 | "n02096437": "Dandie Dinmont, Dandie Dinmont terrier", 216 | "n02096585": "Boston bull, Boston terrier", 217 | "n02097047": "miniature schnauzer", 218 | "n02097130": "giant schnauzer", 219 | "n02097209": "standard schnauzer", 220 | "n02097298": "Scotch terrier, Scottish terrier, Scottie", 221 | "n02097474": "Tibetan terrier, chrysanthemum dog", 222 | "n02097658": "silky terrier, Sydney silky", 223 | "n02098105": "soft-coated wheaten terrier", 224 | "n02098286": "West Highland white terrier", 225 | "n02098413": "Lhasa, Lhasa apso", 226 | "n02099267": "flat-coated retriever", 227 | "n02099429": "curly-coated retriever", 228 | "n02099601": "golden retriever", 229 | "n02099712": "Labrador retriever", 230 | "n02099849": "Chesapeake Bay retriever", 231 | "n02100236": "German short-haired pointer", 232 | "n02100583": "vizsla, Hungarian pointer", 233 | "n02100735": "English setter", 234 | "n02100877": "Irish setter, red setter", 235 | "n02101006": "Gordon setter", 236 | "n02101388": "Brittany spaniel", 237 | "n02101556": "clumber, clumber spaniel", 238 | "n02102040": "English springer, English springer spaniel", 239 | "n02102177": "Welsh springer spaniel", 240 | "n02102318": "cocker spaniel, English cocker spaniel, cocker", 241 | "n02102480": "Sussex spaniel", 242 | "n02102973": "Irish water spaniel", 243 | "n02104029": "kuvasz", 244 | "n02104365": "schipperke", 245 | "n02105056": "groenendael", 246 | "n02105162": "malinois", 247 | "n02105251": "briard", 248 | "n02105412": "kelpie", 249 | "n02105505": "komondor", 250 | "n02105641": "Old English sheepdog, bobtail", 251 | "n02105855": "Shetland sheepdog, Shetland sheep dog, Shetland", 252 | "n02106030": "collie", 253 | "n02106166": "Border collie", 254 | "n02106382": "Bouvier des Flandres, Bouviers des Flandres", 255 | "n02106550": "Rottweiler", 256 | "n02106662": "German shepherd, German shepherd dog, German police dog, alsatian", 257 | "n02107142": "Doberman, Doberman pinscher", 258 | "n02107312": "miniature pinscher", 259 | "n02107574": "Greater Swiss Mountain dog", 260 | "n02107683": "Bernese mountain dog", 261 | "n02107908": "Appenzeller", 262 | "n02108000": "EntleBucher", 263 | "n02108089": "boxer", 264 | "n02108422": "bull mastiff", 265 | "n02108551": "Tibetan mastiff", 266 | "n02108915": "French bulldog", 267 | "n02109047": "Great Dane", 268 | "n02109525": "Saint Bernard, St Bernard", 269 | "n02109961": "Eskimo dog, husky", 270 | "n02110063": "malamute, malemute, Alaskan malamute", 271 | "n02110185": "Siberian husky", 272 | "n02110341": "dalmatian, coach dog, carriage dog", 273 | "n02110627": "affenpinscher, monkey pinscher, monkey dog", 274 | "n02110806": "basenji", 275 | "n02110958": "pug, pug-dog", 276 | "n02111129": "Leonberg", 277 | "n02111277": "Newfoundland, Newfoundland dog", 278 | "n02111500": "Great Pyrenees", 279 | "n02111889": "Samoyed, Samoyede", 280 | "n02112018": "Pomeranian", 281 | "n02112137": "chow, chow chow", 282 | "n02112350": "keeshond", 283 | "n02112706": "Brabancon griffon", 284 | "n02113023": "Pembroke, Pembroke Welsh corgi", 285 | "n02113186": "Cardigan, Cardigan Welsh corgi", 286 | "n02113624": "toy poodle", 287 | "n02113712": "miniature poodle", 288 | "n02113799": "standard poodle", 289 | "n02113978": "Mexican hairless", 290 | "n02114367": "timber wolf, grey wolf, gray wolf, Canis lupus", 291 | "n02114548": "white wolf, Arctic wolf, Canis lupus tundrarum", 292 | "n02114712": "red wolf, maned wolf, Canis rufus, Canis niger", 293 | "n02114855": "coyote, prairie wolf, brush wolf, Canis latrans", 294 | "n02115641": "dingo, warrigal, warragal, Canis dingo", 295 | "n02115913": "dhole, Cuon alpinus", 296 | "n02116738": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", 297 | "n02117135": "hyena, hyaena", 298 | "n02119022": "red fox, Vulpes vulpes", 299 | "n02119789": "kit fox, Vulpes macrotis", 300 | "n02120079": "Arctic fox, white fox, Alopex lagopus", 301 | "n02120505": "grey fox, gray fox, Urocyon cinereoargenteus", 302 | "n02123045": "tabby, tabby cat", 303 | "n02123159": "tiger cat", 304 | "n02123394": "Persian cat", 305 | "n02123597": "Siamese cat, Siamese", 306 | "n02124075": "Egyptian cat", 307 | "n02125311": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", 308 | "n02127052": "lynx, catamount", 309 | "n02128385": "leopard, Panthera pardus", 310 | "n02128757": "snow leopard, ounce, Panthera uncia", 311 | "n02128925": "jaguar, panther, Panthera onca, Felis onca", 312 | "n02129165": "lion, king of beasts, Panthera leo", 313 | "n02129604": "tiger, Panthera tigris", 314 | "n02130308": "cheetah, chetah, Acinonyx jubatus", 315 | "n02132136": "brown bear, bruin, Ursus arctos", 316 | "n02133161": "American black bear, black bear, Ursus americanus, Euarctos americanus", 317 | "n02134084": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", 318 | "n02134418": "sloth bear, Melursus ursinus, Ursus ursinus", 319 | "n02137549": "mongoose", 320 | "n02138441": "meerkat, mierkat", 321 | "n02165105": "tiger beetle", 322 | "n02165456": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", 323 | "n02167151": "ground beetle, carabid beetle", 324 | "n02168699": "long-horned beetle, longicorn, longicorn beetle", 325 | "n02169497": "leaf beetle, chrysomelid", 326 | "n02172182": "dung beetle", 327 | "n02174001": "rhinoceros beetle", 328 | "n02177972": "weevil", 329 | "n02190166": "fly", 330 | "n02206856": "bee", 331 | "n02219486": "ant, emmet, pismire", 332 | "n02226429": "grasshopper, hopper", 333 | "n02229544": "cricket", 334 | "n02231487": "walking stick, walkingstick, stick insect", 335 | "n02233338": "cockroach, roach", 336 | "n02236044": "mantis, mantid", 337 | "n02256656": "cicada, cicala", 338 | "n02259212": "leafhopper", 339 | "n02264363": "lacewing, lacewing fly", 340 | "n02268443": "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", 341 | "n02268853": "damselfly", 342 | "n02276258": "admiral", 343 | "n02277742": "ringlet, ringlet butterfly", 344 | "n02279972": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", 345 | "n02280649": "cabbage butterfly", 346 | "n02281406": "sulphur butterfly, sulfur butterfly", 347 | "n02281787": "lycaenid, lycaenid butterfly", 348 | "n02317335": "starfish, sea star", 349 | "n02319095": "sea urchin", 350 | "n02321529": "sea cucumber, holothurian", 351 | "n02325366": "wood rabbit, cottontail, cottontail rabbit", 352 | "n02326432": "hare", 353 | "n02328150": "Angora, Angora rabbit", 354 | "n02342885": "hamster", 355 | "n02346627": "porcupine, hedgehog", 356 | "n02356798": "fox squirrel, eastern fox squirrel, Sciurus niger", 357 | "n02361337": "marmot", 358 | "n02363005": "beaver", 359 | "n02364673": "guinea pig, Cavia cobaya", 360 | "n02389026": "sorrel", 361 | "n02391049": "zebra", 362 | "n02395406": "hog, pig, grunter, squealer, Sus scrofa", 363 | "n02396427": "wild boar, boar, Sus scrofa", 364 | "n02397096": "warthog", 365 | "n02398521": "hippopotamus, hippo, river horse, Hippopotamus amphibius", 366 | "n02403003": "ox", 367 | "n02408429": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", 368 | "n02410509": "bison", 369 | "n02412080": "ram, tup", 370 | "n02415577": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", 371 | "n02417914": "ibex, Capra ibex", 372 | "n02422106": "hartebeest", 373 | "n02422699": "impala, Aepyceros melampus", 374 | "n02423022": "gazelle", 375 | "n02437312": "Arabian camel, dromedary, Camelus dromedarius", 376 | "n02437616": "llama", 377 | "n02441942": "weasel", 378 | "n02442845": "mink", 379 | "n02443114": "polecat, fitch, foulmart, foumart, Mustela putorius", 380 | "n02443484": "black-footed ferret, ferret, Mustela nigripes", 381 | "n02444819": "otter", 382 | "n02445715": "skunk, polecat, wood pussy", 383 | "n02447366": "badger", 384 | "n02454379": "armadillo", 385 | "n02457408": "three-toed sloth, ai, Bradypus tridactylus", 386 | "n02480495": "orangutan, orang, orangutang, Pongo pygmaeus", 387 | "n02480855": "gorilla, Gorilla gorilla", 388 | "n02481823": "chimpanzee, chimp, Pan troglodytes", 389 | "n02483362": "gibbon, Hylobates lar", 390 | "n02483708": "siamang, Hylobates syndactylus, Symphalangus syndactylus", 391 | "n02484975": "guenon, guenon monkey", 392 | "n02486261": "patas, hussar monkey, Erythrocebus patas", 393 | "n02486410": "baboon", 394 | "n02487347": "macaque", 395 | "n02488291": "langur", 396 | "n02488702": "colobus, colobus monkey", 397 | "n02489166": "proboscis monkey, Nasalis larvatus", 398 | "n02490219": "marmoset", 399 | "n02492035": "capuchin, ringtail, Cebus capucinus", 400 | "n02492660": "howler monkey, howler", 401 | "n02493509": "titi, titi monkey", 402 | "n02493793": "spider monkey, Ateles geoffroyi", 403 | "n02494079": "squirrel monkey, Saimiri sciureus", 404 | "n02497673": "Madagascar cat, ring-tailed lemur, Lemur catta", 405 | "n02500267": "indri, indris, Indri indri, Indri brevicaudatus", 406 | "n02504013": "Indian elephant, Elephas maximus", 407 | "n02504458": "African elephant, Loxodonta africana", 408 | "n02509815": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", 409 | "n02510455": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", 410 | "n02514041": "barracouta, snoek", 411 | "n02526121": "eel", 412 | "n02536864": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", 413 | "n02606052": "rock beauty, Holocanthus tricolor", 414 | "n02607072": "anemone fish", 415 | "n02640242": "sturgeon", 416 | "n02641379": "gar, garfish, garpike, billfish, Lepisosteus osseus", 417 | "n02643566": "lionfish", 418 | "n02655020": "puffer, pufferfish, blowfish, globefish", 419 | "n02666196": "abacus", 420 | "n02667093": "abaya", 421 | "n02669723": "academic gown, academic robe, judge's robe", 422 | "n02672831": "accordion, piano accordion, squeeze box", 423 | "n02676566": "acoustic guitar", 424 | "n02687172": "aircraft carrier, carrier, flattop, attack aircraft carrier", 425 | "n02690373": "airliner", 426 | "n02692877": "airship, dirigible", 427 | "n02699494": "altar", 428 | "n02701002": "ambulance", 429 | "n02704792": "amphibian, amphibious vehicle", 430 | "n02708093": "analog clock", 431 | "n02727426": "apiary, bee house", 432 | "n02730930": "apron", 433 | "n02747177": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", 434 | "n02749479": "assault rifle, assault gun", 435 | "n02769748": "backpack, back pack, knapsack, packsack, rucksack, haversack", 436 | "n02776631": "bakery, bakeshop, bakehouse", 437 | "n02777292": "balance beam, beam", 438 | "n02782093": "balloon", 439 | "n02783161": "ballpoint, ballpoint pen, ballpen, Biro", 440 | "n02786058": "Band Aid", 441 | "n02787622": "banjo", 442 | "n02788148": "bannister, banister, balustrade, balusters, handrail", 443 | "n02790996": "barbell", 444 | "n02791124": "barber chair", 445 | "n02791270": "barbershop", 446 | "n02793495": "barn", 447 | "n02794156": "barometer", 448 | "n02795169": "barrel, cask", 449 | "n02797295": "barrow, garden cart, lawn cart, wheelbarrow", 450 | "n02799071": "baseball", 451 | "n02802426": "basketball", 452 | "n02804414": "bassinet", 453 | "n02804610": "bassoon", 454 | "n02807133": "bathing cap, swimming cap", 455 | "n02808304": "bath towel", 456 | "n02808440": "bathtub, bathing tub, bath, tub", 457 | "n02814533": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", 458 | "n02814860": "beacon, lighthouse, beacon light, pharos", 459 | "n02815834": "beaker", 460 | "n02817516": "bearskin, busby, shako", 461 | "n02823428": "beer bottle", 462 | "n02823750": "beer glass", 463 | "n02825657": "bell cote, bell cot", 464 | "n02834397": "bib", 465 | "n02835271": "bicycle-built-for-two, tandem bicycle, tandem", 466 | "n02837789": "bikini, two-piece", 467 | "n02840245": "binder, ring-binder", 468 | "n02841315": "binoculars, field glasses, opera glasses", 469 | "n02843684": "birdhouse", 470 | "n02859443": "boathouse", 471 | "n02860847": "bobsled, bobsleigh, bob", 472 | "n02865351": "bolo tie, bolo, bola tie, bola", 473 | "n02869837": "bonnet, poke bonnet", 474 | "n02870880": "bookcase", 475 | "n02871525": "bookshop, bookstore, bookstall", 476 | "n02877765": "bottlecap", 477 | "n02879718": "bow", 478 | "n02883205": "bow tie, bow-tie, bowtie", 479 | "n02892201": "brass, memorial tablet, plaque", 480 | "n02892767": "brassiere, bra, bandeau", 481 | "n02894605": "breakwater, groin, groyne, mole, bulwark, seawall, jetty", 482 | "n02895154": "breastplate, aegis, egis", 483 | "n02906734": "broom", 484 | "n02909870": "bucket, pail", 485 | "n02910353": "buckle", 486 | "n02916936": "bulletproof vest", 487 | "n02917067": "bullet train, bullet", 488 | "n02927161": "butcher shop, meat market", 489 | "n02930766": "cab, hack, taxi, taxicab", 490 | "n02939185": "caldron, cauldron", 491 | "n02948072": "candle, taper, wax light", 492 | "n02950826": "cannon", 493 | "n02951358": "canoe", 494 | "n02951585": "can opener, tin opener", 495 | "n02963159": "cardigan", 496 | "n02965783": "car mirror", 497 | "n02966193": "carousel, carrousel, merry-go-round, roundabout, whirligig", 498 | "n02966687": "carpenter's kit, tool kit", 499 | "n02971356": "carton", 500 | "n02974003": "car wheel", 501 | "n02977058": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", 502 | "n02978881": "cassette", 503 | "n02979186": "cassette player", 504 | "n02980441": "castle", 505 | "n02981792": "catamaran", 506 | "n02988304": "CD player", 507 | "n02992211": "cello, violoncello", 508 | "n02992529": "cellular telephone, cellular phone, cellphone, cell, mobile phone", 509 | "n02999410": "chain", 510 | "n03000134": "chainlink fence", 511 | "n03000247": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", 512 | "n03000684": "chain saw, chainsaw", 513 | "n03014705": "chest", 514 | "n03016953": "chiffonier, commode", 515 | "n03017168": "chime, bell, gong", 516 | "n03018349": "china cabinet, china closet", 517 | "n03026506": "Christmas stocking", 518 | "n03028079": "church, church building", 519 | "n03032252": "cinema, movie theater, movie theatre, movie house, picture palace", 520 | "n03041632": "cleaver, meat cleaver, chopper", 521 | "n03042490": "cliff dwelling", 522 | "n03045698": "cloak", 523 | "n03047690": "clog, geta, patten, sabot", 524 | "n03062245": "cocktail shaker", 525 | "n03063599": "coffee mug", 526 | "n03063689": "coffeepot", 527 | "n03065424": "coil, spiral, volute, whorl, helix", 528 | "n03075370": "combination lock", 529 | "n03085013": "computer keyboard, keypad", 530 | "n03089624": "confectionery, confectionary, candy store", 531 | "n03095699": "container ship, containership, container vessel", 532 | "n03100240": "convertible", 533 | "n03109150": "corkscrew, bottle screw", 534 | "n03110669": "cornet, horn, trumpet, trump", 535 | "n03124043": "cowboy boot", 536 | "n03124170": "cowboy hat, ten-gallon hat", 537 | "n03125729": "cradle", 538 | "n03126707": "crane2", 539 | "n03127747": "crash helmet", 540 | "n03127925": "crate", 541 | "n03131574": "crib, cot", 542 | "n03133878": "Crock Pot", 543 | "n03134739": "croquet ball", 544 | "n03141823": "crutch", 545 | "n03146219": "cuirass", 546 | "n03160309": "dam, dike, dyke", 547 | "n03179701": "desk", 548 | "n03180011": "desktop computer", 549 | "n03187595": "dial telephone, dial phone", 550 | "n03188531": "diaper, nappy, napkin", 551 | "n03196217": "digital clock", 552 | "n03197337": "digital watch", 553 | "n03201208": "dining table, board", 554 | "n03207743": "dishrag, dishcloth", 555 | "n03207941": "dishwasher, dish washer, dishwashing machine", 556 | "n03208938": "disk brake, disc brake", 557 | "n03216828": "dock, dockage, docking facility", 558 | "n03218198": "dogsled, dog sled, dog sleigh", 559 | "n03220513": "dome", 560 | "n03223299": "doormat, welcome mat", 561 | "n03240683": "drilling platform, offshore rig", 562 | "n03249569": "drum, membranophone, tympan", 563 | "n03250847": "drumstick", 564 | "n03255030": "dumbbell", 565 | "n03259280": "Dutch oven", 566 | "n03271574": "electric fan, blower", 567 | "n03272010": "electric guitar", 568 | "n03272562": "electric locomotive", 569 | "n03290653": "entertainment center", 570 | "n03291819": "envelope", 571 | "n03297495": "espresso maker", 572 | "n03314780": "face powder", 573 | "n03325584": "feather boa, boa", 574 | "n03337140": "file, file cabinet, filing cabinet", 575 | "n03344393": "fireboat", 576 | "n03345487": "fire engine, fire truck", 577 | "n03347037": "fire screen, fireguard", 578 | "n03355925": "flagpole, flagstaff", 579 | "n03372029": "flute, transverse flute", 580 | "n03376595": "folding chair", 581 | "n03379051": "football helmet", 582 | "n03384352": "forklift", 583 | "n03388043": "fountain", 584 | "n03388183": "fountain pen", 585 | "n03388549": "four-poster", 586 | "n03393912": "freight car", 587 | "n03394916": "French horn, horn", 588 | "n03400231": "frying pan, frypan, skillet", 589 | "n03404251": "fur coat", 590 | "n03417042": "garbage truck, dustcart", 591 | "n03424325": "gasmask, respirator, gas helmet", 592 | "n03425413": "gas pump, gasoline pump, petrol pump, island dispenser", 593 | "n03443371": "goblet", 594 | "n03444034": "go-kart", 595 | "n03445777": "golf ball", 596 | "n03445924": "golfcart, golf cart", 597 | "n03447447": "gondola", 598 | "n03447721": "gong, tam-tam", 599 | "n03450230": "gown", 600 | "n03452741": "grand piano, grand", 601 | "n03457902": "greenhouse, nursery, glasshouse", 602 | "n03459775": "grille, radiator grille", 603 | "n03461385": "grocery store, grocery, food market, market", 604 | "n03467068": "guillotine", 605 | "n03476684": "hair slide", 606 | "n03476991": "hair spray", 607 | "n03478589": "half track", 608 | "n03481172": "hammer", 609 | "n03482405": "hamper", 610 | "n03483316": "hand blower, blow dryer, blow drier, hair dryer, hair drier", 611 | "n03485407": "hand-held computer, hand-held microcomputer", 612 | "n03485794": "handkerchief, hankie, hanky, hankey", 613 | "n03492542": "hard disc, hard disk, fixed disk", 614 | "n03494278": "harmonica, mouth organ, harp, mouth harp", 615 | "n03495258": "harp", 616 | "n03496892": "harvester, reaper", 617 | "n03498962": "hatchet", 618 | "n03527444": "holster", 619 | "n03529860": "home theater, home theatre", 620 | "n03530642": "honeycomb", 621 | "n03532672": "hook, claw", 622 | "n03534580": "hoopskirt, crinoline", 623 | "n03535780": "horizontal bar, high bar", 624 | "n03538406": "horse cart, horse-cart", 625 | "n03544143": "hourglass", 626 | "n03584254": "iPod", 627 | "n03584829": "iron, smoothing iron", 628 | "n03590841": "jack-o'-lantern", 629 | "n03594734": "jean, blue jean, denim", 630 | "n03594945": "jeep, landrover", 631 | "n03595614": "jersey, T-shirt, tee shirt", 632 | "n03598930": "jigsaw puzzle", 633 | "n03599486": "jinrikisha, ricksha, rickshaw", 634 | "n03602883": "joystick", 635 | "n03617480": "kimono", 636 | "n03623198": "knee pad", 637 | "n03627232": "knot", 638 | "n03630383": "lab coat, laboratory coat", 639 | "n03633091": "ladle", 640 | "n03637318": "lampshade, lamp shade", 641 | "n03642806": "laptop, laptop computer", 642 | "n03649909": "lawn mower, mower", 643 | "n03657121": "lens cap, lens cover", 644 | "n03658185": "letter opener, paper knife, paperknife", 645 | "n03661043": "library", 646 | "n03662601": "lifeboat", 647 | "n03666591": "lighter, light, igniter, ignitor", 648 | "n03670208": "limousine, limo", 649 | "n03673027": "liner, ocean liner", 650 | "n03676483": "lipstick, lip rouge", 651 | "n03680355": "Loafer", 652 | "n03690938": "lotion", 653 | "n03691459": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", 654 | "n03692522": "loupe, jeweler's loupe", 655 | "n03697007": "lumbermill, sawmill", 656 | "n03706229": "magnetic compass", 657 | "n03709823": "mailbag, postbag", 658 | "n03710193": "mailbox, letter box", 659 | "n03710637": "maillot", 660 | "n03710721": "maillot, tank suit", 661 | "n03717622": "manhole cover", 662 | "n03720891": "maraca", 663 | "n03721384": "marimba, xylophone", 664 | "n03724870": "mask", 665 | "n03729826": "matchstick", 666 | "n03733131": "maypole", 667 | "n03733281": "maze, labyrinth", 668 | "n03733805": "measuring cup", 669 | "n03742115": "medicine chest, medicine cabinet", 670 | "n03743016": "megalith, megalithic structure", 671 | "n03759954": "microphone, mike", 672 | "n03761084": "microwave, microwave oven", 673 | "n03763968": "military uniform", 674 | "n03764736": "milk can", 675 | "n03769881": "minibus", 676 | "n03770439": "miniskirt, mini", 677 | "n03770679": "minivan", 678 | "n03773504": "missile", 679 | "n03775071": "mitten", 680 | "n03775546": "mixing bowl", 681 | "n03776460": "mobile home, manufactured home", 682 | "n03777568": "Model T", 683 | "n03777754": "modem", 684 | "n03781244": "monastery", 685 | "n03782006": "monitor", 686 | "n03785016": "moped", 687 | "n03786901": "mortar", 688 | "n03787032": "mortarboard", 689 | "n03788195": "mosque", 690 | "n03788365": "mosquito net", 691 | "n03791053": "motor scooter, scooter", 692 | "n03792782": "mountain bike, all-terrain bike, off-roader", 693 | "n03792972": "mountain tent", 694 | "n03793489": "mouse, computer mouse", 695 | "n03794056": "mousetrap", 696 | "n03796401": "moving van", 697 | "n03803284": "muzzle", 698 | "n03804744": "nail", 699 | "n03814639": "neck brace", 700 | "n03814906": "necklace", 701 | "n03825788": "nipple", 702 | "n03832673": "notebook, notebook computer", 703 | "n03837869": "obelisk", 704 | "n03838899": "oboe, hautboy, hautbois", 705 | "n03840681": "ocarina, sweet potato", 706 | "n03841143": "odometer, hodometer, mileometer, milometer", 707 | "n03843555": "oil filter", 708 | "n03854065": "organ, pipe organ", 709 | "n03857828": "oscilloscope, scope, cathode-ray oscilloscope, CRO", 710 | "n03866082": "overskirt", 711 | "n03868242": "oxcart", 712 | "n03868863": "oxygen mask", 713 | "n03871628": "packet", 714 | "n03873416": "paddle, boat paddle", 715 | "n03874293": "paddlewheel, paddle wheel", 716 | "n03874599": "padlock", 717 | "n03876231": "paintbrush", 718 | "n03877472": "pajama, pyjama, pj's, jammies", 719 | "n03877845": "palace", 720 | "n03884397": "panpipe, pandean pipe, syrinx", 721 | "n03887697": "paper towel", 722 | "n03888257": "parachute, chute", 723 | "n03888605": "parallel bars, bars", 724 | "n03891251": "park bench", 725 | "n03891332": "parking meter", 726 | "n03895866": "passenger car, coach, carriage", 727 | "n03899768": "patio, terrace", 728 | "n03902125": "pay-phone, pay-station", 729 | "n03903868": "pedestal, plinth, footstall", 730 | "n03908618": "pencil box, pencil case", 731 | "n03908714": "pencil sharpener", 732 | "n03916031": "perfume, essence", 733 | "n03920288": "Petri dish", 734 | "n03924679": "photocopier", 735 | "n03929660": "pick, plectrum, plectron", 736 | "n03929855": "pickelhaube", 737 | "n03930313": "picket fence, paling", 738 | "n03930630": "pickup, pickup truck", 739 | "n03933933": "pier", 740 | "n03935335": "piggy bank, penny bank", 741 | "n03937543": "pill bottle", 742 | "n03938244": "pillow", 743 | "n03942813": "ping-pong ball", 744 | "n03944341": "pinwheel", 745 | "n03947888": "pirate, pirate ship", 746 | "n03950228": "pitcher, ewer", 747 | "n03954731": "plane, carpenter's plane, woodworking plane", 748 | "n03956157": "planetarium", 749 | "n03958227": "plastic bag", 750 | "n03961711": "plate rack", 751 | "n03967562": "plow, plough", 752 | "n03970156": "plunger, plumber's helper", 753 | "n03976467": "Polaroid camera, Polaroid Land camera", 754 | "n03976657": "pole", 755 | "n03977966": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", 756 | "n03980874": "poncho", 757 | "n03982430": "pool table, billiard table, snooker table", 758 | "n03983396": "pop bottle, soda bottle", 759 | "n03991062": "pot, flowerpot", 760 | "n03992509": "potter's wheel", 761 | "n03995372": "power drill", 762 | "n03998194": "prayer rug, prayer mat", 763 | "n04004767": "printer", 764 | "n04005630": "prison, prison house", 765 | "n04008634": "projectile, missile", 766 | "n04009552": "projector", 767 | "n04019541": "puck, hockey puck", 768 | "n04023962": "punching bag, punch bag, punching ball, punchball", 769 | "n04026417": "purse", 770 | "n04033901": "quill, quill pen", 771 | "n04033995": "quilt, comforter, comfort, puff", 772 | "n04037443": "racer, race car, racing car", 773 | "n04039381": "racket, racquet", 774 | "n04040759": "radiator", 775 | "n04041544": "radio, wireless", 776 | "n04044716": "radio telescope, radio reflector", 777 | "n04049303": "rain barrel", 778 | "n04065272": "recreational vehicle, RV, R.V.", 779 | "n04067472": "reel", 780 | "n04069434": "reflex camera", 781 | "n04070727": "refrigerator, icebox", 782 | "n04074963": "remote control, remote", 783 | "n04081281": "restaurant, eating house, eating place, eatery", 784 | "n04086273": "revolver, six-gun, six-shooter", 785 | "n04090263": "rifle", 786 | "n04099969": "rocking chair, rocker", 787 | "n04111531": "rotisserie", 788 | "n04116512": "rubber eraser, rubber, pencil eraser", 789 | "n04118538": "rugby ball", 790 | "n04118776": "rule, ruler", 791 | "n04120489": "running shoe", 792 | "n04125021": "safe", 793 | "n04127249": "safety pin", 794 | "n04131690": "saltshaker, salt shaker", 795 | "n04133789": "sandal", 796 | "n04136333": "sarong", 797 | "n04141076": "sax, saxophone", 798 | "n04141327": "scabbard", 799 | "n04141975": "scale, weighing machine", 800 | "n04146614": "school bus", 801 | "n04147183": "schooner", 802 | "n04149813": "scoreboard", 803 | "n04152593": "screen, CRT screen", 804 | "n04153751": "screw", 805 | "n04154565": "screwdriver", 806 | "n04162706": "seat belt, seatbelt", 807 | "n04179913": "sewing machine", 808 | "n04192698": "shield, buckler", 809 | "n04200800": "shoe shop, shoe-shop, shoe store", 810 | "n04201297": "shoji", 811 | "n04204238": "shopping basket", 812 | "n04204347": "shopping cart", 813 | "n04208210": "shovel", 814 | "n04209133": "shower cap", 815 | "n04209239": "shower curtain", 816 | "n04228054": "ski", 817 | "n04229816": "ski mask", 818 | "n04235860": "sleeping bag", 819 | "n04238763": "slide rule, slipstick", 820 | "n04239074": "sliding door", 821 | "n04243546": "slot, one-armed bandit", 822 | "n04251144": "snorkel", 823 | "n04252077": "snowmobile", 824 | "n04252225": "snowplow, snowplough", 825 | "n04254120": "soap dispenser", 826 | "n04254680": "soccer ball", 827 | "n04254777": "sock", 828 | "n04258138": "solar dish, solar collector, solar furnace", 829 | "n04259630": "sombrero", 830 | "n04263257": "soup bowl", 831 | "n04264628": "space bar", 832 | "n04265275": "space heater", 833 | "n04266014": "space shuttle", 834 | "n04270147": "spatula", 835 | "n04273569": "speedboat", 836 | "n04275548": "spider web, spider's web", 837 | "n04277352": "spindle", 838 | "n04285008": "sports car, sport car", 839 | "n04286575": "spotlight, spot", 840 | "n04296562": "stage", 841 | "n04310018": "steam locomotive", 842 | "n04311004": "steel arch bridge", 843 | "n04311174": "steel drum", 844 | "n04317175": "stethoscope", 845 | "n04325704": "stole", 846 | "n04326547": "stone wall", 847 | "n04328186": "stopwatch, stop watch", 848 | "n04330267": "stove", 849 | "n04332243": "strainer", 850 | "n04335435": "streetcar, tram, tramcar, trolley, trolley car", 851 | "n04336792": "stretcher", 852 | "n04344873": "studio couch, day bed", 853 | "n04346328": "stupa, tope", 854 | "n04347754": "submarine, pigboat, sub, U-boat", 855 | "n04350905": "suit, suit of clothes", 856 | "n04355338": "sundial", 857 | "n04355933": "sunglass", 858 | "n04356056": "sunglasses, dark glasses, shades", 859 | "n04357314": "sunscreen, sunblock, sun blocker", 860 | "n04366367": "suspension bridge", 861 | "n04367480": "swab, swob, mop", 862 | "n04370456": "sweatshirt", 863 | "n04371430": "swimming trunks, bathing trunks", 864 | "n04371774": "swing", 865 | "n04372370": "switch, electric switch, electrical switch", 866 | "n04376876": "syringe", 867 | "n04380533": "table lamp", 868 | "n04389033": "tank, army tank, armored combat vehicle, armoured combat vehicle", 869 | "n04392985": "tape player", 870 | "n04398044": "teapot", 871 | "n04399382": "teddy, teddy bear", 872 | "n04404412": "television, television system", 873 | "n04409515": "tennis ball", 874 | "n04417672": "thatch, thatched roof", 875 | "n04418357": "theater curtain, theatre curtain", 876 | "n04423845": "thimble", 877 | "n04428191": "thresher, thrasher, threshing machine", 878 | "n04429376": "throne", 879 | "n04435653": "tile roof", 880 | "n04442312": "toaster", 881 | "n04443257": "tobacco shop, tobacconist shop, tobacconist", 882 | "n04447861": "toilet seat", 883 | "n04456115": "torch", 884 | "n04458633": "totem pole", 885 | "n04461696": "tow truck, tow car, wrecker", 886 | "n04462240": "toyshop", 887 | "n04465501": "tractor", 888 | "n04467665": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", 889 | "n04476259": "tray", 890 | "n04479046": "trench coat", 891 | "n04482393": "tricycle, trike, velocipede", 892 | "n04483307": "trimaran", 893 | "n04485082": "tripod", 894 | "n04486054": "triumphal arch", 895 | "n04487081": "trolleybus, trolley coach, trackless trolley", 896 | "n04487394": "trombone", 897 | "n04493381": "tub, vat", 898 | "n04501370": "turnstile", 899 | "n04505470": "typewriter keyboard", 900 | "n04507155": "umbrella", 901 | "n04509417": "unicycle, monocycle", 902 | "n04515003": "upright, upright piano", 903 | "n04517823": "vacuum, vacuum cleaner", 904 | "n04522168": "vase", 905 | "n04523525": "vault", 906 | "n04525038": "velvet", 907 | "n04525305": "vending machine", 908 | "n04532106": "vestment", 909 | "n04532670": "viaduct", 910 | "n04536866": "violin, fiddle", 911 | "n04540053": "volleyball", 912 | "n04542943": "waffle iron", 913 | "n04548280": "wall clock", 914 | "n04548362": "wallet, billfold, notecase, pocketbook", 915 | "n04550184": "wardrobe, closet, press", 916 | "n04552348": "warplane, military plane", 917 | "n04553703": "washbasin, handbasin, washbowl, lavabo, wash-hand basin", 918 | "n04554684": "washer, automatic washer, washing machine", 919 | "n04557648": "water bottle", 920 | "n04560804": "water jug", 921 | "n04562935": "water tower", 922 | "n04579145": "whiskey jug", 923 | "n04579432": "whistle", 924 | "n04584207": "wig", 925 | "n04589890": "window screen", 926 | "n04590129": "window shade", 927 | "n04591157": "Windsor tie", 928 | "n04591713": "wine bottle", 929 | "n04592741": "wing", 930 | "n04596742": "wok", 931 | "n04597913": "wooden spoon", 932 | "n04599235": "wool, woolen, woollen", 933 | "n04604644": "worm fence, snake fence, snake-rail fence, Virginia fence", 934 | "n04606251": "wreck", 935 | "n04612504": "yawl", 936 | "n04613696": "yurt", 937 | "n06359193": "web site, website, internet site, site", 938 | "n06596364": "comic book", 939 | "n06785654": "crossword puzzle, crossword", 940 | "n06794110": "street sign", 941 | "n06874185": "traffic light, traffic signal, stoplight", 942 | "n07248320": "book jacket, dust cover, dust jacket, dust wrapper", 943 | "n07565083": "menu", 944 | "n07579787": "plate", 945 | "n07583066": "guacamole", 946 | "n07584110": "consomme", 947 | "n07590611": "hot pot, hotpot", 948 | "n07613480": "trifle", 949 | "n07614500": "ice cream, icecream", 950 | "n07615774": "ice lolly, lolly, lollipop, popsicle", 951 | "n07684084": "French loaf", 952 | "n07693725": "bagel, beigel", 953 | "n07695742": "pretzel", 954 | "n07697313": "cheeseburger", 955 | "n07697537": "hotdog, hot dog, red hot", 956 | "n07711569": "mashed potato", 957 | "n07714571": "head cabbage", 958 | "n07714990": "broccoli", 959 | "n07715103": "cauliflower", 960 | "n07716358": "zucchini, courgette", 961 | "n07716906": "spaghetti squash", 962 | "n07717410": "acorn squash", 963 | "n07717556": "butternut squash", 964 | "n07718472": "cucumber, cuke", 965 | "n07718747": "artichoke, globe artichoke", 966 | "n07720875": "bell pepper", 967 | "n07730033": "cardoon", 968 | "n07734744": "mushroom", 969 | "n07742313": "Granny Smith", 970 | "n07745940": "strawberry", 971 | "n07747607": "orange", 972 | "n07749582": "lemon", 973 | "n07753113": "fig", 974 | "n07753275": "pineapple, ananas", 975 | "n07753592": "banana", 976 | "n07754684": "jackfruit, jak, jack", 977 | "n07760859": "custard apple", 978 | "n07768694": "pomegranate", 979 | "n07802026": "hay", 980 | "n07831146": "carbonara", 981 | "n07836838": "chocolate sauce, chocolate syrup", 982 | "n07860988": "dough", 983 | "n07871810": "meat loaf, meatloaf", 984 | "n07873807": "pizza, pizza pie", 985 | "n07875152": "potpie", 986 | "n07880968": "burrito", 987 | "n07892512": "red wine", 988 | "n07920052": "espresso", 989 | "n07930864": "cup", 990 | "n07932039": "eggnog", 991 | "n09193705": "alp", 992 | "n09229709": "bubble", 993 | "n09246464": "cliff, drop, drop-off", 994 | "n09256479": "coral reef", 995 | "n09288635": "geyser", 996 | "n09332890": "lakeside, lakeshore", 997 | "n09399592": "promontory, headland, head, foreland", 998 | "n09421951": "sandbar, sand bar", 999 | "n09428293": "seashore, coast, seacoast, sea-coast", 1000 | "n09468604": "valley, vale", 1001 | "n09472597": "volcano", 1002 | "n09835506": "ballplayer, baseball player", 1003 | "n10148035": "groom, bridegroom", 1004 | "n10565667": "scuba diver", 1005 | "n11879895": "rapeseed", 1006 | "n11939491": "daisy", 1007 | "n12057211": "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", 1008 | "n12144580": "corn", 1009 | "n12267677": "acorn", 1010 | "n12620546": "hip, rose hip, rosehip", 1011 | "n12768682": "buckeye, horse chestnut, conker", 1012 | "n12985857": "coral fungus", 1013 | "n12998815": "agaric", 1014 | "n13037406": "gyromitra", 1015 | "n13040303": "stinkhorn, carrion fungus", 1016 | "n13044778": "earthstar", 1017 | "n13052670": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", 1018 | "n13054560": "bolete", 1019 | "n13133613": "ear, spike, capitulum", 1020 | "n15075141": "toilet tissue, toilet paper, bathroom tissue", 1021 | } 1022 | ) 1023 | -------------------------------------------------------------------------------- /make_preprocessing_onnx.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | from typing import List 3 | 4 | import numpy as np 5 | import onnx 6 | import torch 7 | import torch.nn as nn 8 | from onnxsim import simplify 9 | 10 | 11 | class Preprocess(nn.Module): 12 | def __init__(self, input_shape: List[int]): 13 | super(Preprocess, self).__init__() 14 | self.input_shape = tuple(input_shape) 15 | self.mean = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1) 16 | self.std = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1) 17 | 18 | # self.register_buffer( 19 | # "mean", torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1) 20 | # ) 21 | # self.register_buffer( 22 | # "std", torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1) 23 | # ) 24 | 25 | def forward(self, x: torch.Tensor): 26 | x = torch.nn.functional.interpolate( 27 | input=x, 28 | size=self.input_shape[2:], 29 | ) 30 | # x = x * (1.0 / 255.0) 31 | x = x / 255.0 32 | x = (x - self.mean) / self.std 33 | 34 | return x 35 | 36 | 37 | if __name__ == "__main__": 38 | parser = ArgumentParser() 39 | parser.add_argument("-o", "--opset", type=int, default=20, help="onnx opset") 40 | parser.add_argument( 41 | "-s", 42 | "--input_shape", 43 | type=int, 44 | nargs=4, 45 | default=[1, 3, 448, 448], 46 | help="input shape", 47 | ) 48 | 49 | args = parser.parse_args() 50 | 51 | MODEL = f"01_prep" 52 | OPSET = args.opset 53 | INPUT_SHAPE: List[int] = args.input_shape 54 | 55 | model = Preprocess(input_shape=INPUT_SHAPE) 56 | 57 | onnx_file = f"{MODEL}_{'_'.join(map(str, INPUT_SHAPE))}.onnx" 58 | x = torch.randn(INPUT_SHAPE) 59 | 60 | torch.onnx.export( 61 | model, 62 | args=(x), 63 | f=onnx_file, 64 | opset_version=OPSET, 65 | input_names=["input_rgb"], 66 | output_names=["output_prep"], 67 | dynamic_axes={ 68 | "input_rgb": { 69 | 2: "H", 70 | 3: "W", 71 | }, 72 | }, 73 | ) 74 | model_onnx1 = onnx.load(onnx_file) 75 | model_onnx1 = onnx.shape_inference.infer_shapes(model_onnx1) 76 | onnx.save(model_onnx1, onnx_file) 77 | 78 | model_onnx2 = onnx.load(onnx_file) 79 | model_simp, check = simplify(model_onnx2) 80 | onnx.save(model_simp, onnx_file) 81 | -------------------------------------------------------------------------------- /merge_preprocessing_to_model.py: -------------------------------------------------------------------------------- 1 | from snc4onnx import combine 2 | 3 | combined = combine( 4 | srcop_destop=["output_prep", "input"], 5 | input_onnx_file_paths=[ 6 | "01_prep_1_3_448_448.onnx", 7 | "eva02_large_patch14_448.onnx", 8 | ], 9 | op_prefixes_after_merging=["prep", "model"], 10 | ) 11 | 12 | 13 | combined.save("merged_preprocessing_to_model.onnx") 14 | 15 | 16 | # snc4onnx --input_onnx_file_paths 01_prep_1_3_448_448.onnx eva02_large_patch14_448.onnx --output_onnx_file_path merged.onnx --srcop_destop output_prep input --op_prefixes_after_merging init next 17 | -------------------------------------------------------------------------------- /notebooks/01_convert_to_onnx.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "data": { 10 | "text/plain": [ 11 | "Eva(\n", 12 | " (patch_embed): PatchEmbed(\n", 13 | " (proj): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14))\n", 14 | " (norm): Identity()\n", 15 | " )\n", 16 | " (pos_drop): Dropout(p=0.0, inplace=False)\n", 17 | " (rope): RotaryEmbeddingCat()\n", 18 | " (blocks): ModuleList(\n", 19 | " (0-23): 24 x EvaBlock(\n", 20 | " (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", 21 | " (attn): EvaAttention(\n", 22 | " (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n", 23 | " (k_proj): Linear(in_features=1024, out_features=1024, bias=False)\n", 24 | " (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n", 25 | " (attn_drop): Dropout(p=0.0, inplace=False)\n", 26 | " (norm): Identity()\n", 27 | " (proj): Linear(in_features=1024, out_features=1024, bias=True)\n", 28 | " (proj_drop): Dropout(p=0.0, inplace=False)\n", 29 | " )\n", 30 | " (drop_path1): Identity()\n", 31 | " (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", 32 | " (mlp): SwiGLU(\n", 33 | " (fc1_g): Linear(in_features=1024, out_features=2730, bias=True)\n", 34 | " (fc1_x): Linear(in_features=1024, out_features=2730, bias=True)\n", 35 | " (act): SiLU()\n", 36 | " (drop1): Dropout(p=0.0, inplace=False)\n", 37 | " (norm): LayerNorm((2730,), eps=1e-06, elementwise_affine=True)\n", 38 | " (fc2): Linear(in_features=2730, out_features=1024, bias=True)\n", 39 | " (drop2): Dropout(p=0.0, inplace=False)\n", 40 | " )\n", 41 | " (drop_path2): Identity()\n", 42 | " )\n", 43 | " )\n", 44 | " (norm): Identity()\n", 45 | " (fc_norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n", 46 | " (head_drop): Dropout(p=0.0, inplace=False)\n", 47 | " (head): Linear(in_features=1024, out_features=1000, bias=True)\n", 48 | ")" 49 | ] 50 | }, 51 | "execution_count": 1, 52 | "metadata": {}, 53 | "output_type": "execute_result" 54 | } 55 | ], 56 | "source": [ 57 | "import timm\n", 58 | "import torch\n", 59 | "\n", 60 | "model = timm.create_model('eva02_large_patch14_448.mim_m38m_ft_in22k_in1k', pretrained=True)\n", 61 | "model.eval()" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 2, 67 | "metadata": {}, 68 | "outputs": [ 69 | { 70 | "name": "stderr", 71 | "output_type": "stream", 72 | "text": [ 73 | "/home/dnth/mambaforge-pypy3/envs/pt-to-onnx-tensorrt/lib/python3.11/site-packages/torch/__init__.py:1777: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", 74 | " assert condition, message\n" 75 | ] 76 | } 77 | ], 78 | "source": [ 79 | "torch.onnx.export(model, \n", 80 | " torch.randn(1, 3, 448, 448), \n", 81 | " \"eva02_large_patch14_448.onnx\", \n", 82 | " export_params=True, \n", 83 | " opset_version=18,\n", 84 | " do_constant_folding=True, \n", 85 | " input_names=['input'],\n", 86 | " output_names=['output'],\n", 87 | " dynamic_axes={'input': {0: 'batch_size'},\n", 88 | " 'output': {0: 'batch_size'}})" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": {}, 95 | "outputs": [], 96 | "source": [] 97 | } 98 | ], 99 | "metadata": { 100 | "kernelspec": { 101 | "display_name": "pt-to-onnx-tensorrt", 102 | "language": "python", 103 | "name": "python3" 104 | }, 105 | "language_info": { 106 | "codemirror_mode": { 107 | "name": "ipython", 108 | "version": 3 109 | }, 110 | "file_extension": ".py", 111 | "mimetype": "text/x-python", 112 | "name": "python", 113 | "nbconvert_exporter": "python", 114 | "pygments_lexer": "ipython3", 115 | "version": "3.11.9" 116 | } 117 | }, 118 | "nbformat": 4, 119 | "nbformat_minor": 2 120 | } 121 | -------------------------------------------------------------------------------- /notebooks/03_benchmark_onnxruntime_trt.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "data": { 10 | "text/plain": [ 11 | "['TensorrtExecutionProvider', 'CUDAExecutionProvider', 'CPUExecutionProvider']" 12 | ] 13 | }, 14 | "execution_count": 1, 15 | "metadata": {}, 16 | "output_type": "execute_result" 17 | } 18 | ], 19 | "source": [ 20 | "import onnxruntime as ort\n", 21 | "\n", 22 | "# get execution provider\n", 23 | "providers = ort.get_available_providers()\n", 24 | "providers" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 2, 30 | "metadata": {}, 31 | "outputs": [ 32 | { 33 | "name": "stdout", 34 | "output_type": "stream", 35 | "text": [ 36 | "env: LD_LIBRARY_PATH=/home/dnth/mambaforge-pypy3/envs/ram-onnx-inference/lib/python3.11/site-packages/tensorrt_libs:/home/dnth/mambaforge-pypy3/envs/ram-onnx-inference/lib:$LD_LIBRARY_PATH\n" 37 | ] 38 | } 39 | ], 40 | "source": [ 41 | "%env LD_LIBRARY_PATH=/home/dnth/mambaforge-pypy3/envs/ram-onnx-inference/lib/python3.11/site-packages/tensorrt_libs:/home/dnth/mambaforge-pypy3/envs/ram-onnx-inference/lib:$LD_LIBRARY_PATH" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 3, 47 | "metadata": {}, 48 | "outputs": [ 49 | { 50 | "name": "stdout", 51 | "output_type": "stream", 52 | "text": [ 53 | "*************** EP Error ***************\n", 54 | "EP Error /onnxruntime_src/onnxruntime/python/onnxruntime_pybind_state.cc:490 void onnxruntime::python::RegisterTensorRTPluginsAsCustomOps(PySessionOptions&, const onnxruntime::ProviderOptions&) Please install TensorRT libraries as mentioned in the GPU requirements page, make sure they're in the PATH or LD_LIBRARY_PATH, and that your GPU is supported.\n", 55 | " when using ['TensorrtExecutionProvider']\n", 56 | "Falling back to ['CPUExecutionProvider'] and retrying.\n", 57 | "****************************************\n" 58 | ] 59 | }, 60 | { 61 | "name": "stderr", 62 | "output_type": "stream", 63 | "text": [ 64 | "\u001b[1;31m2024-09-10 11:19:52.296452091 [E:onnxruntime:Default, provider_bridge_ort.cc:1978 TryGetProviderInfo_TensorRT] /onnxruntime_src/onnxruntime/core/session/provider_bridge_ort.cc:1637 onnxruntime::Provider& onnxruntime::ProviderLibrary::Get() [ONNXRuntimeError] : 1 : FAIL : Failed to load library libonnxruntime_providers_tensorrt.so with error: libcudnn.so.9: cannot open shared object file: No such file or directory\n", 65 | "\u001b[m\n" 66 | ] 67 | }, 68 | { 69 | "data": { 70 | "text/plain": [ 71 | "[array([[-1.77464217e-01, -1.13817573e-01, -8.76026601e-02,\n", 72 | " 8.69853124e-02, 7.27204144e-01, -4.62081969e-01,\n", 73 | " -1.76158428e-01, -3.96329612e-01, -3.04876298e-01,\n", 74 | " -3.45760286e-01, -4.94259745e-01, -1.62426859e-01,\n", 75 | " -1.31224275e-01, -1.17028162e-01, 1.04696602e-02,\n", 76 | " -1.04521327e-01, -2.02701300e-01, -6.22882172e-02,\n", 77 | " -1.12694219e-01, -6.03306115e-01, -4.64288175e-01,\n", 78 | " 2.93670595e-02, -1.33947253e-01, 2.18217090e-01,\n", 79 | " -5.31307459e-01, 4.84423935e-02, -3.59963179e-02,\n", 80 | " -5.88736832e-01, 3.68650526e-01, -4.85611260e-01,\n", 81 | " -2.56656110e-01, -4.01910871e-01, 5.98749518e-03,\n", 82 | " -9.85038280e-02, -8.46037418e-02, -2.06972569e-01,\n", 83 | " -2.61949062e-01, -3.54980081e-02, 1.25174671e-01,\n", 84 | " -1.63902849e-01, 2.21279696e-01, -1.22567803e-01,\n", 85 | " -5.68985939e-04, -1.88159660e-01, 3.55500519e-01,\n", 86 | " -1.33886173e-01, 3.23494673e-01, -1.90455168e-02,\n", 87 | " -6.46073669e-02, -3.13407779e-01, 8.89043927e-01,\n", 88 | " -7.02455193e-02, -1.69803768e-01, 7.65771031e-01,\n", 89 | " 3.57189834e-01, -6.99264109e-02, 1.13991372e-01,\n", 90 | " 5.11011481e-01, 1.01663852e+00, -3.33006889e-01,\n", 91 | " 8.52113247e-01, -3.76981884e-01, -5.35032868e-01,\n", 92 | " -4.16074604e-01, -2.83766747e-01, -2.41838843e-02,\n", 93 | " -2.30156839e-01, 5.25773406e-01, 6.43586516e-01,\n", 94 | " -4.40973163e-01, -5.82627475e-01, 3.82346660e-01,\n", 95 | " -2.07558900e-01, -5.58340251e-02, -7.63309598e-01,\n", 96 | " 6.67417571e-02, 2.53365308e-01, 1.82066008e-01,\n", 97 | " -5.37919849e-02, 3.34096521e-01, -8.22463185e-02,\n", 98 | " 1.93856031e-01, -7.84156024e-02, 1.56283706e-01,\n", 99 | " -4.87867951e-01, -4.50030603e-02, -8.36386681e-02,\n", 100 | " -7.67570436e-02, -2.38570601e-01, 1.75680220e-02,\n", 101 | " 9.32375044e-02, 1.90884814e-01, -2.39564031e-02,\n", 102 | " -3.05747688e-02, -2.75674403e-01, -9.52119678e-02,\n", 103 | " 2.81520635e-01, -4.02555913e-01, -4.68971759e-01,\n", 104 | " -4.88477170e-01, 2.78480887e-01, -2.67830014e-01,\n", 105 | " -5.18074751e-01, -2.28291705e-01, -3.38255107e-01,\n", 106 | " -5.15162982e-02, -7.54908770e-02, 1.29446536e-01,\n", 107 | " 1.92965880e-01, 1.74253285e-01, 2.84720063e-01,\n", 108 | " -1.04750246e-02, -1.00734249e-01, 9.46742296e-03,\n", 109 | " 3.77253175e-01, 4.21410739e-01, 4.39258963e-01,\n", 110 | " -1.05737224e-01, 7.04464734e-01, 2.99329758e-02,\n", 111 | " 8.26250076e-01, -2.69201130e-01, 5.27863264e-01,\n", 112 | " 6.29038751e-01, 1.84068441e+00, 5.62932789e-02,\n", 113 | " -4.18883979e-01, 1.53198168e-01, 2.18420625e-01,\n", 114 | " 5.53384870e-02, 1.85985208e-01, 8.75055373e-01,\n", 115 | " 2.13766634e-01, -3.89693856e-01, 2.10258394e-01,\n", 116 | " 8.44119906e-01, 3.49457189e-02, 9.82176363e-02,\n", 117 | " 1.65827632e-01, 4.51901555e-01, -4.78691310e-02,\n", 118 | " -5.44361889e-01, -1.63823187e-01, 1.94487318e-01,\n", 119 | " 4.23280150e-01, -3.20765913e-01, 5.87064624e-02,\n", 120 | " -4.03398126e-01, 5.96557558e-03, -3.49945605e-01,\n", 121 | " -6.49668694e-01, -2.03207284e-02, 4.66288924e-02,\n", 122 | " -2.80985057e-01, -4.04974759e-01, 2.34476238e-01,\n", 123 | " 2.61193126e-01, -2.12668031e-01, 3.51347625e-01,\n", 124 | " -1.41420960e-01, -6.90813780e-01, -1.99500889e-01,\n", 125 | " -3.08874607e-01, -3.82496208e-01, 3.77013952e-01,\n", 126 | " 3.53726089e-01, 2.47255981e-01, -3.20007652e-01,\n", 127 | " 4.27977145e-02, -2.74951398e-01, -6.36559784e-01,\n", 128 | " 2.80528396e-01, 2.65734464e-01, 3.94510537e-01,\n", 129 | " -5.69125295e-01, -1.20482326e-01, -5.64793169e-01,\n", 130 | " -2.62564629e-01, -3.48842949e-01, 8.60685110e-03,\n", 131 | " 7.71559924e-02, -2.89599389e-01, -5.71022451e-01,\n", 132 | " 3.05012047e-01, -1.15843281e-01, -8.17696631e-01,\n", 133 | " -1.57954305e-01, -1.67604670e-01, 4.64762092e-01,\n", 134 | " -3.56300771e-02, -2.50135303e-01, -2.66177088e-01,\n", 135 | " -2.41423994e-02, 2.47128636e-01, -2.68848628e-01,\n", 136 | " 7.01069981e-02, -6.89039111e-01, -2.95270920e-01,\n", 137 | " -3.40092659e-01, -4.15817738e-01, -4.90226448e-01,\n", 138 | " -3.19008887e-01, -2.14867532e-01, -7.13696241e-01,\n", 139 | " -7.41977394e-01, 3.82132083e-02, 1.26779079e-02,\n", 140 | " -6.26951218e-01, -5.92160404e-01, -2.13213146e-01,\n", 141 | " 3.36710513e-02, -8.32172483e-03, -3.43653172e-01,\n", 142 | " -6.95916951e-01, 2.55891293e-01, -4.45427686e-01,\n", 143 | " -4.42491621e-01, 8.85643065e-02, -2.65711367e-01,\n", 144 | " -2.08743125e-01, -7.56696314e-02, 4.83546644e-01,\n", 145 | " -1.87744990e-01, -4.55180347e-01, -5.10810852e-01,\n", 146 | " -8.75309467e-01, -1.07435331e-01, -5.15348852e-01,\n", 147 | " 1.10101119e-01, 1.25468075e-02, -1.03986239e+00,\n", 148 | " -1.90422997e-01, -3.84586513e-01, 8.59128833e-02,\n", 149 | " -3.29156369e-01, -1.07874990e+00, -1.62719607e-01,\n", 150 | " 8.44270885e-02, -8.78635108e-01, -5.09217381e-01,\n", 151 | " -4.87684608e-01, -4.43978071e-01, 5.30126318e-03,\n", 152 | " -3.51074785e-01, -6.76186323e-01, 1.09009534e-01,\n", 153 | " -1.36595249e-01, -3.72610211e-01, -2.17524469e-01,\n", 154 | " -2.23059744e-01, 1.18611678e-01, 2.65269697e-01,\n", 155 | " -2.42847294e-01, 5.11406422e-01, -1.05870575e-01,\n", 156 | " -2.31891960e-01, -3.50223780e-01, -4.12743896e-01,\n", 157 | " 9.86467302e-02, -4.85073358e-01, -5.41442275e-01,\n", 158 | " -2.54220188e-01, -8.50616619e-02, -7.26491958e-02,\n", 159 | " 1.31194517e-02, -1.46089256e-01, -1.43315762e-01,\n", 160 | " 2.29069054e-01, -5.34215681e-02, 2.45014071e-01,\n", 161 | " -5.36144614e-01, -3.32225859e-03, -2.03195661e-02,\n", 162 | " -2.31379688e-01, 5.46421409e-02, -2.52851158e-01,\n", 163 | " -5.14267981e-02, -6.20141387e-01, 5.62115610e-02,\n", 164 | " -2.16934606e-01, 2.27291018e-01, -2.88829654e-01,\n", 165 | " -3.34064364e-02, -1.82651773e-01, -3.30626428e-01,\n", 166 | " 3.91619742e-01, 1.26392990e-01, 5.96821308e-01,\n", 167 | " 4.27756011e-01, 4.69976187e-01, -4.24002595e-02,\n", 168 | " 1.39131755e-01, -2.92074114e-01, 2.16686875e-02,\n", 169 | " -8.07516724e-02, -1.09904513e-01, -7.46341825e-01,\n", 170 | " 2.13432506e-01, 1.05391309e-01, -1.98773950e-01,\n", 171 | " 7.06175447e-01, -3.81297708e-01, 5.08826733e-01,\n", 172 | " 3.02942321e-02, 1.64421186e-01, -5.53950667e-03,\n", 173 | " 1.61662586e-02, -1.98371351e-01, -1.53448373e-01,\n", 174 | " 7.51566589e-02, 1.78597599e-01, -1.32668138e-01,\n", 175 | " -3.50967765e-01, -2.45587617e-01, -1.48882493e-02,\n", 176 | " 1.01462677e-01, 2.15802208e-01, -8.10622692e-01,\n", 177 | " -2.75331855e-01, 5.99182472e-02, 2.97598004e-01,\n", 178 | " 3.59772742e-01, 8.16802457e-02, 2.19432622e-01,\n", 179 | " -8.60891044e-01, -6.11777902e-01, -4.78285313e-01,\n", 180 | " -7.55512118e-02, 8.37966800e-04, -2.24770024e-01,\n", 181 | " 4.83425915e-01, 2.95670867e-01, 3.76589954e-01,\n", 182 | " 1.00391909e-01, -3.19647372e-01, 7.48296142e-01,\n", 183 | " 4.25719649e-01, 1.19544961e-01, -1.37200996e-01,\n", 184 | " 1.34158537e-01, 3.25269192e-01, -4.37173426e-01,\n", 185 | " -8.07812333e-01, -2.55575657e-01, -4.34796065e-01,\n", 186 | " 3.50539505e-01, 1.10087171e-01, -1.56987712e-01,\n", 187 | " -5.91273904e-01, 2.68931575e-02, 1.55328959e-02,\n", 188 | " -2.94929802e-01, -1.12215281e-02, 4.72748518e-01,\n", 189 | " -8.43109727e-01, -4.05083239e-01, -3.28638792e-01,\n", 190 | " 3.36347103e-01, -3.63820106e-01, -8.13711762e-01,\n", 191 | " -4.39699858e-01, -4.08574522e-01, -2.63692826e-01,\n", 192 | " -1.88774228e-01, -8.19120109e-01, -4.01011229e-01,\n", 193 | " 2.21700668e-01, 2.38707751e-01, 2.25525618e-01,\n", 194 | " 5.60969055e-01, 5.04317135e-02, 5.07567167e-01,\n", 195 | " 2.81516880e-01, 2.98002809e-01, 2.41596997e-02,\n", 196 | " 5.92108071e-01, 1.56293094e-01, -3.08314085e-01,\n", 197 | " -6.32328510e-01, 2.37724707e-01, -7.71076828e-02,\n", 198 | " -2.01831385e-01, -9.37049270e-01, 2.82729864e-01,\n", 199 | " 6.04585350e-01, -2.49699324e-01, -2.78976202e-01,\n", 200 | " 4.01450127e-01, 1.25728428e-01, -2.37767652e-01,\n", 201 | " -1.05419919e-01, 2.01252908e-01, -1.87099189e-01,\n", 202 | " -1.21968314e-01, -6.60230279e-01, 4.70606759e-02,\n", 203 | " 3.31487089e-01, -8.41346502e-01, -6.69741929e-01,\n", 204 | " -7.45958745e-01, -3.80713880e-01, 4.55440581e-01,\n", 205 | " -1.36585414e-01, -1.45835966e-01, -3.28009039e-01,\n", 206 | " -4.86590654e-01, 4.87779558e-01, 4.17752564e-01,\n", 207 | " -5.16002059e-01, 1.41407728e-01, 3.35349441e-01,\n", 208 | " -1.45714536e-01, 1.78292304e-01, 9.65753198e-02,\n", 209 | " -5.83520830e-01, 3.22777772e+00, -4.02275503e-01,\n", 210 | " -5.39902091e-01, -3.41970444e-01, 7.50866830e-02,\n", 211 | " -1.07151248e-01, -7.87041962e-01, 1.11161619e-01,\n", 212 | " 1.12533160e-01, -2.70992994e-01, 1.74556911e-01,\n", 213 | " -5.99087715e-01, -6.63582385e-02, 1.42058842e-02,\n", 214 | " -5.38131714e-01, -3.09609950e-01, 6.30706906e-01,\n", 215 | " -2.46592417e-01, -2.66126275e-01, -8.94790113e-01,\n", 216 | " 4.52413782e-02, 2.80602336e-01, 1.75735354e-02,\n", 217 | " -4.82562006e-01, -3.55075076e-02, 3.23264077e-02,\n", 218 | " 5.03997445e-01, 6.65925920e-01, 6.47328496e-02,\n", 219 | " -2.64135092e-01, -2.51175672e-01, -8.52034271e-01,\n", 220 | " -9.85510826e-01, -6.16107821e-01, -1.26258314e-01,\n", 221 | " -2.14948386e-01, -3.99429798e-01, -8.28773379e-02,\n", 222 | " -4.99752760e-01, 6.89013302e-03, 2.95909017e-01,\n", 223 | " -2.24400818e-01, -4.78805244e-01, -5.03107190e-01,\n", 224 | " -5.37467241e-01, 7.49759674e-01, 4.87920910e-01,\n", 225 | " -5.46068907e-01, 1.12428159e-01, -2.82024324e-01,\n", 226 | " -3.87740642e-01, -8.70002061e-02, 2.63069838e-01,\n", 227 | " -5.19666612e-01, 8.55742395e-02, 9.94635224e-02,\n", 228 | " 2.69894481e-01, -2.99582005e-01, -2.29027122e-01,\n", 229 | " 5.36441803e-04, -8.56688321e-01, 3.29818785e-01,\n", 230 | " -2.26545155e-01, -2.15637982e-01, 7.62323260e-01,\n", 231 | " -9.16022122e-01, -4.56548333e-01, -5.40909886e-01,\n", 232 | " 1.54995322e-01, -4.96325135e-01, -1.26821756e-01,\n", 233 | " -6.16471410e-01, -6.33814812e-01, -3.89054716e-01,\n", 234 | " 9.69834179e-02, 3.28461140e-01, -3.45575452e-01,\n", 235 | " -1.91356406e-01, -4.09617156e-01, -4.55126435e-01,\n", 236 | " -2.49999151e-01, -9.41431105e-01, 4.21767414e-01,\n", 237 | " 9.37360048e-01, 3.96455258e-01, -7.08950758e-01,\n", 238 | " -6.08837426e-01, -3.49639177e-01, -2.38003463e-01,\n", 239 | " 2.63706779e+00, 1.44104230e+00, 1.32304132e-02,\n", 240 | " -6.55517757e-01, 1.08423628e-01, 9.28785563e-01,\n", 241 | " -2.49366611e-01, 9.27115083e-02, 7.37869143e-02,\n", 242 | " 3.84777188e-02, 6.52380586e-02, -1.50658444e-01,\n", 243 | " -5.90786695e-01, -3.80618334e-01, -7.40501046e-01,\n", 244 | " -1.83240369e-01, -1.20210767e-01, -3.65322679e-01,\n", 245 | " -3.22332352e-01, 9.23897743e-01, 3.07802320e-01,\n", 246 | " 3.64573777e-01, -3.26991975e-02, -3.77545834e-01,\n", 247 | " -6.98603511e-01, -5.45985162e-01, -7.46386826e-01,\n", 248 | " -1.11383080e+00, 3.28298926e-01, 3.56135070e-01,\n", 249 | " -1.00851142e+00, -3.11923921e-02, -4.02723283e-01,\n", 250 | " -1.17862821e-01, 3.38738054e-01, 2.42641211e-01,\n", 251 | " -3.90804589e-01, -1.78951204e-01, -7.00642943e-01,\n", 252 | " 8.05476606e-02, 3.43829691e-02, -5.59280515e-01,\n", 253 | " -7.08073795e-01, 1.01597607e-03, -7.27058798e-02,\n", 254 | " 4.17627871e-01, 1.76746154e+00, 9.13609266e-01,\n", 255 | " 3.12057436e-02, 2.73915708e-01, -7.09260225e-01,\n", 256 | " 7.06747174e-03, -1.04995608e+00, -2.78643131e-01,\n", 257 | " -1.38838664e-01, 7.57192016e-01, -5.63753843e-01,\n", 258 | " 4.86876756e-01, 4.05863971e-01, -1.33373708e-01,\n", 259 | " -3.33112061e-01, 1.39927238e-01, -9.03254151e-02,\n", 260 | " -3.90286833e-01, -5.00543833e-01, -8.89284313e-02,\n", 261 | " -6.15720689e-01, 3.07996631e-01, -1.03830077e-01,\n", 262 | " -8.79454687e-02, 1.78284839e-01, 2.27659672e-01,\n", 263 | " -2.66644418e-01, -2.29847074e-01, -1.66398764e-01,\n", 264 | " -6.78631961e-01, -1.89330727e-02, 3.81144881e-01,\n", 265 | " 1.25110292e+00, -9.84286785e-01, 2.53791958e-01,\n", 266 | " -9.10947859e-01, 3.85231555e-01, -2.79844701e-01,\n", 267 | " -6.24244869e-01, -6.35167480e-01, -6.29135013e-01,\n", 268 | " 3.04925144e-01, -2.52223015e-01, -6.90030038e-01,\n", 269 | " -4.09949005e-01, -1.43751353e-01, -2.13335320e-01,\n", 270 | " -5.62997699e-01, -2.68459439e-01, 4.11153376e-01,\n", 271 | " -2.95673877e-01, 5.89060485e-01, -1.37594789e-02,\n", 272 | " 1.31722748e-01, -4.46539283e-01, -1.02505124e+00,\n", 273 | " -1.10283673e+00, -1.02476358e-01, -2.56328970e-01,\n", 274 | " 4.20407832e-01, 2.47528896e-01, 7.38223433e-01,\n", 275 | " 1.54815271e-01, 5.84579706e-02, -3.31626713e-01,\n", 276 | " -1.71033502e-01, -5.18384576e-01, -4.39960122e-01,\n", 277 | " 9.57486331e-02, -1.12325490e+00, -4.89403754e-01,\n", 278 | " -1.14547700e-01, -1.69845223e-01, -3.63178283e-01,\n", 279 | " 3.60171199e-01, -3.47823739e-01, -9.25692439e-01,\n", 280 | " -9.51392114e-01, -8.25387239e-01, -5.49879849e-01,\n", 281 | " 3.30521762e-02, -6.82583153e-01, -8.69729638e-01,\n", 282 | " -6.85020089e-01, 3.17883134e-01, -1.30235314e-01,\n", 283 | " -5.73356271e-01, -6.31108284e-01, 7.84212947e-01,\n", 284 | " -2.23308891e-01, 5.65437615e-01, -8.46981049e-01,\n", 285 | " -1.41261250e-01, 2.79158741e-01, -4.93367791e-01,\n", 286 | " -2.00498819e-01, -1.03930213e-01, 8.90461445e-01,\n", 287 | " -2.24991441e-01, -8.96919489e-01, -8.73064280e-01,\n", 288 | " -7.58038878e-01, -4.66680199e-01, 7.22678244e-01,\n", 289 | " -5.59149742e-01, -1.24261148e-01, -1.64420426e-01,\n", 290 | " -4.51472163e-01, -4.96546626e-02, 1.02280784e+00,\n", 291 | " -1.05020255e-01, 8.41002584e-01, 5.94654568e-02,\n", 292 | " 1.15070567e-01, -5.03001034e-01, -2.71962509e-02,\n", 293 | " 3.63025486e-01, -1.60034627e-01, 4.11806613e-01,\n", 294 | " -1.13845870e-01, -3.65492284e-01, 8.48379135e-02,\n", 295 | " -2.88552403e-01, 8.12265351e-02, -4.94506657e-01,\n", 296 | " -5.86017191e-01, -5.59142053e-01, -5.25708497e-01,\n", 297 | " 6.74317896e-01, -2.66468972e-01, 9.15713683e-02,\n", 298 | " 5.23049057e-01, -1.10633790e-01, 2.79339224e-01,\n", 299 | " -6.23515606e-01, -2.86670685e-01, -3.16960871e-01,\n", 300 | " 6.68717027e-02, -5.25446892e-01, -1.67678848e-01,\n", 301 | " 3.65337014e-01, -6.42370462e-01, 1.68807888e+00,\n", 302 | " 1.19530901e-01, 2.48779014e-01, -8.48422289e-01,\n", 303 | " -1.01598442e-01, 2.76702642e-02, 3.58119234e-03,\n", 304 | " -1.44004911e-01, 6.04847431e-01, 1.87054291e-01,\n", 305 | " 1.56635031e-01, 9.64419916e-02, -2.20247656e-01,\n", 306 | " -2.48066578e-02, 7.65711010e-01, 1.28327012e-01,\n", 307 | " -1.90867305e-01, -2.85259157e-01, -7.00574994e-01,\n", 308 | " -1.23649478e+00, 2.13845760e-01, -4.25968707e-01,\n", 309 | " -8.10470939e-01, 6.46034956e-01, -2.57002741e-01,\n", 310 | " 2.74780601e-01, -4.61119115e-02, -6.68052375e-01,\n", 311 | " 3.23258370e-01, 7.91189551e-01, -5.13218462e-01,\n", 312 | " 1.28738999e-01, -4.24925864e-01, 4.59689707e-01,\n", 313 | " -5.25384903e-01, -9.43557382e-01, -1.88754350e-01,\n", 314 | " -5.49634039e-01, -4.18326437e-01, -2.71465838e-01,\n", 315 | " -7.64491796e-01, 3.94803345e-01, -1.25615686e-01,\n", 316 | " -3.02387089e-01, -1.58212334e-01, 3.72114241e-01,\n", 317 | " 2.15725780e-01, -2.00074166e-01, -2.45844126e-01,\n", 318 | " 1.07620209e-01, -3.62814635e-01, -7.13834763e-01,\n", 319 | " -9.09253955e-04, -6.71784580e-01, 3.30038399e-01,\n", 320 | " -5.43184459e-01, -5.07325232e-01, -6.04800761e-01,\n", 321 | " 4.84835863e-01, 2.47996897e-01, 4.60729122e-01,\n", 322 | " -4.88370091e-01, -4.21317756e-01, -5.74742436e-01,\n", 323 | " -1.58287644e-01, 2.92152315e-02, -1.52552903e-01,\n", 324 | " -7.19140589e-01, -4.78222430e-01, -4.81176794e-01,\n", 325 | " 2.61144423e+00, 1.66414022e-01, 4.19833630e-01,\n", 326 | " 5.51657319e-01, -7.70756900e-01, -5.56083381e-01,\n", 327 | " 3.49743038e-01, -8.62182260e-01, 2.16021627e-01,\n", 328 | " -1.12311196e+00, -4.52988386e-01, 6.39272571e-01,\n", 329 | " -7.23185241e-01, -1.75254941e-01, -7.00653195e-01,\n", 330 | " 6.56644285e-01, -8.51138532e-02, -3.45448077e-01,\n", 331 | " -1.43988371e-01, 2.04409182e-01, -6.06003523e-01,\n", 332 | " -2.24461257e-01, -1.00270641e+00, -6.07612312e-01,\n", 333 | " -3.33568513e-01, -4.30919230e-03, -4.55460489e-01,\n", 334 | " -2.00799942e-01, -2.77695090e-01, 1.31959930e-01,\n", 335 | " 2.64888376e-01, 3.16366255e-01, -6.06663287e-01,\n", 336 | " -2.61160433e-01, -4.08308268e-01, 3.67855817e-01,\n", 337 | " -6.07873917e-01, -3.92181486e-01, -1.01239614e-01,\n", 338 | " -4.68084186e-01, -1.71389118e-01, 1.08263567e-01,\n", 339 | " -9.70231533e-01, -3.86805326e-01, -5.34243345e-01,\n", 340 | " -7.24829435e-01, 4.32495356e-01, 1.04864824e+00,\n", 341 | " 2.37388968e-01, -3.15175474e-01, 2.88717449e-04,\n", 342 | " 3.74545038e-01, -4.82611209e-01, -4.52872574e-01,\n", 343 | " -3.38566840e-01, 2.77733728e-02, -5.88080168e-01,\n", 344 | " 1.09459460e-02, 1.35544538e-01, -1.31831616e-01,\n", 345 | " -2.13972449e-01, 7.86977112e-02, 4.05162424e-01,\n", 346 | " -3.59765947e-01, -7.75328398e-01, -1.11721718e+00,\n", 347 | " 1.73226535e-01, 1.38722658e-01, -6.37745857e-02,\n", 348 | " 3.63118589e-01, -1.09312147e-01, -7.97997952e-01,\n", 349 | " -6.76416814e-01, -4.87033844e-01, 7.25182891e-03,\n", 350 | " -3.75792563e-01, -8.73541236e-01, -3.26497614e-01,\n", 351 | " -1.55220196e-01, -3.41232419e-01, -8.11710507e-02,\n", 352 | " 2.37196758e-02, -6.81809247e-01, 2.44322851e-01,\n", 353 | " -4.66613591e-01, -2.46285558e-01, -5.10486543e-01,\n", 354 | " -6.48733020e-01, -6.01873159e-01, -3.71754527e-01,\n", 355 | " -4.62958924e-02, -7.52262622e-02, -1.27172381e-01,\n", 356 | " -6.44580305e-01, 1.51268095e-02, -1.51754320e-02,\n", 357 | " 6.04792237e-01, -8.30064118e-01, 2.99534202e-01,\n", 358 | " 1.81418628e-01, -4.66077745e-01, 1.58522815e-01,\n", 359 | " 4.86539721e-01, -9.27816689e-01, 2.48611584e-01,\n", 360 | " -9.27894861e-02, 2.65173721e+00, -2.46923864e-02,\n", 361 | " -4.54429209e-01, 2.58741260e-01, -8.11437249e-01,\n", 362 | " 6.29201829e-02, -2.46115774e-02, -2.19172835e-02,\n", 363 | " -6.91680685e-02, -2.27471069e-01, 5.75737804e-02,\n", 364 | " 5.81942439e-01, 3.11116964e-01, -1.19277215e+00,\n", 365 | " -1.00228572e+00, -7.36471295e-01, 8.51671219e-01,\n", 366 | " 6.70923293e-02, 2.52199352e-01, 3.32008302e-01,\n", 367 | " -5.06502151e-01, -5.52525163e-01, -4.29315269e-02,\n", 368 | " 1.01769185e+00, -3.14344257e-01, -3.98333907e-01,\n", 369 | " -3.98791194e-01, 4.84052479e-01, 2.73545310e-02,\n", 370 | " -4.56411153e-01, -6.18580043e-01, -6.05887175e-01,\n", 371 | " -3.26678455e-01, 5.31159118e-02, -5.99733949e-01,\n", 372 | " 3.23494285e-01, -1.97477430e-01, -8.67545843e-01,\n", 373 | " -1.70324892e-01, -1.17545402e+00, -1.75225258e-01,\n", 374 | " -3.08754444e-02, 1.12252623e-01, -6.84447646e-01,\n", 375 | " 4.39037979e-02, -3.33114743e-01, -2.73166001e-01,\n", 376 | " -6.76139474e-01, -6.70041800e-01, -2.65656024e-01,\n", 377 | " 1.41204739e+00, 8.28536510e-01, -9.01165605e-02,\n", 378 | " -2.59314567e-01, 7.65747845e-01, 2.99408579e+00,\n", 379 | " 8.18895102e-01, 2.70463729e+00, 4.07895625e-01,\n", 380 | " 1.62610412e+00, 2.34459615e+00, 1.54290989e-01,\n", 381 | " 1.47807860e+00, 1.51920748e+00, 1.67193460e+00,\n", 382 | " 1.15129364e+00, 1.68717825e+00, 2.52889901e-01,\n", 383 | " 4.72390324e-01, 1.38908148e-01, 2.44700745e-01,\n", 384 | " -3.39965969e-01, 5.58326483e-01, 1.21900833e+00,\n", 385 | " 2.75626957e-01, -4.93132651e-01, 1.96827859e-01,\n", 386 | " -2.59729326e-02, -1.92156374e-01, -4.67355937e-01,\n", 387 | " 8.71757865e-02, 1.11748195e+00, 6.03729963e-01,\n", 388 | " -9.54830498e-02, 9.18098241e-02, 1.46897942e-01,\n", 389 | " 1.58823207e-01, 2.09848002e-01, 6.37123704e-01,\n", 390 | " -9.82987285e-02, 5.82816124e-01, 8.50259542e-01,\n", 391 | " 3.63346434e+00, 2.06452060e+00, 5.81701279e-01,\n", 392 | " 6.78918004e-01, 1.22173274e+00, 1.61408377e+00,\n", 393 | " 9.90366340e-01, 6.23038483e+00, 3.89325237e+00,\n", 394 | " 3.70068121e+00, 7.46906281e-01, 1.67064965e-02,\n", 395 | " -3.19646358e-01, -1.79403931e-01, 1.70247436e-01,\n", 396 | " 3.73449564e-01, 2.64562190e-01, 1.74119443e-01,\n", 397 | " -2.63605773e-01, -2.49537468e-01, 4.87229347e-01,\n", 398 | " 1.27334967e-01, -6.00214303e-02, -1.93838269e-01,\n", 399 | " -3.17398012e-01, -2.81619281e-01, -3.80523920e-01,\n", 400 | " 1.10514909e-01, -9.14620906e-02, -5.96147120e-01,\n", 401 | " -2.92471200e-02, -7.20769167e-04, -3.92490551e-02,\n", 402 | " 1.45392016e-01, -2.28525937e-01, 5.25221407e-01,\n", 403 | " 7.25887179e-01, -2.78037816e-01, 3.79592836e-01,\n", 404 | " -6.19666219e-01]], dtype=float32)]" 405 | ] 406 | }, 407 | "execution_count": 3, 408 | "metadata": {}, 409 | "output_type": "execute_result" 410 | } 411 | ], 412 | "source": [ 413 | "\n", 414 | "from PIL import Image\n", 415 | "from urllib.request import urlopen\n", 416 | "\n", 417 | "img = Image.open(urlopen(\n", 418 | " 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png'\n", 419 | "))\n", 420 | "\n", 421 | "img\n", 422 | "\n", 423 | "\n", 424 | "import numpy as np\n", 425 | "from PIL import Image\n", 426 | "def transforms_numpy(image: Image.Image):\n", 427 | " image = image.convert('RGB')\n", 428 | " image = image.resize((448, 448), Image.BICUBIC)\n", 429 | " img_numpy = np.array(image).astype(np.float32) / 255.0\n", 430 | " img_numpy = img_numpy.transpose(2, 0, 1)\n", 431 | "\n", 432 | " mean = np.array([0.485, 0.456, 0.406]).reshape(-1, 1, 1)\n", 433 | " std = np.array([0.229, 0.224, 0.225]).reshape(-1, 1, 1)\n", 434 | " img_numpy = (img_numpy - mean) / std\n", 435 | " img_numpy = np.expand_dims(img_numpy, axis=0)\n", 436 | " img_numpy = img_numpy.astype(np.float32)\n", 437 | "\n", 438 | " return img_numpy\n", 439 | "\n", 440 | "\n", 441 | "import onnxruntime as ort\n", 442 | "\n", 443 | "providers = ['TensorrtExecutionProvider']\n", 444 | "session = ort.InferenceSession('eva02_large_patch14_448.onnx', providers=providers)\n", 445 | "\n", 446 | "input_name = session.get_inputs()[0].name\n", 447 | "output_name = session.get_outputs()[0].name\n", 448 | "\n", 449 | "output = session.run([output_name], {input_name: transforms_numpy(img)})\n", 450 | "output" 451 | ] 452 | }, 453 | { 454 | "cell_type": "code", 455 | "execution_count": null, 456 | "metadata": {}, 457 | "outputs": [], 458 | "source": [ 459 | "import time\n", 460 | "\n", 461 | "num_images = 10\n", 462 | "start = time.perf_counter()\n", 463 | "for i in range(num_images):\n", 464 | " output = session.run([output_name], {input_name: transforms_numpy(img)})\n", 465 | "end = time.perf_counter()\n", 466 | "time_taken = end - start\n", 467 | "\n", 468 | "ms_per_image = time_taken / num_images * 1000\n", 469 | "fps = num_images / time_taken\n", 470 | "\n", 471 | "print(f\"Onnxruntime TensorRT: {ms_per_image:.3f} ms per image, FPS: {fps:.2f}\")" 472 | ] 473 | }, 474 | { 475 | "cell_type": "code", 476 | "execution_count": null, 477 | "metadata": {}, 478 | "outputs": [], 479 | "source": [] 480 | } 481 | ], 482 | "metadata": { 483 | "kernelspec": { 484 | "display_name": "pt-to-onnx-tensorrt", 485 | "language": "python", 486 | "name": "python3" 487 | }, 488 | "language_info": { 489 | "codemirror_mode": { 490 | "name": "ipython", 491 | "version": 3 492 | }, 493 | "file_extension": ".py", 494 | "mimetype": "text/x-python", 495 | "name": "python", 496 | "nbconvert_exporter": "python", 497 | "pygments_lexer": "ipython3", 498 | "version": "3.11.9" 499 | } 500 | }, 501 | "nbformat": 4, 502 | "nbformat_minor": 2 503 | } 504 | -------------------------------------------------------------------------------- /notebooks/04_onnx_preprocessing_input.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "env: LD_LIBRARY_PATH=/home/dnth/mambaforge-pypy3/envs/ram-onnx-inference/lib/python3.11/site-packages/tensorrt_libs:/home/dnth/mambaforge-pypy3/envs/ram-onnx-inference/lib:$LD_LIBRARY_PATH\n", 13 | "*************** EP Error ***************\n", 14 | "EP Error /onnxruntime_src/onnxruntime/python/onnxruntime_pybind_state.cc:490 void onnxruntime::python::RegisterTensorRTPluginsAsCustomOps(PySessionOptions&, const onnxruntime::ProviderOptions&) Please install TensorRT libraries as mentioned in the GPU requirements page, make sure they're in the PATH or LD_LIBRARY_PATH, and that your GPU is supported.\n", 15 | " when using [('TensorrtExecutionProvider', {'device_id': 0, 'trt_max_workspace_size': 8589934592, 'trt_fp16_enable': True, 'trt_engine_cache_enable': True, 'trt_engine_cache_path': './trt_cache', 'trt_force_sequential_engine_build': False, 'trt_max_partition_iterations': 10000, 'trt_min_subgraph_size': 1, 'trt_builder_optimization_level': 5, 'trt_timing_cache_enable': True})]\n", 16 | "Falling back to ['CPUExecutionProvider'] and retrying.\n", 17 | "****************************************\n" 18 | ] 19 | }, 20 | { 21 | "name": "stderr", 22 | "output_type": "stream", 23 | "text": [ 24 | "\u001b[1;31m2024-09-11 16:22:28.688929463 [E:onnxruntime:Default, provider_bridge_ort.cc:1978 TryGetProviderInfo_TensorRT] /onnxruntime_src/onnxruntime/core/session/provider_bridge_ort.cc:1637 onnxruntime::Provider& onnxruntime::ProviderLibrary::Get() [ONNXRuntimeError] : 1 : FAIL : Failed to load library libonnxruntime_providers_tensorrt.so with error: libcudnn.so.9: cannot open shared object file: No such file or directory\n", 25 | "\u001b[m\n" 26 | ] 27 | } 28 | ], 29 | "source": [ 30 | "import numpy as np\n", 31 | "import onnxruntime as ort\n", 32 | "from PIL import Image\n", 33 | "from urllib.request import urlopen\n", 34 | "\n", 35 | "\n", 36 | "\n", 37 | "def read_image(image: Image.Image):\n", 38 | " image = image.convert(\"RGB\")\n", 39 | " img_numpy = np.array(image).astype(np.float32)\n", 40 | " img_numpy = img_numpy.transpose(2, 0, 1)\n", 41 | " img_numpy = np.expand_dims(img_numpy, axis=0)\n", 42 | " return img_numpy\n", 43 | "\n", 44 | "\n", 45 | "providers = [\n", 46 | " (\n", 47 | " \"TensorrtExecutionProvider\",\n", 48 | " {\n", 49 | " \"device_id\": 0,\n", 50 | " \"trt_max_workspace_size\": 8589934592,\n", 51 | " \"trt_fp16_enable\": True,\n", 52 | " \"trt_engine_cache_enable\": True,\n", 53 | " \"trt_engine_cache_path\": \"./trt_cache\",\n", 54 | " \"trt_force_sequential_engine_build\": False,\n", 55 | " \"trt_max_partition_iterations\": 10000,\n", 56 | " \"trt_min_subgraph_size\": 1,\n", 57 | " \"trt_builder_optimization_level\": 5,\n", 58 | " \"trt_timing_cache_enable\": True,\n", 59 | " },\n", 60 | " ),\n", 61 | "]\n", 62 | "\n", 63 | "session = ort.InferenceSession(\"merged.onnx\", providers=providers)\n", 64 | "\n", 65 | "input_name = session.get_inputs()[0].name\n", 66 | "output_name = session.get_outputs()[0].name\n", 67 | "\n", 68 | "\n", 69 | "img = Image.open(\n", 70 | " urlopen(\n", 71 | " \"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png\"\n", 72 | " )\n", 73 | ")\n", 74 | "\n", 75 | "\n", 76 | "output = session.run([output_name], {input_name: read_image(img)})" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 4, 82 | "metadata": {}, 83 | "outputs": [ 84 | { 85 | "data": { 86 | "text/plain": [ 87 | "[array([[-9.80133116e-02, -7.09748715e-02, 4.18799594e-02,\n", 88 | " 3.05471003e-01, 8.56352389e-01, -6.08867884e-01,\n", 89 | " -1.14489764e-01, -5.66096365e-01, -3.39009434e-01,\n", 90 | " -4.52989995e-01, -4.44207609e-01, -1.95399925e-01,\n", 91 | " -2.15590537e-01, -5.99971041e-03, 8.06181803e-02,\n", 92 | " -5.37951738e-02, -1.50203824e-01, 1.59872174e-02,\n", 93 | " -1.08945347e-01, -5.41491270e-01, -3.88307571e-01,\n", 94 | " -1.11853182e-01, -1.48164034e-01, 2.78708965e-01,\n", 95 | " -4.75473583e-01, 1.37983754e-01, -1.65390164e-01,\n", 96 | " -5.86463869e-01, 1.19677067e-01, -3.23068619e-01,\n", 97 | " -2.87177920e-01, -3.55236679e-01, -1.68291748e-01,\n", 98 | " 3.55988741e-03, 1.81204945e-01, -9.62484926e-02,\n", 99 | " -3.28296900e-01, 2.45878473e-03, 5.91688305e-02,\n", 100 | " 5.88691309e-02, 3.47869545e-01, -1.48637593e-02,\n", 101 | " 4.14957553e-02, -2.67596275e-01, 1.97183609e-01,\n", 102 | " -2.29016870e-01, 5.51088750e-01, 1.20698214e-02,\n", 103 | " -8.33828747e-02, -2.96069771e-01, 1.04015255e+00,\n", 104 | " -6.84209764e-02, -2.00081974e-01, 8.56118798e-01,\n", 105 | " 4.45255548e-01, -5.94450608e-02, 1.81183279e-01,\n", 106 | " 4.67936128e-01, 8.71906638e-01, -3.84759039e-01,\n", 107 | " 7.99022853e-01, -3.48808020e-01, -7.02691376e-01,\n", 108 | " -4.46108162e-01, -3.54435951e-01, 1.23497650e-01,\n", 109 | " -1.66450307e-01, 5.18813133e-01, 3.68712366e-01,\n", 110 | " -4.00552720e-01, -7.61321425e-01, 4.97318298e-01,\n", 111 | " -4.29534107e-01, -3.75421822e-01, -7.21916974e-01,\n", 112 | " -5.66489324e-02, 4.66411889e-01, 1.86325684e-01,\n", 113 | " -1.37438893e-01, 2.40262672e-01, -1.48152336e-01,\n", 114 | " 3.20368946e-01, -2.19159767e-01, 7.30497241e-02,\n", 115 | " -6.11297905e-01, -6.08578324e-03, -2.02091157e-01,\n", 116 | " -1.04910158e-01, -2.20483035e-01, 8.05020332e-03,\n", 117 | " 1.54154301e-02, 1.42797291e-01, 3.07259709e-02,\n", 118 | " 7.76601136e-02, -1.88360006e-01, -1.15755036e-01,\n", 119 | " 1.24774396e-01, -3.84491146e-01, -3.28935862e-01,\n", 120 | " -5.31901002e-01, 3.15075099e-01, -3.02125722e-01,\n", 121 | " -5.42554259e-01, -2.41964206e-01, -3.03817153e-01,\n", 122 | " -2.12130874e-01, -3.41035783e-01, 1.10024549e-01,\n", 123 | " 9.07383338e-02, 1.56250939e-01, 3.08497638e-01,\n", 124 | " -2.28239477e-01, 3.17099512e-01, -1.96015060e-01,\n", 125 | " 2.43473873e-01, 5.95778346e-01, 3.30212176e-01,\n", 126 | " -1.33550584e-01, 5.97463548e-01, -1.17296740e-01,\n", 127 | " 6.36712134e-01, -6.46838427e-01, 2.42338210e-01,\n", 128 | " 7.43354201e-01, 2.05444455e+00, 4.99181211e-01,\n", 129 | " -5.47181606e-01, 2.10996374e-01, 3.05393308e-01,\n", 130 | " 2.65057981e-01, 1.55281067e-01, 8.80525410e-01,\n", 131 | " 2.78354138e-01, -3.17084789e-01, 2.26173282e-01,\n", 132 | " 9.72657204e-01, 8.93224329e-02, 1.35236025e-01,\n", 133 | " 1.92345709e-01, 5.86795568e-01, -2.97324538e-01,\n", 134 | " -4.91466880e-01, -6.91344589e-02, 1.58679321e-01,\n", 135 | " 6.06055677e-01, -3.01081657e-01, 4.28097099e-02,\n", 136 | " -1.94766819e-01, 1.88012481e-01, -2.46620566e-01,\n", 137 | " -5.64676881e-01, 5.88358939e-02, 6.58147335e-02,\n", 138 | " -2.26516426e-01, -5.42431056e-01, 1.55078501e-01,\n", 139 | " 3.37875009e-01, -1.00750282e-01, 4.35456157e-01,\n", 140 | " -1.22829549e-01, -6.77558362e-01, -9.81343836e-02,\n", 141 | " -1.87431812e-01, -1.98690310e-01, 2.59237915e-01,\n", 142 | " 3.71413231e-01, 1.59832627e-01, -2.99515218e-01,\n", 143 | " 1.36951670e-01, -2.26005316e-01, -6.86368644e-01,\n", 144 | " 3.99957538e-01, 3.91879261e-01, 4.52733815e-01,\n", 145 | " -4.91438508e-01, -1.42258834e-02, -3.89406741e-01,\n", 146 | " -4.51007724e-01, -1.72013193e-01, 1.19485721e-01,\n", 147 | " 1.22034848e-01, -3.72095048e-01, -3.67178470e-01,\n", 148 | " 1.39421776e-01, -3.21538746e-03, -6.29333436e-01,\n", 149 | " 2.56536976e-02, -2.97285557e-01, 4.21239048e-01,\n", 150 | " -4.44046892e-02, -2.69611180e-01, -1.24292925e-01,\n", 151 | " 1.73381537e-01, 3.25252056e-01, -2.91428834e-01,\n", 152 | " 2.69649029e-01, -5.98957300e-01, -3.47460777e-01,\n", 153 | " -2.99486220e-01, -4.04244721e-01, -2.47881085e-01,\n", 154 | " -2.40548402e-01, -1.53891921e-01, -6.04456186e-01,\n", 155 | " -7.00717747e-01, 5.14795780e-02, -1.98974311e-02,\n", 156 | " -5.97443223e-01, -6.81020975e-01, -3.41116250e-01,\n", 157 | " -7.03857690e-02, 1.71640515e-02, -2.92534590e-01,\n", 158 | " -6.54577136e-01, 1.64077863e-01, -3.80347639e-01,\n", 159 | " -2.63671726e-01, 1.65206820e-01, -1.84755892e-01,\n", 160 | " 4.88764793e-02, -2.17447624e-01, 4.36283231e-01,\n", 161 | " -3.88691664e-01, -3.20240617e-01, -3.94455433e-01,\n", 162 | " -6.78140879e-01, 1.35201067e-02, -4.58796650e-01,\n", 163 | " 1.95833877e-01, 1.12028502e-01, -8.25938940e-01,\n", 164 | " -1.26261562e-01, -4.11404073e-01, 8.61544162e-02,\n", 165 | " -1.71770021e-01, -9.59175706e-01, -1.99890345e-01,\n", 166 | " 1.78084865e-01, -9.18368101e-01, -5.66744864e-01,\n", 167 | " -6.06670618e-01, -4.73434865e-01, 2.00826854e-01,\n", 168 | " -1.89059377e-01, -4.39447016e-01, 2.50532329e-01,\n", 169 | " -2.36911878e-01, -2.25648224e-01, -2.98406363e-01,\n", 170 | " -9.23996791e-02, 1.54859841e-01, 3.33530933e-01,\n", 171 | " -2.41004109e-01, 5.60752928e-01, -1.52170524e-01,\n", 172 | " -2.23627791e-01, -2.03725100e-01, -2.51832008e-01,\n", 173 | " 6.43213987e-02, -3.11535597e-01, -5.86382389e-01,\n", 174 | " -1.88233644e-01, -3.97355631e-02, 5.12785017e-02,\n", 175 | " 4.64925840e-02, -1.06442273e-01, 1.10255413e-01,\n", 176 | " 2.64603078e-01, -1.24271862e-01, 3.32253218e-01,\n", 177 | " -5.38990319e-01, -5.11359423e-03, -3.32723036e-02,\n", 178 | " -2.68656552e-01, 2.24650234e-01, -2.90773571e-01,\n", 179 | " -9.22786444e-02, -6.09552979e-01, 2.15618104e-01,\n", 180 | " -1.98268965e-01, 6.84383363e-02, -1.82792947e-01,\n", 181 | " 7.95861185e-02, -1.68078154e-01, -2.19693393e-01,\n", 182 | " 7.09326625e-01, -1.10326089e-01, 7.46640980e-01,\n", 183 | " 5.38040042e-01, 3.69508624e-01, 5.81453741e-03,\n", 184 | " 2.42027968e-01, -1.20744959e-01, 1.09727561e-01,\n", 185 | " -1.32820427e-01, -1.17027760e-02, -7.54619598e-01,\n", 186 | " 1.74482524e-01, 1.29754648e-01, -2.27584571e-01,\n", 187 | " 5.73821187e-01, -3.41045380e-01, 4.88567173e-01,\n", 188 | " -2.43725181e-01, -2.36637667e-02, 4.80251461e-02,\n", 189 | " 1.86373204e-01, -2.94162035e-01, -2.88192987e-01,\n", 190 | " -2.28432715e-02, 5.06057777e-02, -1.32124662e-01,\n", 191 | " -2.87904501e-01, -2.35087246e-01, 1.48285516e-02,\n", 192 | " 1.56510279e-01, 1.81933001e-01, -8.08029413e-01,\n", 193 | " -3.81350547e-01, -7.81343281e-02, 5.09899259e-02,\n", 194 | " 3.55600506e-01, 2.60279402e-02, 2.71263182e-01,\n", 195 | " -8.00917029e-01, -4.61929977e-01, -5.83343148e-01,\n", 196 | " -5.89687526e-02, 9.01426375e-02, -4.18183565e-01,\n", 197 | " 4.08974111e-01, 4.73053038e-01, 4.24556255e-01,\n", 198 | " -8.08655173e-02, -2.69811481e-01, 6.77768707e-01,\n", 199 | " 4.73878324e-01, 2.20427960e-02, -2.22427547e-01,\n", 200 | " 2.67423689e-01, 4.90085840e-01, -6.35159373e-01,\n", 201 | " -7.48221695e-01, -4.22942340e-01, -5.84060073e-01,\n", 202 | " 2.26161584e-01, 8.75125080e-03, -7.41758198e-02,\n", 203 | " -4.24942523e-01, 1.34835050e-01, 1.42711639e-01,\n", 204 | " -3.28406990e-01, -7.05371648e-02, 4.33060944e-01,\n", 205 | " -9.98499990e-01, -4.89400357e-01, -2.59357959e-01,\n", 206 | " 2.00876668e-01, -4.11838591e-01, -5.89948475e-01,\n", 207 | " -4.98830855e-01, -1.85772568e-01, -2.13018775e-01,\n", 208 | " -1.74890041e-01, -8.04917932e-01, -4.73416835e-01,\n", 209 | " 1.10100091e-01, 1.55241519e-01, 2.54663646e-01,\n", 210 | " 4.32200104e-01, -2.17026919e-02, 3.35379511e-01,\n", 211 | " 2.71683961e-01, 1.96729213e-01, 7.38366842e-02,\n", 212 | " 6.85632825e-01, 8.33486542e-02, -5.16906202e-01,\n", 213 | " -7.63974190e-01, 3.02754521e-01, -2.31539115e-01,\n", 214 | " -1.30108058e-01, -9.96959507e-01, 3.79687667e-01,\n", 215 | " 7.90036201e-01, -1.90145060e-01, -2.34687090e-01,\n", 216 | " 4.11347538e-01, 1.73054665e-01, -2.65298843e-01,\n", 217 | " -1.91373229e-02, 3.57837319e-01, -2.27147132e-01,\n", 218 | " -2.67468095e-02, -5.98467648e-01, 1.08793646e-01,\n", 219 | " 2.60733694e-01, -6.86719716e-01, -5.80562592e-01,\n", 220 | " -6.85681760e-01, -4.05867547e-01, 4.68300462e-01,\n", 221 | " -2.47511759e-01, -1.37667611e-01, -2.24578798e-01,\n", 222 | " -3.85352284e-01, 5.42961955e-01, 4.03906465e-01,\n", 223 | " -5.36896586e-01, 1.24282405e-01, 2.88355112e-01,\n", 224 | " -1.67112648e-02, 7.59152547e-02, 1.72205821e-01,\n", 225 | " -2.87840277e-01, 2.85556197e+00, -3.32426310e-01,\n", 226 | " -5.98317623e-01, -2.77655005e-01, 1.27541780e-01,\n", 227 | " -5.09752184e-02, -8.13367248e-01, 3.60042304e-02,\n", 228 | " -1.66388229e-02, -1.33669958e-01, -2.34927088e-02,\n", 229 | " -4.22387928e-01, -2.88436830e-01, -1.62873656e-01,\n", 230 | " -4.53914285e-01, -2.97364235e-01, 4.46950078e-01,\n", 231 | " -2.20146552e-01, -2.45044231e-01, -7.97688186e-01,\n", 232 | " 9.91842896e-03, 1.84222251e-01, 6.51105046e-02,\n", 233 | " -6.98298097e-01, -1.76661134e-01, 1.59333795e-02,\n", 234 | " 4.24387187e-01, 4.94606197e-01, 1.75372154e-01,\n", 235 | " -2.91364521e-01, -1.83547407e-01, -5.78579664e-01,\n", 236 | " -7.35182106e-01, -4.93578583e-01, -3.97166610e-02,\n", 237 | " -4.14905787e-01, -2.88320363e-01, -1.95385009e-01,\n", 238 | " -5.13048232e-01, -4.31745127e-02, 4.33638304e-01,\n", 239 | " -1.10733092e-01, -4.69144195e-01, -4.35997128e-01,\n", 240 | " -4.96057868e-01, 7.16342270e-01, 4.63082105e-01,\n", 241 | " -4.88675117e-01, -8.33671540e-02, -3.37471515e-02,\n", 242 | " -3.17065716e-01, -1.16151810e-01, 2.32326120e-01,\n", 243 | " -3.84108126e-01, -5.16961515e-02, 1.15829974e-01,\n", 244 | " 1.03705660e-01, -2.19947070e-01, -9.66161042e-02,\n", 245 | " 4.93042655e-02, -7.04089701e-01, 1.76784247e-01,\n", 246 | " 5.79543225e-02, 1.31773502e-02, 8.04396987e-01,\n", 247 | " -7.94081211e-01, -1.75622568e-01, -3.89764905e-01,\n", 248 | " -1.98679119e-02, -2.83796251e-01, -1.61517173e-01,\n", 249 | " -7.51328468e-01, -3.07548463e-01, -2.39405721e-01,\n", 250 | " 1.92713082e-01, 1.14773296e-01, -2.83705980e-01,\n", 251 | " -1.36547387e-02, -4.74114001e-01, -3.36663067e-01,\n", 252 | " -3.65539938e-01, -9.68946397e-01, 6.41701996e-01,\n", 253 | " 8.49802256e-01, 2.62948632e-01, -7.03223944e-01,\n", 254 | " -5.04845083e-01, -3.30444574e-01, -1.39470518e-01,\n", 255 | " 2.72776866e+00, 1.71799207e+00, 1.19758695e-01,\n", 256 | " -5.21318793e-01, 4.16872472e-01, 6.77349806e-01,\n", 257 | " -1.28430113e-01, -1.94216371e-02, 2.00297579e-01,\n", 258 | " 3.02576512e-01, 1.26687676e-01, -4.07379508e-01,\n", 259 | " -5.67077696e-01, -4.59493607e-01, -7.09315896e-01,\n", 260 | " -9.16661173e-02, -9.19757485e-02, -4.62344646e-01,\n", 261 | " -1.97581321e-01, 6.99849129e-01, 1.40905797e-01,\n", 262 | " 1.76061422e-01, 1.60922445e-02, -4.53191251e-01,\n", 263 | " -4.27005321e-01, -4.63816851e-01, -5.80704689e-01,\n", 264 | " -8.64838004e-01, 1.54551432e-01, 1.71068504e-01,\n", 265 | " -8.82755995e-01, 9.07033086e-02, -2.80875295e-01,\n", 266 | " -2.72643387e-01, 5.27990162e-01, 2.94412076e-01,\n", 267 | " -2.64773041e-01, -1.59766078e-01, -8.49261999e-01,\n", 268 | " 2.83270717e-01, 1.88072309e-01, -4.72711295e-01,\n", 269 | " -7.40569949e-01, -2.21687064e-01, 9.32242721e-04,\n", 270 | " 5.96540451e-01, 1.81750774e+00, 9.85135794e-01,\n", 271 | " 2.42713153e-01, 2.81198204e-01, -3.88581514e-01,\n", 272 | " 2.40591317e-02, -9.55956221e-01, -3.02939534e-01,\n", 273 | " -2.16846615e-02, 5.91587901e-01, -5.41283548e-01,\n", 274 | " 6.14569604e-01, 3.95822287e-01, 4.58407365e-02,\n", 275 | " -3.43552232e-01, 3.27444166e-01, 8.32027793e-02,\n", 276 | " -9.34319496e-02, -6.15698576e-01, 2.32380740e-02,\n", 277 | " -3.64580870e-01, 2.58547723e-01, -3.95019650e-02,\n", 278 | " -1.95546597e-02, 1.11885101e-01, 1.66525126e-01,\n", 279 | " -3.04721832e-01, -3.90469909e-01, 5.31792641e-02,\n", 280 | " -7.33478665e-01, 1.83833629e-01, 5.42515159e-01,\n", 281 | " 9.33911681e-01, -9.35386062e-01, 7.46131763e-02,\n", 282 | " -6.51914716e-01, 2.53521174e-01, -8.59946012e-05,\n", 283 | " -7.97198534e-01, -5.13710558e-01, -4.31592643e-01,\n", 284 | " 3.94278944e-01, -5.77673614e-02, -5.45558333e-01,\n", 285 | " -4.44256663e-01, -2.29276955e-01, -9.84215811e-02,\n", 286 | " -3.09409857e-01, -4.28502798e-01, 9.04113054e-02,\n", 287 | " -3.42661738e-02, 2.57972121e-01, 9.53063369e-02,\n", 288 | " 2.73988396e-01, -1.45027563e-01, -8.74807656e-01,\n", 289 | " -9.06912625e-01, -4.82507348e-02, -2.27206796e-01,\n", 290 | " 3.62989783e-01, 3.32475245e-01, 7.75308847e-01,\n", 291 | " 1.31565854e-01, 3.06899369e-01, -1.98353082e-01,\n", 292 | " -3.85600924e-02, -4.32143152e-01, -4.91063327e-01,\n", 293 | " -1.00765318e-01, -9.10089254e-01, -1.33745849e-01,\n", 294 | " -3.81454229e-02, 3.60834152e-02, -4.96103317e-02,\n", 295 | " 3.48234445e-01, -3.18226546e-01, -4.97313142e-01,\n", 296 | " -9.78429675e-01, -7.76834309e-01, -3.09844375e-01,\n", 297 | " 2.55904496e-01, -5.17553270e-01, -7.66540587e-01,\n", 298 | " -3.87616456e-01, 2.31802627e-01, 1.21605493e-01,\n", 299 | " -3.98821354e-01, -6.66933894e-01, 8.04156125e-01,\n", 300 | " -2.83970058e-01, 4.92733538e-01, -8.60800385e-01,\n", 301 | " -2.55653083e-01, 5.08211553e-01, -4.30940986e-01,\n", 302 | " -5.36097623e-02, -4.11353856e-02, 8.67326021e-01,\n", 303 | " -2.34290466e-01, -8.32477808e-01, -7.54129469e-01,\n", 304 | " -6.11049652e-01, -3.34981740e-01, 5.46838284e-01,\n", 305 | " -3.86452198e-01, -1.78096920e-01, -8.36616457e-02,\n", 306 | " -4.25402910e-01, -2.78243721e-01, 8.86776567e-01,\n", 307 | " 4.53697443e-02, 6.81079388e-01, 2.06191629e-01,\n", 308 | " 1.23747960e-01, -4.41266716e-01, -1.59061342e-01,\n", 309 | " 2.31142417e-01, -2.45428979e-02, 5.25068104e-01,\n", 310 | " -8.36544856e-02, -2.34345391e-01, 6.04769923e-02,\n", 311 | " -2.13150829e-01, 2.71877348e-01, -4.26770359e-01,\n", 312 | " -5.57892501e-01, -4.95914996e-01, -4.41893697e-01,\n", 313 | " 5.93007445e-01, 1.08125634e-01, 1.70931816e-02,\n", 314 | " 5.47236502e-01, 1.08268693e-01, 4.87307936e-01,\n", 315 | " -3.94411534e-01, -2.35678792e-01, -8.90167505e-02,\n", 316 | " 2.14322150e-01, -4.58852053e-01, -2.71948934e-01,\n", 317 | " 2.33547017e-01, -6.17711246e-01, 1.53134227e+00,\n", 318 | " -5.54062426e-02, 3.78960729e-01, -6.77389979e-01,\n", 319 | " 1.57011807e-01, 5.75193763e-02, 1.44809335e-01,\n", 320 | " -1.05176367e-01, 6.38028145e-01, 1.51492625e-01,\n", 321 | " 3.43820751e-02, -1.11124068e-02, -2.79504210e-02,\n", 322 | " 1.18824095e-02, 6.88060582e-01, 3.66235822e-01,\n", 323 | " -2.13462159e-01, 1.02727450e-02, -1.48301691e-01,\n", 324 | " -8.99726272e-01, 8.33447576e-02, -5.12650609e-01,\n", 325 | " -7.75059640e-01, 4.79501903e-01, 1.30111486e-01,\n", 326 | " 3.42155874e-01, -8.34183246e-02, -4.38668758e-01,\n", 327 | " 3.71970028e-01, 1.03491879e+00, -4.44220543e-01,\n", 328 | " 2.48788849e-01, -2.65998602e-01, 3.87766451e-01,\n", 329 | " -5.58730662e-01, -7.52335548e-01, -2.54664540e-01,\n", 330 | " -5.37918031e-01, -6.77528858e-01, -1.80834234e-01,\n", 331 | " -6.30301118e-01, 2.87091583e-01, -1.66119277e-01,\n", 332 | " -1.90627262e-01, -3.01997215e-01, 2.95026988e-01,\n", 333 | " 3.01403552e-01, -3.87924373e-01, -1.28300339e-01,\n", 334 | " 1.35329664e-01, -2.24850908e-01, -6.03868783e-01,\n", 335 | " -4.99482602e-02, -5.94265223e-01, 3.87509257e-01,\n", 336 | " -4.08898175e-01, -1.71564668e-01, -4.99517679e-01,\n", 337 | " 6.72587633e-01, 2.74639517e-01, 5.89970171e-01,\n", 338 | " -4.36519235e-01, -2.11684704e-01, -4.89077419e-01,\n", 339 | " -2.49557778e-01, 6.73936307e-02, -3.44033539e-03,\n", 340 | " -4.86616850e-01, -5.18684804e-01, -2.21948057e-01,\n", 341 | " 2.24672794e+00, 4.62222695e-02, 2.72710145e-01,\n", 342 | " 3.06668043e-01, -9.75608826e-01, -3.32899690e-01,\n", 343 | " 2.42987707e-01, -6.82486057e-01, 4.78229225e-01,\n", 344 | " -1.00148582e+00, -3.94655555e-01, 9.22384977e-01,\n", 345 | " -4.69960123e-01, -6.27866685e-02, -7.28494644e-01,\n", 346 | " 6.76112115e-01, -5.49282506e-02, -3.88828695e-01,\n", 347 | " -2.46683002e-01, 2.63122380e-01, -4.77379203e-01,\n", 348 | " -1.05496317e-01, -6.90751135e-01, -4.20489639e-01,\n", 349 | " -3.96724373e-01, 1.78387478e-01, -4.04329926e-01,\n", 350 | " -3.09976518e-01, -3.45511347e-01, 1.89258054e-01,\n", 351 | " 4.05621171e-01, 1.01618290e-01, -2.93858677e-01,\n", 352 | " -2.20789567e-01, -5.35571694e-01, 6.06625736e-01,\n", 353 | " -5.40695906e-01, -4.79047686e-01, 6.54642358e-02,\n", 354 | " -4.31744635e-01, -2.94048488e-01, 2.74880193e-02,\n", 355 | " -7.96093702e-01, -2.68981099e-01, -3.64304483e-01,\n", 356 | " -5.53001463e-01, 4.04899418e-02, 1.16110241e+00,\n", 357 | " 8.80086422e-02, -1.64549395e-01, 2.25857526e-01,\n", 358 | " 3.87905359e-01, -3.62848938e-01, -5.10387063e-01,\n", 359 | " -1.78511471e-01, 9.76254791e-03, -3.96734744e-01,\n", 360 | " -2.82329768e-02, 9.11311358e-02, -4.17846024e-01,\n", 361 | " -2.94083923e-01, 3.86902332e-01, 3.85026038e-01,\n", 362 | " -4.13265795e-01, -7.33691275e-01, -7.61151671e-01,\n", 363 | " 2.03410804e-01, 1.59843877e-01, 3.11421007e-02,\n", 364 | " 4.08721924e-01, -1.63058192e-03, -6.39764726e-01,\n", 365 | " -7.14428544e-01, -4.85781670e-01, -1.52986586e-01,\n", 366 | " -3.34077448e-01, -7.03293204e-01, -2.69029468e-01,\n", 367 | " -7.38326013e-02, -2.48745590e-01, -7.04066455e-03,\n", 368 | " -9.00772586e-02, -4.16443706e-01, 4.02538300e-01,\n", 369 | " -2.81377614e-01, -3.29353929e-01, -4.10155267e-01,\n", 370 | " -2.77683914e-01, -6.33888483e-01, -2.23821193e-01,\n", 371 | " -1.69226229e-02, -5.88809289e-02, -1.73631430e-01,\n", 372 | " -6.70432508e-01, 1.71632916e-02, -1.75893784e-01,\n", 373 | " 5.11891305e-01, -5.19546747e-01, 4.26807106e-02,\n", 374 | " 2.08472520e-01, -1.85726374e-01, 1.94505915e-01,\n", 375 | " 5.39497972e-01, -9.26923037e-01, 1.40068054e-01,\n", 376 | " -2.08878487e-01, 2.47632170e+00, -1.65070727e-01,\n", 377 | " -4.12032068e-01, 3.31078708e-01, -6.52710676e-01,\n", 378 | " 6.73033297e-02, -1.46627575e-01, 8.57072175e-02,\n", 379 | " -2.41638526e-01, -2.70910442e-01, 2.08886325e-01,\n", 380 | " 4.88946438e-01, 2.70951122e-01, -1.20266557e+00,\n", 381 | " -7.88686514e-01, -6.09229863e-01, 9.48015332e-01,\n", 382 | " 2.02414781e-01, 9.99399424e-02, 4.25131977e-01,\n", 383 | " -7.03932881e-01, -5.47604442e-01, 2.04160199e-01,\n", 384 | " 8.68144155e-01, -2.39724964e-01, -8.27495009e-04,\n", 385 | " -3.98846447e-01, 4.15533930e-01, -1.41388029e-02,\n", 386 | " -1.96410000e-01, -4.03657317e-01, -6.47875249e-01,\n", 387 | " -5.21326721e-01, -3.78623307e-01, -2.64058709e-01,\n", 388 | " 2.86304951e-01, -3.24610859e-01, -7.81181872e-01,\n", 389 | " -2.74629712e-01, -1.14938617e+00, -9.11252722e-02,\n", 390 | " -1.85611844e-02, 3.23632687e-01, -6.65680885e-01,\n", 391 | " -2.60714948e-01, -2.52072692e-01, -2.21639454e-01,\n", 392 | " -6.07900262e-01, -8.83966327e-01, -2.44845688e-01,\n", 393 | " 1.43495488e+00, 6.59724593e-01, -3.63365710e-01,\n", 394 | " -3.43842179e-01, 7.20794320e-01, 2.13428712e+00,\n", 395 | " 7.27582097e-01, 1.82785463e+00, -2.07051918e-01,\n", 396 | " 9.76226330e-01, 2.39751196e+00, 6.86568469e-02,\n", 397 | " 7.05418587e-01, 1.30835724e+00, 1.31112194e+00,\n", 398 | " 1.00403726e+00, 1.40288806e+00, 3.35863173e-01,\n", 399 | " 3.07080567e-01, 2.03253746e-01, 1.36577487e-01,\n", 400 | " -3.05013657e-01, 1.99181736e-02, 9.39092577e-01,\n", 401 | " 8.45612064e-02, -5.47130764e-01, 6.66220486e-02,\n", 402 | " -1.86092257e-02, -1.61099851e-01, -3.64384919e-01,\n", 403 | " 1.15466744e-01, 7.75636554e-01, 6.92520142e-01,\n", 404 | " -1.87483281e-01, -1.84648573e-01, 3.49686712e-01,\n", 405 | " 9.82572734e-02, 7.74769932e-02, 2.24918067e-01,\n", 406 | " -7.85844624e-02, 3.87075543e-01, 5.90782404e-01,\n", 407 | " 3.14594698e+00, 1.99727964e+00, 9.16315243e-02,\n", 408 | " 6.98518217e-01, 5.60364783e-01, 1.29787898e+00,\n", 409 | " 7.00638771e-01, 6.39878416e+00, 3.60839224e+00,\n", 410 | " 2.61788368e+00, 5.89001179e-01, -5.55055588e-03,\n", 411 | " -3.24705899e-01, -4.71008345e-02, 2.34110266e-01,\n", 412 | " 4.09047008e-01, 1.34829193e-01, 2.42660969e-01,\n", 413 | " -1.32250965e-01, -3.95434260e-01, 4.41398948e-01,\n", 414 | " 3.13426137e-01, -1.69582233e-01, -1.91859305e-02,\n", 415 | " -4.40604836e-01, -1.85483262e-01, -1.88311353e-01,\n", 416 | " -2.75436938e-02, -4.74101156e-02, -6.08623266e-01,\n", 417 | " -1.73647240e-01, 6.42412752e-02, -1.01475112e-01,\n", 418 | " 6.04286939e-02, -3.99039268e-01, 4.36801940e-01,\n", 419 | " 2.34948710e-01, -2.29043141e-01, 2.09915683e-01,\n", 420 | " -3.62638235e-01]], dtype=float32)]" 421 | ] 422 | }, 423 | "execution_count": 4, 424 | "metadata": {}, 425 | "output_type": "execute_result" 426 | } 427 | ], 428 | "source": [ 429 | "output\n" 430 | ] 431 | }, 432 | { 433 | "cell_type": "code", 434 | "execution_count": 5, 435 | "metadata": {}, 436 | "outputs": [ 437 | { 438 | "data": { 439 | "text/plain": [ 440 | "(1, 1000)" 441 | ] 442 | }, 443 | "execution_count": 5, 444 | "metadata": {}, 445 | "output_type": "execute_result" 446 | } 447 | ], 448 | "source": [ 449 | "output[0].shape" 450 | ] 451 | }, 452 | { 453 | "cell_type": "code", 454 | "execution_count": 6, 455 | "metadata": {}, 456 | "outputs": [ 457 | { 458 | "name": "stdout", 459 | "output_type": "stream", 460 | "text": [ 461 | "torch.Size([1, 1000])\n" 462 | ] 463 | } 464 | ], 465 | "source": [ 466 | "import torch\n", 467 | "output = torch.from_numpy(output[0])\n", 468 | "print(output.shape)" 469 | ] 470 | }, 471 | { 472 | "cell_type": "code", 473 | "execution_count": 9, 474 | "metadata": {}, 475 | "outputs": [ 476 | { 477 | "name": "stdout", 478 | "output_type": "stream", 479 | "text": [ 480 | "espresso: 33.73%\n", 481 | "cup: 2.07%\n", 482 | "chocolate sauce, chocolate syrup: 1.30%\n", 483 | "bakery, bakeshop, bakehouse: 0.98%\n", 484 | "coffee mug: 0.86%\n", 485 | "eggnog: 0.77%\n", 486 | "tray: 0.67%\n", 487 | "ice cream, icecream: 0.62%\n", 488 | "restaurant, eating house, eating place, eatery: 0.53%\n", 489 | "plate: 0.47%\n" 490 | ] 491 | } 492 | ], 493 | "source": [ 494 | "from imagenet_classes import IMAGENET2012_CLASSES\n", 495 | "\n", 496 | "top5_probabilities, top5_class_indices = torch.topk(output.softmax(dim=1) * 100, k=10)\n", 497 | "\n", 498 | "im_classes = list(IMAGENET2012_CLASSES.values())\n", 499 | "class_names = [im_classes[i] for i in top5_class_indices[0]]\n", 500 | "\n", 501 | "# Print class names and probabilities\n", 502 | "for name, prob in zip(class_names, top5_probabilities[0]):\n", 503 | " print(f\"{name}: {prob:.2f}%\")" 504 | ] 505 | }, 506 | { 507 | "cell_type": "code", 508 | "execution_count": null, 509 | "metadata": {}, 510 | "outputs": [], 511 | "source": [] 512 | } 513 | ], 514 | "metadata": { 515 | "kernelspec": { 516 | "display_name": "ram-onnx-inference", 517 | "language": "python", 518 | "name": "python3" 519 | }, 520 | "language_info": { 521 | "codemirror_mode": { 522 | "name": "ipython", 523 | "version": 3 524 | }, 525 | "file_extension": ".py", 526 | "mimetype": "text/x-python", 527 | "name": "python", 528 | "nbconvert_exporter": "python", 529 | "pygments_lexer": "ipython3", 530 | "version": "3.11.9" 531 | } 532 | }, 533 | "nbformat": 4, 534 | "nbformat_minor": 2 535 | } 536 | -------------------------------------------------------------------------------- /output.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dnth/supercharge-your-pytorch-image-models-blogpost/55253d09dd72c530c4bd2ff550e277a96914930c/output.mp4 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiofiles==23.2.1 2 | annotated-types==0.7.0 3 | anyio==4.6.0 4 | certifi==2024.8.30 5 | charset-normalizer==3.3.2 6 | click==8.1.7 7 | coloredlogs==15.0.1 8 | contourpy==1.3.0 9 | cupy-cuda12x==13.3.0 10 | cycler==0.12.1 11 | fastapi==0.115.0 12 | fastrlock==0.8.2 13 | ffmpy==0.4.0 14 | filelock==3.16.1 15 | flatbuffers==24.3.25 16 | fonttools==4.54.1 17 | fsspec==2024.9.0 18 | gradio==4.44.1 19 | gradio_client==1.3.0 20 | h11==0.14.0 21 | httpcore==1.0.6 22 | httpx==0.27.2 23 | huggingface-hub==0.25.1 24 | humanfriendly==10.0 25 | idna==3.10 26 | importlib_resources==6.4.5 27 | Jinja2==3.1.4 28 | kiwisolver==1.4.7 29 | markdown-it-py==3.0.0 30 | MarkupSafe==2.1.5 31 | matplotlib==3.9.2 32 | mdurl==0.1.2 33 | mpmath==1.3.0 34 | networkx==3.3 35 | numpy==2.1.1 36 | nvidia-cublas-cu12==12.1.3.1 37 | nvidia-cuda-cupti-cu12==12.1.105 38 | nvidia-cuda-nvrtc-cu12==12.1.105 39 | nvidia-cuda-runtime-cu12==12.1.105 40 | nvidia-cudnn-cu12==9.1.0.70 41 | nvidia-cufft-cu12==11.0.2.54 42 | nvidia-curand-cu12==10.3.2.106 43 | nvidia-cusolver-cu12==11.4.5.107 44 | nvidia-cusparse-cu12==12.1.0.106 45 | nvidia-nccl-cu12==2.20.5 46 | nvidia-nvjitlink-cu12==12.6.68 47 | nvidia-nvtx-cu12==12.1.105 48 | onnx==1.16.2 49 | onnxruntime-gpu==1.19.2 50 | onnxsim==0.4.36 51 | opencv-python==4.10.0.84 52 | orjson==3.10.7 53 | packaging==24.1 54 | pandas==2.2.3 55 | pillow==10.4.0 56 | protobuf==5.28.2 57 | pydantic==2.9.2 58 | pydantic_core==2.23.4 59 | pydub==0.25.1 60 | Pygments==2.18.0 61 | pyparsing==3.1.4 62 | python-dateutil==2.9.0.post0 63 | python-multipart==0.0.12 64 | pytz==2024.2 65 | PyYAML==6.0.2 66 | requests==2.32.3 67 | rich==13.8.1 68 | ruff==0.6.9 69 | safetensors==0.4.5 70 | semantic-version==2.10.0 71 | shellingham==1.5.4 72 | six==1.16.0 73 | sniffio==1.3.1 74 | starlette==0.38.6 75 | sympy==1.13.3 76 | tensorrt==10.1.0 77 | tensorrt-cu12==10.1.0 78 | tensorrt-cu12-bindings==10.1.0 79 | tensorrt-cu12-libs==10.1.0 80 | timm==1.0.9 81 | tomlkit==0.12.0 82 | torch==2.4.1 83 | torchvision==0.19.1 84 | tqdm==4.66.5 85 | triton==3.0.0 86 | typer==0.12.5 87 | typing_extensions==4.12.2 88 | tzdata==2024.2 89 | urllib3==2.2.3 90 | uvicorn==0.31.0 91 | websockets==12.0 92 | -------------------------------------------------------------------------------- /run_trt.py: -------------------------------------------------------------------------------- 1 | import time 2 | from urllib.request import urlopen 3 | 4 | import cupy as cp 5 | import numpy as np 6 | import onnxruntime as ort 7 | from PIL import Image 8 | 9 | img = Image.open( 10 | urlopen( 11 | "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png" 12 | ) 13 | ) 14 | 15 | 16 | def transforms_numpy(image: Image.Image): 17 | image = image.convert("RGB") 18 | image = image.resize((448, 448), Image.BICUBIC) 19 | img_numpy = np.array(image).astype(np.float32) / 255.0 20 | img_numpy = img_numpy.transpose(2, 0, 1) 21 | 22 | mean = np.array([0.485, 0.456, 0.406]).reshape(-1, 1, 1) 23 | std = np.array([0.229, 0.224, 0.225]).reshape(-1, 1, 1) 24 | img_numpy = (img_numpy - mean) / std 25 | img_numpy = np.expand_dims(img_numpy, axis=0) 26 | img_numpy = img_numpy.astype(np.float32) 27 | 28 | return img_numpy 29 | 30 | 31 | def transforms_cupy(image: Image.Image): 32 | # Convert image to RGB and resize 33 | image = image.convert("RGB") 34 | image = image.resize((448, 448), Image.BICUBIC) 35 | 36 | # Convert to CuPy array and normalize 37 | img_cupy = cp.array(image, dtype=cp.float32) / 255.0 38 | img_cupy = img_cupy.transpose(2, 0, 1) 39 | 40 | # Apply mean and std normalization 41 | mean = cp.array([0.485, 0.456, 0.406], dtype=cp.float32).reshape(-1, 1, 1) 42 | std = cp.array([0.229, 0.224, 0.225], dtype=cp.float32).reshape(-1, 1, 1) 43 | img_cupy = (img_cupy - mean) / std 44 | 45 | # Add batch dimension 46 | img_cupy = cp.expand_dims(img_cupy, axis=0) 47 | 48 | return img_cupy 49 | 50 | 51 | providers = [ 52 | ( 53 | "TensorrtExecutionProvider", 54 | { 55 | "device_id": 0, 56 | "trt_max_workspace_size": 8589934592, 57 | "trt_fp16_enable": True, 58 | "trt_engine_cache_enable": True, 59 | "trt_engine_cache_path": "./trt_cache", 60 | "trt_force_sequential_engine_build": False, 61 | "trt_max_partition_iterations": 10000, 62 | "trt_min_subgraph_size": 1, 63 | "trt_builder_optimization_level": 5, 64 | "trt_timing_cache_enable": True, 65 | }, 66 | ), 67 | ] 68 | 69 | session = ort.InferenceSession("eva02_large_patch14_448.onnx", providers=providers) 70 | 71 | input_name = session.get_inputs()[0].name 72 | output_name = session.get_outputs()[0].name 73 | 74 | output = session.run([output_name], {input_name: transforms_numpy(img)}) 75 | 76 | 77 | num_images = 10 78 | start = time.perf_counter() 79 | for i in range(num_images): 80 | output = session.run([output_name], {input_name: transforms_numpy(img)}) 81 | end = time.perf_counter() 82 | time_taken = end - start 83 | 84 | ms_per_image = time_taken / num_images * 1000 85 | fps = num_images / time_taken 86 | 87 | print(f"Onnxruntime TensorRT: {ms_per_image:.3f} ms per image, FPS: {fps:.2f}") 88 | -------------------------------------------------------------------------------- /sample.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dnth/supercharge-your-pytorch-image-models-blogpost/55253d09dd72c530c4bd2ff550e277a96914930c/sample.mp4 --------------------------------------------------------------------------------