├── .gitignore ├── README.md ├── agents └── example_python │ ├── 1 │ └── model.py │ ├── client.py │ └── config.pbtxt ├── face ├── README.md ├── face_ensemble_python │ ├── 1 │ │ ├── README.md │ │ ├── facenet.py │ │ ├── model.py │ │ ├── mtcnn.py │ │ └── utils.py │ ├── client.py │ └── config.pbtxt ├── facenet │ └── config.pbtxt ├── mtcnn_ensemble_python │ ├── 1 │ │ ├── model.py │ │ └── utils.py │ ├── client.py │ └── config.pbtxt ├── onet │ └── config.pbtxt ├── pnet │ ├── client.py │ └── config.pbtxt └── rnet │ └── config.pbtxt ├── models ├── example_python │ ├── 1 │ │ └── model.py │ ├── client.py │ └── config.pbtxt ├── identity │ ├── 1 │ │ └── .gitkeep │ ├── client.py │ └── config.pbtxt ├── resnet50_dali │ ├── config.pbtxt │ ├── dali_client.py │ └── dali_pipeline.py ├── resnet50_ensemble │ ├── 1 │ │ └── .gitkeep │ ├── config.pbtxt │ ├── labels.txt │ └── resnet_ensemble.py └── resnet50_pytorch │ ├── config.pbtxt │ ├── labels.txt │ ├── resnet_client.py │ └── resnet_pytorch.py ├── quick └── resnet50_pytorch │ ├── cat.jpg │ ├── client_example.py │ ├── config.pbtxt │ ├── labels.txt │ ├── resnet_client.py │ └── resnet_pytorch.py └── rate ├── example_python_a ├── 1 │ └── model.py ├── client.py └── config.pbtxt ├── example_python_b ├── 1 │ └── model.py ├── client.py └── config.pbtxt ├── example_python_c ├── 1 │ └── model.py ├── client.py └── config.pbtxt └── example_python_d ├── 1 └── model.py ├── client.py └── config.pbtxt /.gitignore: -------------------------------------------------------------------------------- 1 | *.pt 2 | *.plan 3 | *.dali 4 | *.pb 5 | *.onnx 6 | *.engine 7 | *tar.gz 8 | 9 | .vscode/ 10 | *.pyc 11 | *.jpg 12 | *.png 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # triton 2 | -------------------------------------------------------------------------------- /agents/example_python/1/model.py: -------------------------------------------------------------------------------- 1 | import json 2 | import triton_python_backend_utils as pb_utils 3 | 4 | 5 | class TritonPythonModel: 6 | 7 | def initialize(self, args): 8 | self.model_config = model_config = json.loads(args['model_config']) 9 | output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") 10 | output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") 11 | self.output0_dtype = pb_utils.triton_string_to_numpy(output0_config['data_type']) 12 | self.output1_dtype = pb_utils.triton_string_to_numpy(output1_config['data_type']) 13 | 14 | def execute(self, requests): 15 | output0_dtype = self.output0_dtype 16 | output1_dtype = self.output1_dtype 17 | responses = [] 18 | for request in requests: 19 | in_0 = pb_utils.get_input_tensor_by_name(request, 'INPUT0') 20 | in_1 = pb_utils.get_input_tensor_by_name(request, 'INPUT1') 21 | out_0, out_1 = (in_0.as_numpy() + in_1.as_numpy(), 22 | in_0.as_numpy() - in_1.as_numpy()) 23 | out_tensor_0 = pb_utils.Tensor('OUTPUT0', out_0.astype(output0_dtype)) 24 | out_tensor_1 = pb_utils.Tensor('OUTPUT1', out_1.astype(output1_dtype)) 25 | inference_response = pb_utils.InferenceResponse(output_tensors=[out_tensor_0, out_tensor_1]) 26 | responses.append(inference_response) 27 | return responses 28 | 29 | def finalize(self): 30 | print('Cleaning up...') 31 | -------------------------------------------------------------------------------- /agents/example_python/client.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tritonclient.http as httpclient 3 | 4 | 5 | if __name__ == '__main__': 6 | triton_client = httpclient.InferenceServerClient(url='127.0.0.1:8000') 7 | 8 | inputs = [] 9 | inputs.append(httpclient.InferInput('INPUT0', [4], "FP32")) 10 | inputs.append(httpclient.InferInput('INPUT1', [4], "FP32")) 11 | input_data0 = np.random.randn(4).astype(np.float32) 12 | input_data1 = np.random.randn(4).astype(np.float32) 13 | inputs[0].set_data_from_numpy(input_data0, binary_data=False) 14 | inputs[1].set_data_from_numpy(input_data1, binary_data=False) 15 | outputs = [] 16 | outputs.append(httpclient.InferRequestedOutput('OUTPUT0', binary_data=False)) 17 | outputs.append(httpclient.InferRequestedOutput('OUTPUT1', binary_data=False)) 18 | 19 | results = triton_client.infer('example_python', inputs=inputs, outputs=outputs) 20 | output_data0 = results.as_numpy('OUTPUT0') 21 | output_data1 = results.as_numpy('OUTPUT1') 22 | 23 | print(input_data0) 24 | print(input_data1) 25 | print(output_data0) 26 | print(output_data1) 27 | -------------------------------------------------------------------------------- /agents/example_python/config.pbtxt: -------------------------------------------------------------------------------- 1 | name: "example_python" 2 | backend: "python" 3 | input [ 4 | { 5 | name: "INPUT0" 6 | data_type: TYPE_FP32 7 | dims: [ 4 ] 8 | } 9 | ] 10 | input [ 11 | { 12 | name: "INPUT1" 13 | data_type: TYPE_FP32 14 | dims: [ 4 ] 15 | } 16 | ] 17 | output [ 18 | { 19 | name: "OUTPUT0" 20 | data_type: TYPE_FP32 21 | dims: [ 4 ] 22 | } 23 | ] 24 | output [ 25 | { 26 | name: "OUTPUT1" 27 | data_type: TYPE_FP32 28 | dims: [ 4 ] 29 | } 30 | ] 31 | 32 | instance_group [ 33 | { 34 | count: 2 35 | kind: KIND_CPU 36 | } 37 | ] 38 | 39 | model_repository_agents 40 | { 41 | agents [ 42 | { 43 | name: "checksum", 44 | parameters 45 | { 46 | key: "MD5:1/model.py", 47 | value: "4e2c177998feb5539d8ec8d820f990bd" 48 | } 49 | } 50 | ] 51 | } 52 | -------------------------------------------------------------------------------- /face/README.md: -------------------------------------------------------------------------------- 1 | reference: 2 | 3 | https://github.com/MAhaitao999/mtcnn-align-facenet-deployment 4 | 5 | -------------------------------------------------------------------------------- /face/face_ensemble_python/1/README.md: -------------------------------------------------------------------------------- 1 | 这里需要一个 `face_dataset` 文件夹，放置人脸图片。启动的时候，会扫描一遍。 2 | -------------------------------------------------------------------------------- /face/face_ensemble_python/1/facenet.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import triton_python_backend_utils as pb_utils 3 | import utils 4 | 5 | 6 | class facenet(object): 7 | def __init__(self): 8 | self.Facenet_inputs = ['input_1'] 9 | self.Facenet_outputs = ['Bottleneck_BatchNorm'] 10 | 11 | def calc_128_vec(self, img): 12 | face_img = utils.pre_process(img) 13 | inference_request = pb_utils.InferenceRequest( 14 | model_name='facenet', 15 | requested_output_names=[self.Facenet_outputs[0]], 16 | inputs=[pb_utils.Tensor(self.Facenet_inputs[0], face_img.astype(np.float32))] 17 | ) 18 | inference_response = inference_request.exec() 19 | pre = utils.pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Facenet_outputs[0])) 20 | pre = utils.l2_normalize(np.concatenate(pre)) 21 | pre = np.reshape(pre, [128]) 22 | 23 | return pre 24 | 25 | -------------------------------------------------------------------------------- /face/face_ensemble_python/1/model.py: -------------------------------------------------------------------------------- 1 | import json 2 | from time import time 3 | import triton_python_backend_utils as pb_utils 4 | import numpy as np 5 | import cv2 6 | import os 7 | import utils 8 | from mtcnn import mtcnn 9 | from facenet import facenet 10 | 11 | 12 | class TritonPythonModel: 13 | 14 | def initialize(self, args): 15 | self.model_config = model_config = json.loads(args['model_config']) 16 | output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") 17 | self.output0_dtype = pb_utils.triton_string_to_numpy(output0_config['data_type']) 18 | 19 | self.is_init = False 20 | 21 | def _build_face_dataset(self): 22 | self.mtcnn_model = mtcnn() 23 | self.threshold = [0.5, 0.6, 0.8] 24 | self.facenet_model = facenet() 25 | face_list = os.listdir('face_dataset') 26 | self.known_face_encodings = [] 27 | self.known_face_names = [] 28 | for face in face_list: 29 | name = face.split('.')[0] 30 | img = cv2.imread('./face_dataset/' + face) 31 | img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 32 | #---------------------# 33 | # 检测人脸 34 | #---------------------# 35 | rectangles = self.mtcnn_model.detect_face(img, self.threshold) 36 | if len(rectangles) <= 0: 37 | continue 38 | #---------------------# 39 | # 转化成正方形 40 | #---------------------# 41 | rectangles = utils.rect2square(np.array(rectangles)) 42 | #-----------------------------------------------# 43 | # facenet要传入一个160x160的图片 44 | # 利用landmark对人脸进行矫正 45 | #-----------------------------------------------# 46 | rectangle = rectangles[0] 47 | landmark = np.reshape(rectangle[5:15], (5, 2)) - np.array([int(rectangle[0]), int(rectangle[1])]) 48 | crop_img = img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] 49 | print(crop_img.shape) 50 | crop_img, _ = utils.Alignment_1(crop_img, landmark) 51 | # cv2.imwrite(name+'.jpg', crop_img) 52 | # print(crop_img.shape) 53 | crop_img = np.expand_dims(cv2.resize(crop_img, (160, 160)), 0) 54 | #--------------------------------------------------------------------# 55 | # 将检测到的人脸传入到facenet的模型中，实现128维特征向量的提取 56 | #--------------------------------------------------------------------# 57 | face_encoding = self.facenet_model.calc_128_vec(crop_img) 58 | 59 | self.known_face_encodings.append(face_encoding) 60 | self.known_face_names.append(name) 61 | 62 | def execute(self, requests): 63 | if not self.is_init: 64 | self._build_face_dataset() 65 | self.is_init = True 66 | output0_dtype = self.output0_dtype 67 | responses = [] 68 | for request in requests: 69 | in_0 = pb_utils.get_input_tensor_by_name(request, 'INPUT0') 70 | t1 = time() 71 | img = self._recognize(in_0.as_numpy()) 72 | t2 = time() 73 | print('inference time cost: {} ms'.format(1000 * (t2 - t1))) 74 | out_tensor_0 = pb_utils.Tensor('OUTPUT0', img.astype(output0_dtype)) 75 | inference_response = pb_utils.InferenceResponse(output_tensors=[out_tensor_0]) 76 | responses.append(inference_response) 77 | return responses 78 | 79 | def finalize(self): 80 | print('Cleaning up...') 81 | 82 | def _recognize(self, draw): 83 | #-----------------------------------------------# 84 | # 人脸识别 85 | # 先定位，再进行数据库匹配 86 | #-----------------------------------------------# 87 | height, width, _ = np.shape(draw) 88 | draw_rgb = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB) 89 | 90 | #--------------------------------# 91 | # 检测人脸 92 | #--------------------------------# 93 | rectangles = self.mtcnn_model.detect_face(draw_rgb, self.threshold) 94 | 95 | if len(rectangles)==0: 96 | return 97 | 98 | # 转化成正方形 99 | rectangles = utils.rect2square(np.array(rectangles,dtype=np.int32)) 100 | rectangles[:, [0,2]] = np.clip(rectangles[:, [0,2]], 0, width) 101 | rectangles[:, [1,3]] = np.clip(rectangles[:, [1,3]], 0, height) 102 | 103 | #-----------------------------------------------# 104 | # 对检测到的人脸进行编码 105 | #-----------------------------------------------# 106 | face_encodings = [] 107 | for rectangle in rectangles: 108 | #---------------# 109 | # 截取图像 110 | #---------------# 111 | landmark = np.reshape(rectangle[5:15], (5,2)) - np.array([int(rectangle[0]), int(rectangle[1])]) 112 | crop_img = draw_rgb[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] 113 | #-----------------------------------------------# 114 | # 利用人脸关键点进行人脸对齐 115 | #-----------------------------------------------# 116 | crop_img, _ = utils.Alignment_1(crop_img, landmark) 117 | crop_img = np.expand_dims(cv2.resize(crop_img, (160, 160)), 0) 118 | 119 | face_encoding = self.facenet_model.calc_128_vec(crop_img) 120 | face_encodings.append(face_encoding) 121 | 122 | face_names = [] 123 | for face_encoding in face_encodings: 124 | #-------------------------------------------------------# 125 | # 取出一张脸并与数据库中所有的人脸进行对比，计算得分 126 | #-------------------------------------------------------# 127 | matches = utils.compare_faces(self.known_face_encodings, face_encoding, tolerance = 0.5) 128 | name = "Unknown" 129 | #-------------------------------------------------------# 130 | # 找出距离最近的人脸 131 | #-------------------------------------------------------# 132 | face_distances = utils.face_distance(self.known_face_encodings, face_encoding) 133 | #-------------------------------------------------------# 134 | # 取出这个最近人脸的评分 135 | #-------------------------------------------------------# 136 | best_match_index = np.argmin(face_distances) 137 | if matches[best_match_index]: 138 | name = self.known_face_names[best_match_index] 139 | face_names.append(name) 140 | 141 | rectangles = rectangles[:, 0:4] 142 | #-----------------------------------------------# 143 | # 画框~!~ 144 | #-----------------------------------------------# 145 | for (left, top, right, bottom), name in zip(rectangles, face_names): 146 | cv2.rectangle(draw, (left, top), (right, bottom), (0, 0, 255), 2) 147 | font = cv2.FONT_HERSHEY_SIMPLEX 148 | cv2.putText(draw, name, (left , bottom - 15), font, 0.75, (255, 255, 255), 2) 149 | return draw 150 | 151 | -------------------------------------------------------------------------------- /face/face_ensemble_python/1/mtcnn.py: -------------------------------------------------------------------------------- 1 | ######################################################################### 2 | # File Name: mtcnn.py 3 | # Author: Henry Ma 4 | # mail: iloveicRazavi@gmail.com 5 | # Create Time: 2021年02月05日 星期五 18时16分17秒 6 | ######################################################################### 7 | 8 | # !/usr/bin/python 9 | # -*- coding:utf-8 -*- 10 | 11 | import time 12 | import cv2 13 | import numpy as np 14 | import triton_python_backend_utils as pb_utils 15 | import utils 16 | 17 | 18 | class mtcnn(object): 19 | 20 | def __init__(self): 21 | #-----------------------------# 22 | # mtcnn的第一段 23 | # 粗略获取人脸框 24 | # 输出bbox位置和是否有人脸 25 | #-----------------------------# 26 | #-----------------------------# 27 | # mtcnn的第二段 28 | # 精修框 29 | #-----------------------------# 30 | #-----------------------------# 31 | # mtcnn的第三段 32 | # 精修框并获得五个点 33 | #-----------------------------# 34 | self.Pnet_inputs = ['input_1'] 35 | self.Pnet_outputs = ['conv4-1', 'conv4-2'] 36 | self.Rnet_inputs = ['input_1'] 37 | self.Rnet_outputs = ['conv5-1', 'conv5-2'] 38 | self.Onet_inputs = ['input_1'] 39 | self.Onet_outputs = ['conv6-1', 'conv6-2', 'conv6-3'] 40 | 41 | def detect_face(self, img, threshold): 42 | #-----------------------------# 43 | # 归一化 44 | #-----------------------------# 45 | copy_img = (img.copy() - 127.5) / 127.5 46 | origin_h, origin_w, _ = copy_img.shape 47 | # print("orgin image's shape is: ", origin_h, origin_w) 48 | #-----------------------------# 49 | # 计算原始输入图像 50 | # 每一次缩放的比例 51 | #-----------------------------# 52 | scales = utils.calculateScales(img) 53 | 54 | out = [] 55 | 56 | #-----------------------------# 57 | # 粗略计算人脸框 58 | # pnet部分 59 | #-----------------------------# 60 | count = 0 61 | for scale in scales: 62 | hs = int(origin_h * scale) 63 | ws = int(origin_w * scale) 64 | scale_img = cv2.resize(copy_img, (ws, hs)) 65 | inputs = np.expand_dims(scale_img, 0).astype(np.float32) 66 | inference_request = pb_utils.InferenceRequest( 67 | model_name='pnet', 68 | requested_output_names=[self.Pnet_outputs[0], self.Pnet_outputs[1]], 69 | inputs=[pb_utils.Tensor(self.Pnet_inputs[0], inputs)] 70 | ) 71 | inference_response = inference_request.exec() 72 | 73 | out0 = utils.pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Pnet_outputs[0]))[0] 74 | out1 = utils.pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Pnet_outputs[1]))[0] 75 | 76 | output = [out0, out1] 77 | out.append(output) 78 | count += 1 79 | 80 | rectangles = [] 81 | #-------------------------------------------------# 82 | # 在这个地方我们对图像金字塔的预测结果进行循环 83 | # 取出每张图片的种类预测和回归预测结果 84 | #-------------------------------------------------# 85 | for i in range(len(scales)): 86 | # print(out[i][0][:, :, 1].shape) 87 | #------------------------------------------------------------------# 88 | # 为了方便理解，这里和视频上看到的不太一样 89 | # 因为我们在上面对图像金字塔循环的时候就把batch_size维度给去掉了 90 | #------------------------------------------------------------------# 91 | cls_prob = out[i][0][:, :, 1] 92 | roi = out[i][1] 93 | #--------------------------------------------# 94 | # 取出每个缩放后图片的高宽 95 | #--------------------------------------------# 96 | out_h, out_w = cls_prob.shape 97 | out_side = max(out_h, out_w) 98 | #--------------------------------------------# 99 | # 解码的过程 100 | #--------------------------------------------# 101 | rectangle = utils.detect_face_12net(cls_prob, roi, out_side, 1 / scales[i], origin_w, origin_h, threshold[0]) 102 | rectangles.extend(rectangle) 103 | 104 | #-----------------------------------------# 105 | # 进行非极大抑制 106 | #-----------------------------------------# 107 | rectangles = np.array(utils.NMS(rectangles, 0.7)) 108 | # print(rectangles) 109 | 110 | if len(rectangles) == 0: 111 | return rectangles 112 | 113 | #-----------------------------------------# 114 | # 稍微精确计算人脸框 115 | # Rnet部分 116 | #-----------------------------------------# 117 | predict_24_batch = [] 118 | for rectangle in rectangles: 119 | #--------------------------------------------# 120 | # 利用获取到的粗略坐标，在原图上进行截取 121 | #--------------------------------------------# 122 | crop_img = copy_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] 123 | #--------------------------------------------# 124 | # 将截取到的图片进行resize，调整成24x24的大小 125 | #--------------------------------------------# 126 | scale_img = cv2.resize(crop_img, (24, 24)) 127 | predict_24_batch.append(scale_img) 128 | 129 | # print('rnet start Inference') 130 | inference_request = pb_utils.InferenceRequest( 131 | model_name='rnet', 132 | requested_output_names=[self.Rnet_outputs[0], self.Rnet_outputs[1]], 133 | inputs=[pb_utils.Tensor(self.Rnet_inputs[0], np.array(predict_24_batch).astype(np.float32))] 134 | ) 135 | inference_response = inference_request.exec() 136 | out0 = utils.pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Rnet_outputs[0])) 137 | out1 = utils.pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Rnet_outputs[1])) 138 | cls_prob = out0 139 | roi_prob = out1 140 | # print('rnet finish Inference') 141 | 142 | #-------------------------------------# 143 | # 解码的过程 144 | #-------------------------------------# 145 | rectangles = utils.filter_face_24net(cls_prob, roi_prob, rectangles, origin_w, origin_h, threshold[1]) 146 | 147 | if len(rectangles) == 0: 148 | return rectangles 149 | 150 | #-----------------------------# 151 | # 计算人脸框 152 | # onet部分 153 | #-----------------------------# 154 | predict_batch = [] 155 | for rectangle in rectangles: 156 | #------------------------------------------# 157 | # 利用获取到的粗略坐标，在原图上进行截取 158 | #------------------------------------------# 159 | crop_img = copy_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] 160 | #-----------------------------------------------# 161 | # 将截取到的图片进行resize，调整成48x48的大小 162 | #-----------------------------------------------# 163 | scale_img = cv2.resize(crop_img, (48, 48)) 164 | predict_batch.append(scale_img) 165 | 166 | # print('onet start Inference') 167 | inference_request = pb_utils.InferenceRequest( 168 | model_name='onet', 169 | requested_output_names=[self.Onet_outputs[0], self.Onet_outputs[1], self.Onet_outputs[2]], 170 | inputs=[pb_utils.Tensor(self.Onet_inputs[0], np.array(predict_batch).astype(np.float32))] 171 | ) 172 | inference_response = inference_request.exec() 173 | out0 = utils.pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Onet_outputs[0])) 174 | out1 = utils.pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Onet_outputs[1])) 175 | out2 = utils.pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Onet_outputs[2])) 176 | cls_prob = out0 177 | roi_prob = out1 178 | pts_prob = out2 179 | #-------------------------------------# 180 | # 解码的过程 181 | #-------------------------------------# 182 | rectangles = utils.filter_face_48net(cls_prob, roi_prob, pts_prob, rectangles, origin_w, origin_h, threshold[2]) 183 | 184 | return rectangles 185 | 186 | def detect_face_on_img(self, img): 187 | threshold = [0.5, 0.6, 0.7] 188 | temp_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 189 | t1 = time.time() 190 | rectangles = self._detect_face(temp_img, threshold) 191 | draw = img.copy() 192 | 193 | for rectangle in rectangles: 194 | cv2.rectangle(draw, (int(rectangle[0]), int(rectangle[1])), 195 | (int(rectangle[2]), int(rectangle[3])), (0, 0, 255), 2) 196 | 197 | for i in range(5, 15, 2): 198 | cv2.circle(draw, (int(rectangle[i+0]), int(rectangle[i+1])), 1, (255, 0, 0), 4) 199 | 200 | t2 = time.time() 201 | print('mtcnn inference time is: {}ms'.format(1000 * (t2 - t1))) 202 | return draw 203 | -------------------------------------------------------------------------------- /face/face_ensemble_python/1/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import math 3 | import cv2 4 | from torch.utils.dlpack import from_dlpack 5 | 6 | 7 | def pb_tensor_to_numpy(pb_tensor): 8 | if pb_tensor.is_cpu(): 9 | return pb_tensor.as_numpy() 10 | else: 11 | pytorch_tensor = from_dlpack(pb_tensor.to_dlpack()) 12 | return pytorch_tensor.cpu().numpy() 13 | 14 | 15 | #-----------------------------# 16 | # 计算原始输入图像 17 | # 每一次缩放的比例 18 | #-----------------------------# 19 | def calculateScales(img): 20 | pr_scale = 1.0 21 | h, w, _ = img.shape 22 | 23 | #--------------------------------------------# 24 | # 将最大的图像大小进行一个固定 25 | # 如果图像的短边大于500，则将短边固定为500 26 | # 如果图像的长边小于500，则将长边固定为500 27 | #--------------------------------------------# 28 | if min(w, h) > 500: 29 | pr_scale = 500.0 / min(h, w) 30 | w = int(w * pr_scale) 31 | h = int(h * pr_scale) 32 | elif max(w, h) < 500: 33 | pr_scale = 500.0 / max(h, w) 34 | w = int(w * pr_scale) 35 | h = int(h * pr_scale) 36 | 37 | #------------------------------------------------# 38 | # 建立图像金字塔的scales，防止图像的宽高小于12 39 | #------------------------------------------------# 40 | scales = [] 41 | factor = 0.709 42 | factor_count = 0 43 | minl = min(h, w) 44 | while minl >= 12: 45 | scales.append(pr_scale*pow(factor, factor_count)) 46 | minl *= factor 47 | factor_count += 1 48 | return scales 49 | 50 | 51 | #-----------------------------# 52 | # 将长方形调整为正方形 53 | #-----------------------------# 54 | def rect2square(rectangles): 55 | w = rectangles[:, 2] - rectangles[:, 0] 56 | h = rectangles[:, 3] - rectangles[:, 1] 57 | l = np.maximum(w, h).T 58 | rectangles[:, 0] = rectangles[:, 0] + w * 0.5 - l * 0.5 59 | rectangles[:, 1] = rectangles[:, 1] + h * 0.5 - l * 0.5 60 | rectangles[:, 2:4] = rectangles[:, 0:2] + np.repeat([l], 2, axis=0).T 61 | return rectangles 62 | 63 | 64 | #-------------------------------------# 65 | # 非极大抑制 66 | #-------------------------------------# 67 | def NMS(rectangles,threshold): 68 | if len(rectangles) == 0: 69 | return rectangles 70 | boxes = np.array(rectangles) 71 | x1 = boxes[:, 0] 72 | y1 = boxes[:, 1] 73 | x2 = boxes[:, 2] 74 | y2 = boxes[:, 3] 75 | s = boxes[:, 4] 76 | area = np.multiply(x2-x1+1, y2-y1+1) 77 | I = np.array(s.argsort()) 78 | pick = [] 79 | while len(I) > 0: 80 | xx1 = np.maximum(x1[I[-1]], x1[I[0:-1]]) # I[-1] have hightest prob score, I[0:-1]->others 81 | yy1 = np.maximum(y1[I[-1]], y1[I[0:-1]]) 82 | xx2 = np.minimum(x2[I[-1]], x2[I[0:-1]]) 83 | yy2 = np.minimum(y2[I[-1]], y2[I[0:-1]]) 84 | w = np.maximum(0.0, xx2 - xx1 + 1) 85 | h = np.maximum(0.0, yy2 - yy1 + 1) 86 | inter = w * h 87 | o = inter / (area[I[-1]] + area[I[0:-1]] - inter) 88 | pick.append(I[-1]) 89 | I = I[np.where(o <= threshold)[0]] 90 | result_rectangle = boxes[pick].tolist() 91 | return result_rectangle 92 | 93 | 94 | #-------------------------------------# 95 | # 对pnet处理后的结果进行处理 96 | # 为了方便理解，我将代码进行了重构 97 | # 具体代码与视频有较大区别 98 | #-------------------------------------# 99 | def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold): 100 | #-------------------------------------# 101 | # 计算特征点之间的步长 102 | #-------------------------------------# 103 | stride = 0 104 | if out_side != 1: 105 | stride = float(2*out_side-1)/(out_side-1) 106 | 107 | #-------------------------------------# 108 | # 获得满足得分门限的特征点的坐标 109 | #-------------------------------------# 110 | (y, x) = np.where(cls_prob >= threshold) 111 | 112 | #-----------------------------------------# 113 | # 获得满足得分门限的特征点得分 114 | # 最终获得的score的shape为：[num_box, 1] 115 | #-------------------------------------------# 116 | score = np.expand_dims(cls_prob[y, x], -1) 117 | 118 | #-------------------------------------------------------# 119 | # 将对应的特征点的坐标转换成位于原图上的先验框的坐标 120 | # 利用回归网络的预测结果对先验框的左上角与右下角进行调整 121 | # 获得对应的粗略预测框 122 | # 最终获得的boundingbox的shape为：[num_box, 4] 123 | #-------------------------------------------------------# 124 | boundingbox = np.concatenate([np.expand_dims(x, -1), np.expand_dims(y, -1)], axis=-1) 125 | top_left = np.fix(stride * boundingbox + 0) 126 | bottom_right = np.fix(stride * boundingbox + 11) 127 | boundingbox = np.concatenate((top_left, bottom_right), axis=-1) 128 | boundingbox = (boundingbox + roi[y, x] * 12.0) * scale 129 | 130 | #-------------------------------------------------------# 131 | # 将预测框和得分进行堆叠，并转换成正方形 132 | # 最终获得的rectangles的shape为：[num_box, 5] 133 | #-------------------------------------------------------# 134 | rectangles = np.concatenate((boundingbox, score), axis=-1) 135 | rectangles = rect2square(rectangles) 136 | 137 | rectangles[:, [1, 3]] = np.clip(rectangles[:, [1, 3]], 0, height) 138 | rectangles[:, [0, 2]] = np.clip(rectangles[:, [0, 2]], 0, width) 139 | return rectangles 140 | 141 | 142 | #-------------------------------------# 143 | # 对Rnet处理后的结果进行处理 144 | # 为了方便理解，我将代码进行了重构 145 | # 具体代码与视频有较大区别 146 | #-------------------------------------# 147 | def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold): 148 | #-------------------------------------# 149 | # 利用得分进行筛选 150 | #-------------------------------------# 151 | pick = cls_prob[:, 1] >= threshold 152 | 153 | score = cls_prob[pick, 1:2] 154 | rectangles = rectangles[pick, :4] 155 | roi = roi[pick, :] 156 | 157 | #-------------------------------------------------------# 158 | # 利用Rnet网络的预测结果对粗略预测框进行调整 159 | # 最终获得的rectangles的shape为：[num_box, 4] 160 | #-------------------------------------------------------# 161 | w = np.expand_dims(rectangles[:, 2] - rectangles[:, 0], -1) 162 | h = np.expand_dims(rectangles[:, 3] - rectangles[:, 1], -1) 163 | rectangles[:, [0, 2]] = rectangles[:, [0, 2]] + roi[:, [0, 2]] * w 164 | rectangles[:, [1, 3]] = rectangles[:, [1, 3]] + roi[:, [1, 3]] * w 165 | 166 | #-------------------------------------------------------# 167 | # 将预测框和得分进行堆叠，并转换成正方形 168 | # 最终获得的rectangles的shape为：[num_box, 5] 169 | #-------------------------------------------------------# 170 | rectangles = np.concatenate((rectangles, score), axis=-1) 171 | rectangles = rect2square(rectangles) 172 | 173 | rectangles[:, [1, 3]] = np.clip(rectangles[:, [1, 3]], 0, height) 174 | rectangles[:, [0, 2]] = np.clip(rectangles[:, [0, 2]], 0, width) 175 | return np.array(NMS(rectangles, 0.7)) 176 | 177 | 178 | #-------------------------------------# 179 | # 对onet处理后的结果进行处理 180 | # 为了方便理解，我将代码进行了重构 181 | # 具体代码与视频有较大区别 182 | #-------------------------------------# 183 | def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold): 184 | #-------------------------------------# 185 | # 利用得分进行筛选 186 | #-------------------------------------# 187 | pick = cls_prob[:, 1] >= threshold 188 | 189 | score = cls_prob[pick, 1:2] 190 | rectangles = rectangles[pick, :4] 191 | pts = pts[pick, :] 192 | roi = roi[pick, :] 193 | 194 | w = np.expand_dims(rectangles[:, 2] - rectangles[:, 0], -1) 195 | h = np.expand_dims(rectangles[:, 3] - rectangles[:, 1], -1) 196 | #-------------------------------------------------------# 197 | # 利用Onet网络的预测结果对预测框进行调整 198 | # 通过解码获得人脸关键点与预测框的坐标 199 | # 最终获得的face_marks的shape为：[num_box, 10] 200 | # 最终获得的rectangles的shape为：[num_box, 4] 201 | #-------------------------------------------------------# 202 | face_marks = np.zeros_like(pts) 203 | face_marks[:, [0, 2, 4, 6, 8]] = w * pts[:, [0, 1, 2, 3, 4]] + rectangles[:, 0:1] 204 | face_marks[:, [1, 3, 5, 7, 9]] = h * pts[:, [5, 6, 7, 8, 9]] + rectangles[:, 1:2] 205 | rectangles[:, [0, 2]] = rectangles[:, [0, 2]] + roi[:, [0, 2]] * w 206 | rectangles[:, [1, 3]] = rectangles[:, [1, 3]] + roi[:, [1, 3]] * w 207 | #-------------------------------------------------------# 208 | # 将预测框和得分进行堆叠 209 | # 最终获得的rectangles的shape为：[num_box, 15] 210 | #-------------------------------------------------------# 211 | rectangles = np.concatenate((rectangles, score, face_marks), axis=-1) 212 | 213 | rectangles[:, [1, 3]] = np.clip(rectangles[:, [1, 3]], 0, height) 214 | rectangles[:, [0, 2]] = np.clip(rectangles[:, [0, 2]], 0, width) 215 | return np.array(NMS(rectangles, 0.3)) 216 | 217 | 218 | #-------------------------------------# 219 | # 人脸对齐 220 | #-------------------------------------# 221 | def Alignment_1(img, landmark): 222 | if landmark.shape[0] == 68: 223 | x = landmark[36, 0] - landmark[45, 0] 224 | y = landmark[36, 1] - landmark[45, 1] 225 | elif landmark.shape[0] == 5: 226 | x = landmark[0, 0] - landmark[1, 0] 227 | y = landmark[0, 1] - landmark[1, 1] 228 | 229 | if x == 0: 230 | angle = 0 231 | else: 232 | angle = math.atan(y / x) * 180 / math.pi 233 | 234 | center = (img.shape[1]//2, img.shape[0]//2) 235 | 236 | RotationMatrix = cv2.getRotationMatrix2D(center, angle, 1) 237 | new_img = cv2.warpAffine(img, RotationMatrix, (img.shape[1], img.shape[0])) 238 | 239 | RotationMatrix = np.array(RotationMatrix) 240 | new_landmark = [] 241 | for i in range(landmark.shape[0]): 242 | pts = [] 243 | pts.append(RotationMatrix[0, 0] * landmark[i, 0] + RotationMatrix[0, 1] * landmark[i, 1] + RotationMatrix[0, 2]) 244 | pts.append(RotationMatrix[1, 0] * landmark[i, 0] + RotationMatrix[1, 1] * landmark[i, 1] + RotationMatrix[1, 2]) 245 | new_landmark.append(pts) 246 | 247 | new_landmark = np.array(new_landmark) 248 | 249 | return new_img, new_landmark 250 | 251 | 252 | def Alignment_2(img, std_landmark, landmark): 253 | def Transformation(std_landmark, landmark): 254 | std_landmark = np.matrix(std_landmark).astype(np.float64) 255 | landmark = np.matrix(landmark).astype(np.float64) 256 | 257 | c1 = np.mean(std_landmark, axis=0) 258 | c2 = np.mean(landmark, axis=0) 259 | std_landmark -= c1 260 | landmark -= c2 261 | 262 | s1 = np.std(std_landmark) 263 | s2 = np.std(landmark) 264 | std_landmark /= s1 265 | landmark /= s2 266 | 267 | U, S, Vt = np.linalg.svd(std_landmark.T * landmark) 268 | R = (U * Vt).T 269 | 270 | return np.vstack([np.hstack(((s2 / s1) * R, c2.T - (s2 / s1) * R * c1.T)), np.matrix([0., 0., 1.])]) 271 | 272 | Trans_Matrix = Transformation(std_landmark, landmark) # Shape: 3 * 3 273 | Trans_Matrix = Trans_Matrix[:2] 274 | Trans_Matrix = cv2.invertAffineTransform(Trans_Matrix) 275 | new_img = cv2.warpAffine(img, Trans_Matrix, (img.shape[1], img.shape[0])) 276 | 277 | Trans_Matrix = np.array(Trans_Matrix) 278 | new_landmark = [] 279 | for i in range(landmark.shape[0]): 280 | pts = [] 281 | pts.append(Trans_Matrix[0, 0] * landmark[i, 0] + Trans_Matrix[0, 1] * landmark[i, 1] + Trans_Matrix[0, 2]) 282 | pts.append(Trans_Matrix[1, 0] * landmark[i, 0] + Trans_Matrix[1, 1] * landmark[i, 1] + Trans_Matrix[1, 2]) 283 | new_landmark.append(pts) 284 | 285 | new_landmark = np.array(new_landmark) 286 | 287 | return new_img, new_landmark 288 | 289 | 290 | #---------------------------------# 291 | # 图片预处理 292 | # 高斯归一化 293 | #---------------------------------# 294 | def pre_process(x): 295 | if x.ndim == 4: 296 | axis = (1, 2, 3) 297 | size = x[0].size 298 | elif x.ndim == 3: 299 | axis = (0, 1, 2) 300 | size = x.size 301 | else: 302 | raise ValueError('Dimension should be 3 or 4') 303 | 304 | mean = np.mean(x, axis=axis, keepdims=True) 305 | std = np.std(x, axis=axis, keepdims=True) 306 | std_adj = np.maximum(std, 1.0/np.sqrt(size)) 307 | y = (x - mean) / std_adj 308 | return y 309 | 310 | 311 | #---------------------------------# 312 | # l2标准化 313 | #---------------------------------# 314 | def l2_normalize(x, axis=-1, epsilon=1e-10): 315 | output = x / np.sqrt(np.maximum(np.sum(np.square(x), axis=axis, keepdims=True), epsilon)) 316 | return output 317 | 318 | 319 | #---------------------------------# 320 | # 计算人脸距离 321 | #---------------------------------# 322 | def face_distance(face_encodings, face_to_compare): 323 | if len(face_encodings) == 0: 324 | return np.empty((0)) 325 | return np.linalg.norm(face_encodings - face_to_compare, axis=1) 326 | 327 | 328 | #---------------------------------# 329 | # 比较人脸 330 | #---------------------------------# 331 | def compare_faces(known_face_encodings, face_encoding_to_check, tolerance=0.5): 332 | dis = face_distance(known_face_encodings, face_encoding_to_check) 333 | return list(dis <= tolerance) 334 | -------------------------------------------------------------------------------- /face/face_ensemble_python/client.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import tritonclient.http as httpclient 4 | import time 5 | 6 | 7 | if __name__ == '__main__': 8 | triton_client = httpclient.InferenceServerClient(url='127.0.0.1:8000') 9 | 10 | img = cv2.imread('batman.jpg').astype(np.float32) 11 | 12 | inputs = [] 13 | inputs.append(httpclient.InferInput('INPUT0', [*img.shape], "FP32")) 14 | # binary_data 默认是 True, 表示传输的时候使用二进制格式, 否则使用 JSON 文本(大小不一样) 15 | inputs[0].set_data_from_numpy(img, binary_data=True) 16 | outputs = [] 17 | outputs.append(httpclient.InferRequestedOutput('OUTPUT0', binary_data=False)) 18 | 19 | t1 = time.time() 20 | results = triton_client.infer('face_ensemble_python', inputs=inputs, outputs=outputs) 21 | t2 = time.time() 22 | print('inference time is: {}ms'.format(1000 * (t2 - t1))) 23 | output_data0 = results.as_numpy('OUTPUT0') 24 | 25 | print(img.shape) 26 | print(output_data0.shape) 27 | cv2.imwrite('out.jpg', output_data0.astype(np.uint8)) 28 | -------------------------------------------------------------------------------- /face/face_ensemble_python/config.pbtxt: -------------------------------------------------------------------------------- 1 | name: "face_ensemble_python" 2 | backend: "python" 3 | input [ 4 | { 5 | name: "INPUT0" 6 | data_type: TYPE_FP32 7 | dims: [ -1, -1, 3 ] 8 | } 9 | ] 10 | 11 | output [ 12 | { 13 | name: "OUTPUT0" 14 | data_type: TYPE_FP32 15 | dims: [ -1, -1, 3 ] 16 | } 17 | ] 18 | 19 | instance_group [ 20 | { 21 | count: 1 22 | kind: KIND_GPU 23 | gpus: [ 0 ] 24 | } 25 | ] 26 | 27 | parameters: { 28 | key: "EXECUTION_ENV_PATH", 29 | value: {string_value: "$$TRITON_MODEL_DIRECTORY/triton.tar.gz"} 30 | } 31 | -------------------------------------------------------------------------------- /face/facenet/config.pbtxt: -------------------------------------------------------------------------------- 1 | name:"facenet" 2 | platform: "tensorrt_plan" 3 | max_batch_size: 16 4 | input [ 5 | { 6 | name: "input_1" 7 | data_type: TYPE_FP32 8 | dims: [160, 160, 3] 9 | } 10 | ] 11 | 12 | output [ 13 | { 14 | name: "Bottleneck_BatchNorm" 15 | data_type: TYPE_FP32 16 | dims: [ 128 ] 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /face/mtcnn_ensemble_python/1/model.py: -------------------------------------------------------------------------------- 1 | import json 2 | import triton_python_backend_utils as pb_utils 3 | import numpy as np 4 | import cv2 5 | import os 6 | import utils 7 | import time 8 | from torch.utils.dlpack import from_dlpack 9 | 10 | 11 | def pb_tensor_to_numpy(pb_tensor): 12 | if pb_tensor.is_cpu(): 13 | return pb_tensor.as_numpy() 14 | else: 15 | pytorch_tensor = from_dlpack(pb_tensor.to_dlpack()) 16 | return pytorch_tensor.cpu().numpy() 17 | 18 | 19 | class TritonPythonModel: 20 | 21 | def initialize(self, args): 22 | self.model_config = model_config = json.loads(args['model_config']) 23 | output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") 24 | self.output0_dtype = pb_utils.triton_string_to_numpy(output0_config['data_type']) 25 | print('----------------------------------') 26 | print(output0_config['data_type']) 27 | print(self.output0_dtype) 28 | print(os.getcwd()) 29 | print('----------------------------------') 30 | 31 | self.Pnet_inputs = ['input_1'] 32 | self.Pnet_outputs = ['conv4-1', 'conv4-2'] 33 | self.Rnet_inputs = ['input_1'] 34 | self.Rnet_outputs = ['conv5-1', 'conv5-2'] 35 | self.Onet_inputs = ['input_1'] 36 | self.Onet_outputs = ['conv6-1', 'conv6-2', 'conv6-3'] 37 | 38 | def execute(self, requests): 39 | output0_dtype = self.output0_dtype 40 | responses = [] 41 | for request in requests: 42 | in_0 = pb_utils.get_input_tensor_by_name(request, 'INPUT0') 43 | img = self._detect_face_on_img(in_0.as_numpy()) 44 | out_tensor_0 = pb_utils.Tensor('OUTPUT0', img.astype(output0_dtype)) 45 | inference_response = pb_utils.InferenceResponse(output_tensors=[out_tensor_0]) 46 | responses.append(inference_response) 47 | return responses 48 | 49 | def finalize(self): 50 | print('Cleaning up...') 51 | 52 | def _detect_face(self, img, threshold): 53 | #-----------------------------# 54 | # 归一化 55 | #-----------------------------# 56 | copy_img = (img.copy() - 127.5) / 127.5 57 | origin_h, origin_w, _ = copy_img.shape 58 | # print("orgin image's shape is: ", origin_h, origin_w) 59 | #-----------------------------# 60 | # 计算原始输入图像 61 | # 每一次缩放的比例 62 | #-----------------------------# 63 | scales = utils.calculateScales(img) 64 | 65 | out = [] 66 | 67 | #-----------------------------# 68 | # 粗略计算人脸框 69 | # pnet部分 70 | #-----------------------------# 71 | count = 0 72 | for scale in scales: 73 | # print('{}/{}'.format(count, len(scales))) 74 | hs = int(origin_h * scale) 75 | ws = int(origin_w * scale) 76 | scale_img = cv2.resize(copy_img, (ws, hs)) 77 | inputs = np.expand_dims(scale_img, 0).astype(np.float32) 78 | inference_request = pb_utils.InferenceRequest( 79 | model_name='pnet', 80 | requested_output_names=[self.Pnet_outputs[0], self.Pnet_outputs[1]], 81 | inputs=[pb_utils.Tensor(self.Pnet_inputs[0], inputs)] 82 | ) 83 | inference_response = inference_request.exec() 84 | # print('pnet inference finished ' + str(pb_utils.get_output_tensor_by_name(inference_response, self.Pnet_outputs[0]).is_cpu())) 85 | # print(pb_utils.get_output_tensor_by_name(inference_response, self.Pnet_outputs[0]).triton_dtype) 86 | 87 | out0 = pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Pnet_outputs[0]))[0] 88 | out1 = pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Pnet_outputs[1]))[0] 89 | 90 | output = [out0, out1] 91 | out.append(output) 92 | # print(out0.shape, out1.shape) 93 | count += 1 94 | 95 | # print('finish') 96 | rectangles = [] 97 | #-------------------------------------------------# 98 | # 在这个地方我们对图像金字塔的预测结果进行循环 99 | # 取出每张图片的种类预测和回归预测结果 100 | #-------------------------------------------------# 101 | for i in range(len(scales)): 102 | # print(out[i][0][:, :, 1].shape) 103 | #------------------------------------------------------------------# 104 | # 为了方便理解，这里和视频上看到的不太一样 105 | # 因为我们在上面对图像金字塔循环的时候就把batch_size维度给去掉了 106 | #------------------------------------------------------------------# 107 | cls_prob = out[i][0][:, :, 1] 108 | roi = out[i][1] 109 | #--------------------------------------------# 110 | # 取出每个缩放后图片的高宽 111 | #--------------------------------------------# 112 | out_h, out_w = cls_prob.shape 113 | out_side = max(out_h, out_w) 114 | #--------------------------------------------# 115 | # 解码的过程 116 | #--------------------------------------------# 117 | rectangle = utils.detect_face_12net(cls_prob, roi, out_side, 1 / scales[i], origin_w, origin_h, threshold[0]) 118 | rectangles.extend(rectangle) 119 | 120 | #-----------------------------------------# 121 | # 进行非极大抑制 122 | #-----------------------------------------# 123 | rectangles = np.array(utils.NMS(rectangles, 0.7)) 124 | # print(rectangles) 125 | 126 | if len(rectangles) == 0: 127 | return rectangles 128 | 129 | #-----------------------------------------# 130 | # 稍微精确计算人脸框 131 | # Rnet部分 132 | #-----------------------------------------# 133 | predict_24_batch = [] 134 | for rectangle in rectangles: 135 | #--------------------------------------------# 136 | # 利用获取到的粗略坐标，在原图上进行截取 137 | #--------------------------------------------# 138 | crop_img = copy_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] 139 | #--------------------------------------------# 140 | # 将截取到的图片进行resize，调整成24x24的大小 141 | #--------------------------------------------# 142 | scale_img = cv2.resize(crop_img, (24, 24)) 143 | predict_24_batch.append(scale_img) 144 | 145 | # print('rnet start Inference') 146 | inference_request = pb_utils.InferenceRequest( 147 | model_name='rnet', 148 | requested_output_names=[self.Rnet_outputs[0], self.Rnet_outputs[1]], 149 | inputs=[pb_utils.Tensor(self.Rnet_inputs[0], np.array(predict_24_batch).astype(np.float32))] 150 | ) 151 | inference_response = inference_request.exec() 152 | out0 = pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Rnet_outputs[0])) 153 | out1 = pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Rnet_outputs[1])) 154 | cls_prob = out0 155 | roi_prob = out1 156 | # print('rnet finish Inference') 157 | 158 | #-------------------------------------# 159 | # 解码的过程 160 | #-------------------------------------# 161 | rectangles = utils.filter_face_24net(cls_prob, roi_prob, rectangles, origin_w, origin_h, threshold[1]) 162 | 163 | if len(rectangles) == 0: 164 | return rectangles 165 | 166 | # print(rectangles) 167 | 168 | #-----------------------------# 169 | # 计算人脸框 170 | # onet部分 171 | #-----------------------------# 172 | predict_batch = [] 173 | for rectangle in rectangles: 174 | #------------------------------------------# 175 | # 利用获取到的粗略坐标，在原图上进行截取 176 | #------------------------------------------# 177 | crop_img = copy_img[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] 178 | #-----------------------------------------------# 179 | # 将截取到的图片进行resize，调整成48x48的大小 180 | #-----------------------------------------------# 181 | scale_img = cv2.resize(crop_img, (48, 48)) 182 | predict_batch.append(scale_img) 183 | 184 | # print('onet start Inference') 185 | inference_request = pb_utils.InferenceRequest( 186 | model_name='onet', 187 | requested_output_names=[self.Onet_outputs[0], self.Onet_outputs[1], self.Onet_outputs[2]], 188 | inputs=[pb_utils.Tensor(self.Onet_inputs[0], np.array(predict_batch).astype(np.float32))] 189 | ) 190 | inference_response = inference_request.exec() 191 | out0 = pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Onet_outputs[0])) 192 | out1 = pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Onet_outputs[1])) 193 | out2 = pb_tensor_to_numpy(pb_utils.get_output_tensor_by_name(inference_response, self.Onet_outputs[2])) 194 | cls_prob = out0 195 | roi_prob = out1 196 | pts_prob = out2 197 | # print('onet finish Inference') 198 | 199 | #-------------------------------------# 200 | # 解码的过程 201 | #-------------------------------------# 202 | # print('cls_prob:') 203 | # print(cls_prob) 204 | # print('roi_prob:') 205 | # print(roi_prob) 206 | # print('pts_prob:') 207 | # print(pts_prob) 208 | rectangles = utils.filter_face_48net(cls_prob, roi_prob, pts_prob, rectangles, origin_w, origin_h, threshold[2]) 209 | 210 | return rectangles 211 | 212 | def _detect_face_on_img(self, img): 213 | threshold = [0.5, 0.6, 0.7] 214 | temp_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 215 | t1 = time.time() 216 | rectangles = self._detect_face(temp_img, threshold) 217 | draw = img.copy() 218 | 219 | for rectangle in rectangles: 220 | cv2.rectangle(draw, (int(rectangle[0]), int(rectangle[1])), 221 | (int(rectangle[2]), int(rectangle[3])), (0, 0, 255), 2) 222 | 223 | for i in range(5, 15, 2): 224 | cv2.circle(draw, (int(rectangle[i+0]), int(rectangle[i+1])), 1, (255, 0, 0), 4) 225 | 226 | t2 = time.time() 227 | print('inference time is: {}ms'.format(1000 * (t2 - t1))) 228 | # cv2.imwrite('out.jpg', draw) 229 | return draw 230 | -------------------------------------------------------------------------------- /face/mtcnn_ensemble_python/1/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | #-----------------------------# 5 | # 计算原始输入图像 6 | # 每一次缩放的比例 7 | #-----------------------------# 8 | def calculateScales(img): 9 | pr_scale = 1.0 10 | h, w, _ = img.shape 11 | 12 | #-----------------------------------------------------# 13 | # 将最大的图像大小进行一个固定 14 | # 如果图像的短边大于500，则将短边固定为500 15 | # 如果图像的长边小于500，则将长边固定为500 16 | #-----------------------------------------------------# 17 | if min(w, h) > 500: 18 | pr_scale = 500.0 / min(h, w) 19 | w = int(w * pr_scale) 20 | h = int(h * pr_scale) 21 | elif max(w, h) < 500: 22 | pr_scale = 500.0 / max(h, w) 23 | w = int(w * pr_scale) 24 | h = int(h * pr_scale) 25 | 26 | #-----------------------------------------------------# 27 | # 建立图像金字塔的scales，防止图像的宽高小于12 28 | #-----------------------------------------------------# 29 | scales = [] 30 | factor = 0.709 31 | factor_count = 0 32 | minl = min(h, w) 33 | while minl >= 12: 34 | scales.append(pr_scale * pow(factor, factor_count)) 35 | minl *= factor 36 | factor_count += 1 37 | 38 | # print(scales) 39 | return scales 40 | 41 | 42 | #---------------------------------# 43 | # 将长方形调整为正方形 44 | #---------------------------------# 45 | def rect2square(rectangles): 46 | w = rectangles[:, 2] - rectangles[:, 0] 47 | h = rectangles[:, 3] - rectangles[:, 1] 48 | l = np.maximum(w, h).T 49 | rectangles[:, 0] = rectangles[:, 0] + w * 0.5 - l * 0.5 50 | rectangles[:, 1] = rectangles[:, 1] + h * 0.5 - l * 0.5 51 | rectangles[:, 2:4] = rectangles[:, 0:2] + np.repeat([l], 2, axis=0).T 52 | return rectangles 53 | 54 | 55 | #-------------------------------------# 56 | # 非极大抑制 57 | #-------------------------------------# 58 | def NMS(rectangles, threshold): 59 | if len(rectangles) == 0: 60 | return rectangles 61 | boxes = np.array(rectangles) 62 | x1 = boxes[:, 0] 63 | y1 = boxes[:, 1] 64 | x2 = boxes[:, 2] 65 | y2 = boxes[:, 3] 66 | s = boxes[:, 4] 67 | area = np.multiply(x2-x1+1, y2-y1+1) 68 | I = np.array(s.argsort()) 69 | pick = [] 70 | while len(I) > 0: 71 | xx1 = np.maximum(x1[I[-1]], x1[I[0:-1]]) # I[-1] have hightest prob score, I[0:-1]->others 72 | yy1 = np.maximum(y1[I[-1]], y1[I[0:-1]]) 73 | xx2 = np.minimum(x2[I[-1]], x2[I[0:-1]]) 74 | yy2 = np.minimum(y2[I[-1]], y2[I[0:-1]]) 75 | w = np.maximum(0.0, xx2 - xx1 + 1) 76 | h = np.maximum(0.0, yy2 - yy1 + 1) 77 | inter = w * h 78 | o = inter / (area[I[-1]] + area[I[0:-1]] - inter) 79 | pick.append(I[-1]) 80 | I = I[np.where(o <= threshold)[0]] 81 | result_rectangle = boxes[pick].tolist() 82 | return result_rectangle 83 | 84 | 85 | #--------------------------------------------# 86 | # 对pnet处理后的结果进行处理 87 | # 为了方便理解, 我将代码进行了重构 88 | # 具体代码与视频有较大区别 89 | #--------------------------------------------# 90 | def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold): 91 | #-------------------------------------# 92 | # 计算特征点之间的步长 93 | #-------------------------------------# 94 | stride = 0 95 | if out_side != 1: 96 | stride = float(2 * out_side - 1) / (out_side - 1) 97 | 98 | #-------------------------------------# 99 | # 获得满足得分门限的特征点的坐标 100 | #-------------------------------------# 101 | (y, x) = np.where(cls_prob >= threshold) 102 | 103 | #----------------------------------------# 104 | # 获得满足得分门限的特征点得分 105 | # 最终获得的score的shape为: [num_box, 1] 106 | #----------------------------------------# 107 | score = np.expand_dims(cls_prob[y, x], -1) 108 | 109 | #--------------------------------------------------# 110 | # 将对应的特征点的坐标转换成位于原图上的先验框的坐标 111 | # 利用回归网络的预测结果对先验框的左上角与右下角进行调整 112 | # 获得对应的粗略预测框 113 | # 最终获得的boundingbox的shape为: [num_box, 4] 114 | #--------------------------------------------------# 115 | boundingbox = np.concatenate([np.expand_dims(x, -1), np.expand_dims(y, -1)], axis=-1) 116 | top_left = np.fix(stride * boundingbox + 0) 117 | bottom_right = np.fix(stride * boundingbox + 11) 118 | boundingbox = np.concatenate((top_left, bottom_right), axis=-1) 119 | boundingbox = (boundingbox + roi[y, x] * 12.0) * scale 120 | 121 | #--------------------------------------------------# 122 | # 将预测框和得分进行叠加，并转化成正方形 123 | # 最终获得的rectangles的shape为: [num_box, 5] 124 | #--------------------------------------------------# 125 | rectangles = np.concatenate((boundingbox, score), axis=-1) 126 | rectangles = rect2square(rectangles) 127 | 128 | rectangles[:, [1, 3]] = np.clip(rectangles[:, [1, 3]], 0, height) 129 | rectangles[:, [0, 2]] = np.clip(rectangles[:, [0, 2]], 0, width) 130 | return rectangles 131 | 132 | 133 | #-------------------------------------# 134 | # 对Rnet处理后的结果进行处理 135 | # 为了方便理解，我将代码进行了重构 136 | # 具体代码与视频有较大区别 137 | #-------------------------------------# 138 | def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold): 139 | #---------------------------------------# 140 | # 利用得分进行筛选 141 | #---------------------------------------# 142 | pick = cls_prob[:, 1] >= threshold 143 | 144 | score = cls_prob[pick, 1:2] 145 | rectangles = rectangles[pick, :4] 146 | roi = roi[pick, :] 147 | 148 | #-----------------------------------------------# 149 | # 利用Rnet网络的预测结果对粗略预测进行调整 150 | # 最终获得的rectangles的shape为: [num_box, 4] 151 | #-----------------------------------------------# 152 | w = np.expand_dims(rectangles[:, 2] - rectangles[:, 0], -1) 153 | h = np.expand_dims(rectangles[:, 3] - rectangles[:, 1], -1) 154 | rectangles[:, [0, 2]] = rectangles[:, [0, 2]] + roi[:, [0, 2]] * w 155 | rectangles[:, [1, 3]] = rectangles[:, [1, 3]] + roi[:, [1, 3]] * w 156 | 157 | #-------------------------------------------------------# 158 | # 将预测框和得分进行堆叠，并转换成正方形 159 | # 最终获得的rectangles的shape为：[num_box, 5] 160 | #-------------------------------------------------------# 161 | rectangles = np.concatenate((rectangles, score), axis=-1) 162 | rectangles = rect2square(rectangles) 163 | 164 | rectangles[:, [1, 3]] = np.clip(rectangles[:, [1, 3]], 0, height) 165 | rectangles[:, [0, 2]] = np.clip(rectangles[:, [0, 2]], 0, width) 166 | return np.array(NMS(rectangles, 0.7)) 167 | 168 | 169 | #-------------------------------------# 170 | # 对onet处理后的结果进行处理 171 | # 为了方便理解，我将代码进行了重构 172 | # 具体代码与视频有较大区别 173 | #-------------------------------------# 174 | def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold): 175 | #-------------------------------------# 176 | # 利用得分进行筛选 177 | #-------------------------------------# 178 | pick = cls_prob[:, 1] >= threshold 179 | 180 | score = cls_prob[pick, 1:2] 181 | rectangles = rectangles[pick, :4] 182 | pts = pts[pick, :] 183 | roi = roi[pick, :] 184 | 185 | w = np.expand_dims(rectangles[:, 2] - rectangles[:, 0], -1) 186 | h = np.expand_dims(rectangles[:, 3] - rectangles[:, 1], -1) 187 | #-------------------------------------------------------# 188 | # 利用Onet网络的预测结果对预测框进行调整 189 | # 通过解码获得人脸关键点与预测框的坐标 190 | # 最终获得的face_marks的shape为：[num_box, 10] 191 | # 最终获得的rectangles的shape为：[num_box, 4] 192 | #-------------------------------------------------------# 193 | face_marks = np.zeros_like(pts) 194 | face_marks[:, [0, 2, 4, 6, 8]] = w * pts[:, [0, 1, 2, 3, 4]] + rectangles[:, 0:1] 195 | face_marks[:, [1, 3, 5, 7, 9]] = h * pts[:, [5, 6, 7, 8, 9]] + rectangles[:, 1:2] 196 | rectangles[:, [0, 2]] = rectangles[:, [0, 2]] + roi[:, [0, 2]] * w 197 | rectangles[:, [1, 3]] = rectangles[:, [1, 3]] + roi[:, [1, 3]] * w 198 | #-------------------------------------------------------# 199 | # 将预测框和得分进行堆叠 200 | # 最终获得的rectangles的shape为：[num_box, 15] 201 | #-------------------------------------------------------# 202 | rectangles = np.concatenate((rectangles, score, face_marks), axis=-1) 203 | 204 | rectangles[:, [1, 3]] = np.clip(rectangles[:, [1, 3]], 0, height) 205 | rectangles[:, [0, 2]] = np.clip(rectangles[:, [0, 2]], 0, width) 206 | return np.array(NMS(rectangles, 0.3)) 207 | 208 | -------------------------------------------------------------------------------- /face/mtcnn_ensemble_python/client.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import tritonclient.http as httpclient 4 | import time 5 | 6 | 7 | if __name__ == '__main__': 8 | triton_client = httpclient.InferenceServerClient(url='127.0.0.1:8000') 9 | 10 | img = cv2.imread('test.jpg').astype(np.float32) 11 | 12 | inputs = [] 13 | inputs.append(httpclient.InferInput('INPUT0', [*img.shape], "FP32")) 14 | # binary_data 默认是 True, 表示传输的时候使用二进制格式, 否则使用 JSON 文本(大小不一样) 15 | inputs[0].set_data_from_numpy(img, binary_data=True) 16 | outputs = [] 17 | outputs.append(httpclient.InferRequestedOutput('OUTPUT0', binary_data=False)) 18 | 19 | t1 = time.time() 20 | results = triton_client.infer('mtcnn_ensemble_python', inputs=inputs, outputs=outputs) 21 | t2 = time.time() 22 | print('inference time is: {}ms'.format(1000 * (t2 - t1))) 23 | output_data0 = results.as_numpy('OUTPUT0') 24 | 25 | print(img.shape) 26 | print(output_data0.shape) 27 | # cv2.imwrite('out.jpg', output_data0.astype(np.uint8)) 28 | -------------------------------------------------------------------------------- /face/mtcnn_ensemble_python/config.pbtxt: -------------------------------------------------------------------------------- 1 | name: "mtcnn_ensemble_python" 2 | backend: "python" 3 | input [ 4 | { 5 | name: "INPUT0" 6 | data_type: TYPE_FP32 7 | dims: [ -1, -1, 3 ] 8 | } 9 | ] 10 | 11 | output [ 12 | { 13 | name: "OUTPUT0" 14 | data_type: TYPE_FP32 15 | dims: [ -1, -1, 3 ] 16 | } 17 | ] 18 | 19 | instance_group [ 20 | { 21 | count: 1 22 | kind: KIND_GPU 23 | gpus: [ 0 ] 24 | } 25 | ] 26 | 27 | parameters: { 28 | key: "EXECUTION_ENV_PATH", 29 | value: {string_value: "$$TRITON_MODEL_DIRECTORY/triton.tar.gz"} 30 | } 31 | -------------------------------------------------------------------------------- /face/onet/config.pbtxt: -------------------------------------------------------------------------------- 1 | name:"onet" 2 | platform: "tensorrt_plan" 3 | max_batch_size: 256 4 | input [ 5 | { 6 | name: "input_1" 7 | data_type: TYPE_FP32 8 | dims: [48, 48, 3] 9 | } 10 | ] 11 | 12 | output [ 13 | { 14 | name: "conv6-3" 15 | data_type: TYPE_FP32 16 | dims: [ 10 ] 17 | }, 18 | 19 | { 20 | name: "conv6-2" 21 | data_type: TYPE_FP32 22 | dims: [ 4 ] 23 | }, 24 | 25 | { 26 | name: "conv6-1" 27 | data_type: TYPE_FP32 28 | dims: [ 2 ] 29 | } 30 | ] 31 | 32 | -------------------------------------------------------------------------------- /face/pnet/client.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import tritonclient.http as httpclient 4 | import time 5 | 6 | 7 | if __name__ == '__main__': 8 | triton_client = httpclient.InferenceServerClient(url='127.0.0.1:8000') 9 | 10 | img = cv2.imread('cat.jpg').astype(np.float32) 11 | img = np.expand_dims(img, axis=0) 12 | 13 | inputs = [] 14 | inputs.append(httpclient.InferInput('input_1', [*img.shape], "FP32")) 15 | inputs[0].set_data_from_numpy(img, binary_data=True) 16 | outputs = [] 17 | outputs.append(httpclient.InferRequestedOutput('conv4-2', binary_data=True)) 18 | outputs.append(httpclient.InferRequestedOutput('conv4-1', binary_data=True)) 19 | 20 | t1 = time.time() 21 | results = triton_client.infer('pnet', inputs=inputs, outputs=outputs) 22 | t2 = time.time() 23 | print('inference time is: {}ms'.format(1000 * (t2 - t1))) 24 | output_data0 = results.as_numpy('conv4-2') 25 | 26 | print(img.shape) 27 | print(output_data0.shape) 28 | 29 | -------------------------------------------------------------------------------- /face/pnet/config.pbtxt: -------------------------------------------------------------------------------- 1 | name:"pnet" 2 | platform: "tensorrt_plan" 3 | max_batch_size: 8 4 | input [ 5 | { 6 | name: "input_1" 7 | data_type: TYPE_FP32 8 | dims: [-1, -1, 3] 9 | } 10 | ] 11 | 12 | output [ 13 | { 14 | name: "conv4-2" 15 | data_type: TYPE_FP32 16 | dims: [-1, -1, 4] 17 | }, 18 | 19 | { 20 | name: "conv4-1" 21 | data_type: TYPE_FP32 22 | dims: [-1, -1, 2] 23 | } 24 | ] 25 | 26 | -------------------------------------------------------------------------------- /face/rnet/config.pbtxt: -------------------------------------------------------------------------------- 1 | name:"rnet" 2 | platform: "tensorrt_plan" 3 | max_batch_size: 4096 4 | input [ 5 | { 6 | name: "input_1" 7 | data_type: TYPE_FP32 8 | dims: [24, 24, 3] 9 | } 10 | ] 11 | 12 | output [ 13 | { 14 | name: "conv5-2" 15 | data_type: TYPE_FP32 16 | dims: [ 4 ] 17 | }, 18 | 19 | { 20 | name: "conv5-1" 21 | data_type: TYPE_FP32 22 | dims: [ 2 ] 23 | } 24 | ] 25 | 26 | -------------------------------------------------------------------------------- /models/example_python/1/model.py: -------------------------------------------------------------------------------- 1 | import json 2 | import triton_python_backend_utils as pb_utils 3 | 4 | 5 | class TritonPythonModel: 6 | 7 | def initialize(self, args): 8 | self.model_config = model_config = json.loads(args['model_config']) 9 | output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") 10 | output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") 11 | self.output0_dtype = pb_utils.triton_string_to_numpy(output0_config['data_type']) 12 | self.output1_dtype = pb_utils.triton_string_to_numpy(output1_config['data_type']) 13 | 14 | def execute(self, requests): 15 | output0_dtype = self.output0_dtype 16 | output1_dtype = self.output1_dtype 17 | responses = [] 18 | for request in requests: 19 | in_0 = pb_utils.get_input_tensor_by_name(request, 'INPUT0') 20 | in_1 = pb_utils.get_input_tensor_by_name(request, 'INPUT1') 21 | out_0, out_1 = (in_0.as_numpy() + in_1.as_numpy(), 22 | in_0.as_numpy() - in_1.as_numpy()) 23 | out_tensor_0 = pb_utils.Tensor('OUTPUT0', out_0.astype(output0_dtype)) 24 | out_tensor_1 = pb_utils.Tensor('OUTPUT1', out_1.astype(output1_dtype)) 25 | inference_response = pb_utils.InferenceResponse(output_tensors=[out_tensor_0, out_tensor_1]) 26 | responses.append(inference_response) 27 | return responses 28 | 29 | def finalize(self): 30 | print('Cleaning up...') 31 | -------------------------------------------------------------------------------- /models/example_python/client.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tritonclient.http as httpclient 3 | 4 | 5 | if __name__ == '__main__': 6 | triton_client = httpclient.InferenceServerClient(url='127.0.0.1:8000') 7 | 8 | inputs = [] 9 | inputs.append(httpclient.InferInput('INPUT0', [4], "FP32")) 10 | inputs.append(httpclient.InferInput('INPUT1', [4], "FP32")) 11 | input_data0 = np.random.randn(4).astype(np.float32) 12 | input_data1 = np.random.randn(4).astype(np.float32) 13 | inputs[0].set_data_from_numpy(input_data0, binary_data=False) 14 | inputs[1].set_data_from_numpy(input_data1, binary_data=False) 15 | outputs = [] 16 | outputs.append(httpclient.InferRequestedOutput('OUTPUT0', binary_data=False)) 17 | outputs.append(httpclient.InferRequestedOutput('OUTPUT1', binary_data=False)) 18 | 19 | results = triton_client.infer('example_python', inputs=inputs, outputs=outputs) 20 | output_data0 = results.as_numpy('OUTPUT0') 21 | output_data1 = results.as_numpy('OUTPUT1') 22 | 23 | print(input_data0) 24 | print(input_data1) 25 | print(output_data0) 26 | print(output_data1) 27 | -------------------------------------------------------------------------------- /models/example_python/config.pbtxt: -------------------------------------------------------------------------------- 1 | name: "example_python" 2 | backend: "python" 3 | input [ 4 | { 5 | name: "INPUT0" 6 | data_type: TYPE_FP32 7 | dims: [ 4 ] 8 | } 9 | ] 10 | input [ 11 | { 12 | name: "INPUT1" 13 | data_type: TYPE_FP32 14 | dims: [ 4 ] 15 | } 16 | ] 17 | output [ 18 | { 19 | name: "OUTPUT0" 20 | data_type: TYPE_FP32 21 | dims: [ 4 ] 22 | } 23 | ] 24 | output [ 25 | { 26 | name: "OUTPUT1" 27 | data_type: TYPE_FP32 28 | dims: [ 4 ] 29 | } 30 | ] 31 | 32 | instance_group [ 33 | { 34 | count: 10 35 | kind: KIND_CPU 36 | } 37 | ] 38 | -------------------------------------------------------------------------------- /models/identity/1/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzk0/triton/9197448568c5ae964a1dc936624526e86ed05b4e/models/identity/1/.gitkeep -------------------------------------------------------------------------------- /models/identity/client.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tritonclient.http as httpclient 3 | 4 | 5 | if __name__ == '__main__': 6 | triton_client = httpclient.InferenceServerClient(url='127.0.0.1:8000') 7 | 8 | inputs = [] 9 | inputs.append(httpclient.InferInput('INPUT0', [1, 10], "FP32")) 10 | input_data0 = np.random.randn(1, 10).astype(np.float32) 11 | inputs[0].set_data_from_numpy(input_data0, binary_data=True) 12 | outputs = [] 13 | outputs.append(httpclient.InferRequestedOutput('OUTPUT0', binary_data=True)) 14 | 15 | results = triton_client.infer('identity', inputs=inputs, outputs=outputs) 16 | output_data0 = results.as_numpy('OUTPUT0') 17 | 18 | print(input_data0) 19 | print(output_data0) 20 | -------------------------------------------------------------------------------- /models/identity/config.pbtxt: -------------------------------------------------------------------------------- 1 | name: "identity" 2 | backend: "identity" 3 | max_batch_size: 1 4 | input [ 5 | { 6 | name: "INPUT0" 7 | data_type: TYPE_FP32 8 | dims: [ 10 ] 9 | } 10 | ] 11 | output [ 12 | { 13 | name: "OUTPUT0" 14 | data_type: TYPE_FP32 15 | dims: [ 10 ] 16 | } 17 | ] 18 | 19 | instance_group [ 20 | { 21 | count: 1 22 | kind: KIND_CPU 23 | } 24 | ] 25 | 26 | -------------------------------------------------------------------------------- /models/resnet50_dali/config.pbtxt: -------------------------------------------------------------------------------- 1 | name: "resnet50_dali" 2 | backend: "dali" 3 | max_batch_size: 128 4 | input [ 5 | { 6 | name: "DALI_INPUT_0" 7 | data_type: TYPE_FP32 8 | dims: [ -1, -1, 3 ] 9 | } 10 | ] 11 | 12 | output [ 13 | { 14 | name: "DALI_OUTPUT_0" 15 | data_type: TYPE_FP32 16 | dims: [ 3, 224, 224 ] 17 | } 18 | ] 19 | instance_group [ 20 | { 21 | count: 1 22 | kind: KIND_GPU 23 | gpus: [ 0 ] 24 | } 25 | ] 26 | -------------------------------------------------------------------------------- /models/resnet50_dali/dali_client.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tritonclient.http as httpclient 3 | import torch 4 | from PIL import Image 5 | 6 | 7 | if __name__ == '__main__': 8 | triton_client = httpclient.InferenceServerClient(url='172.17.0.2:8000') 9 | 10 | image = Image.open('../resources/images/cat.jpg') 11 | image = np.asarray(image) 12 | image = np.expand_dims(image, axis=0) 13 | image = image.astype(np.float32) 14 | 15 | inputs = [] 16 | inputs.append(httpclient.InferInput('DALI_INPUT_0', image.shape, "FP32")) 17 | inputs[0].set_data_from_numpy(image, binary_data=False) 18 | outputs = [] 19 | outputs.append(httpclient.InferRequestedOutput('DALI_OUTPUT_0', binary_data=False)) 20 | 21 | results = triton_client.infer('resnet50_dali', inputs=inputs, outputs=outputs) 22 | output_data0 = results.as_numpy('DALI_OUTPUT_0') 23 | print(output_data0.shape) 24 | -------------------------------------------------------------------------------- /models/resnet50_dali/dali_pipeline.py: -------------------------------------------------------------------------------- 1 | import nvidia.dali as dali 2 | import nvidia.dali.fn as fn 3 | import multiprocess as mp 4 | 5 | @dali.pipeline_def(batch_size=128, num_threads=4, device_id=0) 6 | def pipeline(): 7 | images = fn.external_source(device='cpu', name='DALI_INPUT_0') 8 | images = fn.resize(images, resize_x=224, resize_y=224) 9 | images = fn.transpose(images, perm=[2, 0, 1]) 10 | images = images / 255 11 | return images 12 | 13 | 14 | pipeline().serialize(filename='./1/model.dali') 15 | -------------------------------------------------------------------------------- /models/resnet50_ensemble/1/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzk0/triton/9197448568c5ae964a1dc936624526e86ed05b4e/models/resnet50_ensemble/1/.gitkeep -------------------------------------------------------------------------------- /models/resnet50_ensemble/config.pbtxt: -------------------------------------------------------------------------------- 1 | name: "resnet50_ensemble" 2 | platform: "ensemble" 3 | max_batch_size: 128 4 | input [ 5 | { 6 | name: "ENSEMBLE_INPUT_0" 7 | data_type: TYPE_FP32 8 | dims: [ -1, -1, 3 ] 9 | } 10 | ] 11 | output [ 12 | { 13 | name: "ENSEMBLE_OUTPUT_0" 14 | data_type: TYPE_FP32 15 | dims: [ 1000 ] 16 | } 17 | ] 18 | ensemble_scheduling { 19 | step [ 20 | { 21 | model_name: "resnet50_dali" 22 | model_version: 1 23 | input_map: { 24 | key: "DALI_INPUT_0" 25 | value: "ENSEMBLE_INPUT_0" 26 | } 27 | output_map: { 28 | key: "DALI_OUTPUT_0" 29 | value: "preprocessed_image" 30 | } 31 | }, 32 | { 33 | model_name: "resnet50_pytorch" 34 | model_version: 1 35 | input_map: { 36 | key: "INPUT__0" 37 | value: "preprocessed_image" 38 | } 39 | output_map: { 40 | key: "OUTPUT__0" 41 | value: "ENSEMBLE_OUTPUT_0" 42 | } 43 | } 44 | ] 45 | } 46 | 47 | model_warmup [ 48 | { 49 | name: "random_input" 50 | batch_size: 1 51 | inputs: { 52 | key: "ENSEMBLE_INPUT_0" 53 | value: { 54 | data_type: TYPE_FP32 55 | dims: [768, 768, 3] 56 | zero_data: true 57 | } 58 | } 59 | } 60 | ] 61 | -------------------------------------------------------------------------------- /models/resnet50_ensemble/labels.txt: -------------------------------------------------------------------------------- 1 | TENCH 2 | GOLDFISH 3 | WHITE SHARK 4 | TIGER SHARK 5 | HAMMERHEAD SHARK 6 | ELECTRIC RAY 7 | STINGRAY 8 | ROOSTER 9 | HEN 10 | OSTRICH 11 | BRAMBLING 12 | GOLDFINCH 13 | HOUSE FINCH 14 | SNOWBIRD 15 | INDIGO FINCH 16 | ROBIN 17 | BULBUL 18 | JAY 19 | MAGPIE 20 | CHICKADEE 21 | WATER OUZEL 22 | KITE 23 | BALD EAGLE 24 | VULTURE 25 | GREAT GREY OWL 26 | FIRE SALAMANDER 27 | NEWT 28 | EFT 29 | SPOTTED SALAMANDER 30 | AXOLOTL 31 | BULL FROG 32 | TREE FROG 33 | TAILED FROG 34 | LOGGERHEAD 35 | LEATHERBACK TURTLE 36 | MUD TURTLE 37 | TERRAPIN 38 | BOX TURTLE 39 | BANDED GECKO 40 | COMMON IGUANA 41 | AMERICAN CHAMELEON 42 | WHIPTAIL 43 | AGAMA 44 | FRILLED LIZARD 45 | ALLIGATOR LIZARD 46 | GILA MONSTER 47 | GREEN LIZARD 48 | AFRICAN CHAMELEON 49 | KOMODO DRAGON 50 | AFRICAN CROCODILE 51 | AMERICAN ALLIGATOR 52 | TRICERATOPS 53 | THUNDER SNAKE 54 | RINGNECK SNAKE 55 | HOGNOSE SNAKE 56 | GREEN SNAKE 57 | KING SNAKE 58 | GARTER SNAKE 59 | WATER SNAKE 60 | VINE SNAKE 61 | NIGHT SNAKE 62 | BOA 63 | ROCK PYTHON 64 | COBRA 65 | GREEN MAMBA 66 | SEA SNAKE 67 | HORNED VIPER 68 | DIAMONDBACK 69 | SIDEWINDER 70 | TRILOBITE 71 | HARVESTMAN 72 | SCORPION 73 | GARDEN SPIDER 74 | BARN SPIDER 75 | GARDEN SPIDER 76 | BLACK WIDOW 77 | TARANTULA 78 | WOLF SPIDER 79 | TICK 80 | CENTIPEDE 81 | GROUSE 82 | PTARMIGAN 83 | RUFFED GROUSE 84 | PRAIRIE CHICKEN 85 | PEACOCK 86 | QUAIL 87 | PARTRIDGE 88 | AFRICAN GREY 89 | MACAW 90 | COCKATOO 91 | LORIKEET 92 | COUCAL 93 | BEE EATER 94 | HORNBILL 95 | HUMMINGBIRD 96 | JACAMAR 97 | TOUCAN 98 | DRAKE 99 | MERGANSER 100 | GOOSE 101 | BLACK SWAN 102 | TUSKER 103 | ECHIDNA 104 | PLATYPUS 105 | WALLABY 106 | KOALA 107 | WOMBAT 108 | JELLYFISH 109 | SEA ANEMONE 110 | BRAIN CORAL 111 | FLATWORM 112 | NEMATODE 113 | CONCH 114 | SNAIL 115 | SLUG 116 | SEA SLUG 117 | CHITON 118 | CHAMBERED NAUTILUS 119 | DUNGENESS CRAB 120 | ROCK CRAB 121 | FIDDLER CRAB 122 | KING CRAB 123 | AMERICAN LOBSTER 124 | SPINY LOBSTER 125 | CRAYFISH 126 | HERMIT CRAB 127 | ISOPOD 128 | WHITE STORK 129 | BLACK STORK 130 | SPOONBILL 131 | FLAMINGO 132 | LITTLE BLUE HERON 133 | AMERICAN EGRET 134 | BITTERN 135 | CRANE 136 | LIMPKIN 137 | EUROPEAN GALLINULE 138 | AMERICAN COOT 139 | BUSTARD 140 | RUDDY TURNSTONE 141 | RED-BACKED SANDPIPER 142 | REDSHANK 143 | DOWITCHER 144 | OYSTERCATCHER 145 | PELICAN 146 | KING PENGUIN 147 | ALBATROSS 148 | GREY WHALE 149 | KILLER WHALE 150 | DUGONG 151 | SEA LION 152 | CHIHUAHUA 153 | JAPANESE SPANIEL 154 | MALTESE DOG 155 | PEKINESE 156 | SHIH-TZU 157 | BLENHEIM SPANIEL 158 | PAPILLON 159 | TOY TERRIER 160 | RHODESIAN RIDGEBACK 161 | AFGHAN HOUND 162 | BASSET 163 | BEAGLE 164 | BLOODHOUND 165 | BLUETICK 166 | COONHOUND 167 | WALKER HOUND 168 | ENGLISH FOXHOUND 169 | REDBONE 170 | BORZOI 171 | IRISH WOLFHOUND 172 | ITALIAN GREYHOUND 173 | WHIPPET 174 | IBIZAN HOUND 175 | NORWEGIAN ELKHOUND 176 | OTTERHOUND 177 | SALUKI 178 | SCOTTISH DEERHOUND 179 | WEIMARANER 180 | STAFFORDSHIRE BULLTERRIER 181 | STAFFORDSHIRE TERRIER 182 | BEDLINGTON TERRIER 183 | BORDER TERRIER 184 | KERRY BLUE TERRIER 185 | IRISH TERRIER 186 | NORFOLK TERRIER 187 | NORWICH TERRIER 188 | YORKSHIRE TERRIER 189 | WIRE-HAIRED FOX TERRIER 190 | LAKELAND TERRIER 191 | SEALYHAM TERRIER 192 | AIREDALE 193 | CAIRN 194 | AUSTRALIAN TERRIER 195 | DANDIE DINMONT 196 | BOSTON BULL 197 | MINIATURE SCHNAUZER 198 | GIANT SCHNAUZER 199 | STANDARD SCHNAUZER 200 | SCOTCH TERRIER 201 | TIBETAN TERRIER 202 | SILKY TERRIER 203 | WHEATEN TERRIER 204 | WHITE TERRIER 205 | LHASA 206 | RETRIEVER 207 | CURLY-COATED RETRIEVER 208 | GOLDEN RETRIEVER 209 | LABRADOR RETRIEVER 210 | CHESAPEAKE BAY RETRIEVER 211 | SHORT-HAIRED POINTER 212 | VISLA 213 | ENGLISH SETTER 214 | IRISH SETTER 215 | GORDON SETTER 216 | BRITTANY SPANIEL 217 | CLUMBER 218 | ENGLISH SPRINGER 219 | WELSH SPRINGER SPANIEL 220 | COCKER SPANIEL 221 | SUSSEX SPANIEL 222 | IRISH WATERSPANIEL 223 | KUVASZ 224 | SCHIPPERKE 225 | GROENENDAEL 226 | MALINOIS 227 | BRIARD 228 | KELPIE 229 | KOMONDOR 230 | OLD ENGLISH SHEEPDOG 231 | SHETLAND SHEEPDOG 232 | COLLIE 233 | BORDER COLLIE 234 | BOUVIER DES FLANDRES 235 | ROTTWEILER 236 | GERMAN SHEPHERD 237 | DOBERMAN 238 | MINIATURE PINSCHER 239 | GREATER SWISS MOUNTAIN DOG 240 | BERNESE MOUNTAIN DOG 241 | APPENZELLER 242 | ENTLEBUCHER 243 | BOXER 244 | BULL MASTIFF 245 | TIBETAN MASTIFF 246 | FRENCH BULLDOG 247 | GREAT DANE 248 | SAINT BERNARD 249 | ESKIMO DOG 250 | MALAMUTE 251 | SIBERIAN HUSKY 252 | DALMATIAN 253 | AFFENPINSCHER 254 | BASENJI 255 | PUG 256 | LEONBERG 257 | NEWFOUNDLAND 258 | GREAT PYRENEES 259 | SAMOYED 260 | POMERANIAN 261 | CHOW 262 | KEESHOND 263 | BRABANCON GRIFFON 264 | PEMBROKE 265 | CARDIGAN 266 | TOY POODLE 267 | MINIATURE POODLE 268 | STANDARD POODLE 269 | MEXICAN HAIRLESS 270 | TIMBER WOLF 271 | WHITE WOLF 272 | RED WOLF 273 | COYOTE 274 | DINGO 275 | DHOLE 276 | AFRICAN HUNTING DOG 277 | HYENA 278 | RED FOX 279 | KIT FOX 280 | ARCTIC FOX 281 | GREY FOX 282 | TABBY 283 | TIGER CAT 284 | PERSIAN CAT 285 | SIAMESE CAT 286 | EGYPTIAN CAT 287 | COUGAR 288 | LYNX 289 | LEOPARD 290 | SNOW LEOPARD 291 | JAGUAR 292 | LION 293 | TIGER 294 | CHEETAH 295 | BROWN BEAR 296 | AMERICAN BLACK BEAR 297 | ICE BEAR 298 | SLOTH BEAR 299 | MONGOOSE 300 | MEERKAT 301 | TIGER BEETLE 302 | LADYBUG 303 | GROUND BEETLE 304 | LONG-HORNED BEETLE 305 | LEAF BEETLE 306 | DUNG BEETLE 307 | RHINOCEROS BEETLE 308 | WEEVIL 309 | FLY 310 | BEE 311 | ANT 312 | GRASSHOPPER 313 | CRICKET 314 | WALKING STICK 315 | COCKROACH 316 | MANTIS 317 | CICADA 318 | LEAFHOPPER 319 | LACEWING 320 | DRAGONFLY 321 | DAMSELFLY 322 | ADMIRAL 323 | RINGLET 324 | MONARCH 325 | CABBAGE BUTTERFLY 326 | SULPHUR BUTTERFLY 327 | LYCAENID 328 | STARFISH 329 | SEA URCHIN 330 | SEA CUCUMBER 331 | WOOD RABBIT 332 | HARE 333 | ANGORA 334 | HAMSTER 335 | PORCUPINE 336 | FOX SQUIRREL 337 | MARMOT 338 | BEAVER 339 | GUINEA PIG 340 | SORREL 341 | ZEBRA 342 | HOG 343 | WILD BOAR 344 | WARTHOG 345 | HIPPOPOTAMUS 346 | OX 347 | WATER BUFFALO 348 | BISON 349 | RAM 350 | BIGHORN 351 | IBEX 352 | HARTEBEEST 353 | IMPALA 354 | GAZELLE 355 | ARABIAN CAMEL 356 | LLAMA 357 | WEASEL 358 | MINK 359 | POLECAT 360 | BLACK-FOOTED FERRET 361 | OTTER 362 | SKUNK 363 | BADGER 364 | ARMADILLO 365 | THREE-TOED SLOTH 366 | ORANGUTAN 367 | GORILLA 368 | CHIMPANZEE 369 | GIBBON 370 | SIAMANG 371 | GUENON 372 | PATAS 373 | BABOON 374 | MACAQUE 375 | LANGUR 376 | COLOBUS 377 | PROBOSCIS MONKEY 378 | MARMOSET 379 | CAPUCHIN 380 | HOWLER MONKEY 381 | TITI 382 | SPIDER MONKEY 383 | SQUIRREL MONKEY 384 | MADAGASCAR CAT 385 | INDRI 386 | INDIAN ELEPHANT 387 | AFRICAN ELEPHANT 388 | LESSER PANDA 389 | GIANT PANDA 390 | BARRACOUTA 391 | EEL 392 | COHO 393 | ROCK BEAUTY 394 | ANEMONE FISH 395 | STURGEON 396 | GAR 397 | LIONFISH 398 | PUFFER 399 | ABACUS 400 | ABAYA 401 | ACADEMIC GOWN 402 | ACCORDION 403 | ACOUSTIC GUITAR 404 | AIRCRAFT CARRIER 405 | AIRLINER 406 | AIRSHIP 407 | ALTAR 408 | AMBULANCE 409 | AMPHIBIAN 410 | ANALOG CLOCK 411 | APIARY 412 | APRON 413 | ASHCAN 414 | ASSAULT RIFLE 415 | BACKPACK 416 | BAKERY 417 | BALANCE BEAM 418 | BALLOON 419 | BALLPOINT 420 | BAND AID 421 | BANJO 422 | BANNISTER 423 | BARBELL 424 | BARBER CHAIR 425 | BARBERSHOP 426 | BARN 427 | BAROMETER 428 | BARREL 429 | BARROW 430 | BASEBALL 431 | BASKETBALL 432 | BASSINET 433 | BASSOON 434 | BATHING CAP 435 | BATH TOWEL 436 | BATHTUB 437 | BEACH WAGON 438 | BEACON 439 | BEAKER 440 | BEARSKIN 441 | BEER BOTTLE 442 | BEER GLASS 443 | BELL COTE 444 | BIB 445 | BICYCLE-BUILT-FOR-TWO 446 | BIKINI 447 | BINDER 448 | BINOCULARS 449 | BIRDHOUSE 450 | BOATHOUSE 451 | BOBSLED 452 | BOLO TIE 453 | BONNET 454 | BOOKCASE 455 | BOOKSHOP 456 | BOTTLECAP 457 | BOW 458 | BOW TIE 459 | BRASS 460 | BRASSIERE 461 | BREAKWATER 462 | BREASTPLATE 463 | BROOM 464 | BUCKET 465 | BUCKLE 466 | BULLETPROOF VEST 467 | BULLET TRAIN 468 | BUTCHER SHOP 469 | CAB 470 | CALDRON 471 | CANDLE 472 | CANNON 473 | CANOE 474 | CAN OPENER 475 | CARDIGAN 476 | CAR MIRROR 477 | CAROUSEL 478 | CARPENTERS KIT 479 | CARTON 480 | CAR WHEEL 481 | CASH MACHINE 482 | CASSETTE 483 | CASSETTE PLAYER 484 | CASTLE 485 | CATAMARAN 486 | CD PLAYER 487 | CELLO 488 | CELLULAR TELEPHONE 489 | CHAIN 490 | CHAINLINK FENCE 491 | CHAIN MAIL 492 | CHAIN SAW 493 | CHEST 494 | CHIFFONIER 495 | CHIME 496 | CHINA CABINET 497 | CHRISTMAS STOCKING 498 | CHURCH 499 | CINEMA 500 | CLEAVER 501 | CLIFF DWELLING 502 | CLOAK 503 | CLOG 504 | COCKTAIL SHAKER 505 | COFFEE MUG 506 | COFFEEPOT 507 | COIL 508 | COMBINATION LOCK 509 | COMPUTER KEYBOARD 510 | CONFECTIONERY 511 | CONTAINER SHIP 512 | CONVERTIBLE 513 | CORKSCREW 514 | CORNET 515 | COWBOY BOOT 516 | COWBOY HAT 517 | CRADLE 518 | CRANE 519 | CRASH HELMET 520 | CRATE 521 | CRIB 522 | CROCK POT 523 | CROQUET BALL 524 | CRUTCH 525 | CUIRASS 526 | DAM 527 | DESK 528 | DESKTOP COMPUTER 529 | DIAL TELEPHONE 530 | DIAPER 531 | DIGITAL CLOCK 532 | DIGITAL WATCH 533 | DINING TABLE 534 | DISHRAG 535 | DISHWASHER 536 | DISK BRAKE 537 | DOCK 538 | DOGSLED 539 | DOME 540 | DOORMAT 541 | DRILLING PLATFORM 542 | DRUM 543 | DRUMSTICK 544 | DUMBBELL 545 | DUTCH OVEN 546 | ELECTRIC FAN 547 | ELECTRIC GUITAR 548 | ELECTRIC LOCOMOTIVE 549 | ENTERTAINMENT CENTER 550 | ENVELOPE 551 | ESPRESSO MAKER 552 | FACE POWDER 553 | FEATHER BOA 554 | FILE 555 | FIREBOAT 556 | FIRE ENGINE 557 | FIRE SCREEN 558 | FLAGPOLE 559 | FLUTE 560 | FOLDING CHAIR 561 | FOOTBALL HELMET 562 | FORKLIFT 563 | FOUNTAIN 564 | FOUNTAIN PEN 565 | FOUR-POSTER 566 | FREIGHT CAR 567 | FRENCH HORN 568 | FRYING PAN 569 | FUR COAT 570 | GARBAGE TRUCK 571 | GASMASK 572 | GAS PUMP 573 | GOBLET 574 | GO-KART 575 | GOLF BALL 576 | GOLFCART 577 | GONDOLA 578 | GONG 579 | GOWN 580 | GRAND PIANO 581 | GREENHOUSE 582 | GRILLE 583 | GROCERY STORE 584 | GUILLOTINE 585 | HAIR SLIDE 586 | HAIR SPRAY 587 | HALF TRACK 588 | HAMMER 589 | HAMPER 590 | HAND BLOWER 591 | HAND-HELD COMPUTER 592 | HANDKERCHIEF 593 | HARD DISC 594 | HARMONICA 595 | HARP 596 | HARVESTER 597 | HATCHET 598 | HOLSTER 599 | HOME THEATER 600 | HONEYCOMB 601 | HOOK 602 | HOOPSKIRT 603 | HORIZONTAL BAR 604 | HORSE CART 605 | HOURGLASS 606 | IPOD 607 | IRON 608 | JACK-O-LANTERN 609 | JEAN 610 | JEEP 611 | JERSEY 612 | JIGSAW PUZZLE 613 | JINRIKISHA 614 | JOYSTICK 615 | KIMONO 616 | KNEE PAD 617 | KNOT 618 | LAB COAT 619 | LADLE 620 | LAMPSHADE 621 | LAPTOP 622 | LAWN MOWER 623 | LENS CAP 624 | LETTER OPENER 625 | LIBRARY 626 | LIFEBOAT 627 | LIGHTER 628 | LIMOUSINE 629 | LINER 630 | LIPSTICK 631 | LOAFER 632 | LOTION 633 | LOUDSPEAKER 634 | LOUPE 635 | LUMBERMILL 636 | MAGNETIC COMPASS 637 | MAILBAG 638 | MAILBOX 639 | MAILLOT 640 | MAILLOT 641 | MANHOLE COVER 642 | MARACA 643 | MARIMBA 644 | MASK 645 | MATCHSTICK 646 | MAYPOLE 647 | MAZE 648 | MEASURING CUP 649 | MEDICINE CHEST 650 | MEGALITH 651 | MICROPHONE 652 | MICROWAVE 653 | MILITARY UNIFORM 654 | MILK CAN 655 | MINIBUS 656 | MINISKIRT 657 | MINIVAN 658 | MISSILE 659 | MITTEN 660 | MIXING BOWL 661 | MOBILE HOME 662 | MODEL T 663 | MODEM 664 | MONASTERY 665 | MONITOR 666 | MOPED 667 | MORTAR 668 | MORTARBOARD 669 | MOSQUE 670 | MOSQUITO NET 671 | MOTOR SCOOTER 672 | MOUNTAIN BIKE 673 | MOUNTAIN TENT 674 | MOUSE 675 | MOUSETRAP 676 | MOVING VAN 677 | MUZZLE 678 | NAIL 679 | NECK BRACE 680 | NECKLACE 681 | NIPPLE 682 | NOTEBOOK 683 | OBELISK 684 | OBOE 685 | OCARINA 686 | ODOMETER 687 | OIL FILTER 688 | ORGAN 689 | OSCILLOSCOPE 690 | OVERSKIRT 691 | OXCART 692 | OXYGEN MASK 693 | PACKET 694 | PADDLE 695 | PADDLEWHEEL 696 | PADLOCK 697 | PAINTBRUSH 698 | PAJAMA 699 | PALACE 700 | PANPIPE 701 | PAPER TOWEL 702 | PARACHUTE 703 | PARALLEL BARS 704 | PARK BENCH 705 | PARKING METER 706 | PASSENGER CAR 707 | PATIO 708 | PAY-PHONE 709 | PEDESTAL 710 | PENCIL BOX 711 | PENCIL SHARPENER 712 | PERFUME 713 | PETRI DISH 714 | PHOTOCOPIER 715 | PICK 716 | PICKELHAUBE 717 | PICKET FENCE 718 | PICKUP 719 | PIER 720 | PIGGY BANK 721 | PILL BOTTLE 722 | PILLOW 723 | PING-PONG BALL 724 | PINWHEEL 725 | PIRATE 726 | PITCHER 727 | PLANE 728 | PLANETARIUM 729 | PLASTIC BAG 730 | PLATE RACK 731 | PLOW 732 | PLUNGER 733 | POLAROID CAMERA 734 | POLE 735 | POLICE VAN 736 | PONCHO 737 | POOL TABLE 738 | POP BOTTLE 739 | POT 740 | POTTERS WHEEL 741 | POWER DRILL 742 | PRAYER RUG 743 | PRINTER 744 | PRISON 745 | PROJECTILE 746 | PROJECTOR 747 | PUCK 748 | PUNCHING BAG 749 | PURSE 750 | QUILL 751 | QUILT 752 | RACER 753 | RACKET 754 | RADIATOR 755 | RADIO 756 | RADIO TELESCOPE 757 | RAIN BARREL 758 | RECREATIONAL VEHICLE 759 | REEL 760 | REFLEX CAMERA 761 | REFRIGERATOR 762 | REMOTE CONTROL 763 | RESTAURANT 764 | REVOLVER 765 | RIFLE 766 | ROCKING CHAIR 767 | ROTISSERIE 768 | RUBBER ERASER 769 | RUGBY BALL 770 | RULE 771 | RUNNING SHOE 772 | SAFE 773 | SAFETY PIN 774 | SALTSHAKER 775 | SANDAL 776 | SARONG 777 | SAX 778 | SCABBARD 779 | SCALE 780 | SCHOOL BUS 781 | SCHOONER 782 | SCOREBOARD 783 | SCREEN 784 | SCREW 785 | SCREWDRIVER 786 | SEAT BELT 787 | SEWING MACHINE 788 | SHIELD 789 | SHOE SHOP 790 | SHOJI 791 | SHOPPING BASKET 792 | SHOPPING CART 793 | SHOVEL 794 | SHOWER CAP 795 | SHOWER CURTAIN 796 | SKI 797 | SKI MASK 798 | SLEEPING BAG 799 | SLIDE RULE 800 | SLIDING DOOR 801 | SLOT 802 | SNORKEL 803 | SNOWMOBILE 804 | SNOWPLOW 805 | SOAP DISPENSER 806 | SOCCER BALL 807 | SOCK 808 | SOLAR DISH 809 | SOMBRERO 810 | SOUP BOWL 811 | SPACE BAR 812 | SPACE HEATER 813 | SPACE SHUTTLE 814 | SPATULA 815 | SPEEDBOAT 816 | SPIDER WEB 817 | SPINDLE 818 | SPORTS CAR 819 | SPOTLIGHT 820 | STAGE 821 | STEAM LOCOMOTIVE 822 | STEEL ARCH BRIDGE 823 | STEEL DRUM 824 | STETHOSCOPE 825 | STOLE 826 | STONE WALL 827 | STOPWATCH 828 | STOVE 829 | STRAINER 830 | STREETCAR 831 | STRETCHER 832 | STUDIO COUCH 833 | STUPA 834 | SUBMARINE 835 | SUIT 836 | SUNDIAL 837 | SUNGLASS 838 | SUNGLASSES 839 | SUNSCREEN 840 | SUSPENSION BRIDGE 841 | SWAB 842 | SWEATSHIRT 843 | SWIMMING TRUNKS 844 | SWING 845 | SWITCH 846 | SYRINGE 847 | TABLE LAMP 848 | TANK 849 | TAPE PLAYER 850 | TEAPOT 851 | TEDDY 852 | TELEVISION 853 | TENNIS BALL 854 | THATCH 855 | THEATER CURTAIN 856 | THIMBLE 857 | THRESHER 858 | THRONE 859 | TILE ROOF 860 | TOASTER 861 | TOBACCO SHOP 862 | TOILET SEAT 863 | TORCH 864 | TOTEM POLE 865 | TOW TRUCK 866 | TOYSHOP 867 | TRACTOR 868 | TRAILER TRUCK 869 | TRAY 870 | TRENCH COAT 871 | TRICYCLE 872 | TRIMARAN 873 | TRIPOD 874 | TRIUMPHAL ARCH 875 | TROLLEYBUS 876 | TROMBONE 877 | TUB 878 | TURNSTILE 879 | TYPEWRITER KEYBOARD 880 | UMBRELLA 881 | UNICYCLE 882 | UPRIGHT 883 | VACUUM 884 | VASE 885 | VAULT 886 | VELVET 887 | VENDING MACHINE 888 | VESTMENT 889 | VIADUCT 890 | VIOLIN 891 | VOLLEYBALL 892 | WAFFLE IRON 893 | WALL CLOCK 894 | WALLET 895 | WARDROBE 896 | WARPLANE 897 | WASHBASIN 898 | WASHER 899 | WATER BOTTLE 900 | WATER JUG 901 | WATER TOWER 902 | WHISKEY JUG 903 | WHISTLE 904 | WIG 905 | WINDOW SCREEN 906 | WINDOW SHADE 907 | WINDSOR TIE 908 | WINE BOTTLE 909 | WING 910 | WOK 911 | WOODEN SPOON 912 | WOOL 913 | WORM FENCE 914 | WRECK 915 | YAWL 916 | YURT 917 | WEB SITE 918 | COMIC BOOK 919 | CROSSWORD PUZZLE 920 | STREET SIGN 921 | TRAFFIC LIGHT 922 | BOOK JACKET 923 | MENU 924 | PLATE 925 | GUACAMOLE 926 | CONSOMME 927 | HOT POT 928 | TRIFLE 929 | ICE CREAM 930 | ICE LOLLY 931 | FRENCH LOAF 932 | BAGEL 933 | PRETZEL 934 | CHEESEBURGER 935 | HOTDOG 936 | MASHED POTATO 937 | HEAD CABBAGE 938 | BROCCOLI 939 | CAULIFLOWER 940 | ZUCCHINI 941 | SPAGHETTI SQUASH 942 | ACORN SQUASH 943 | BUTTERNUT SQUASH 944 | CUCUMBER 945 | ARTICHOKE 946 | BELL PEPPER 947 | CARDOON 948 | MUSHROOM 949 | GRANNY SMITH 950 | STRAWBERRY 951 | ORANGE 952 | LEMON 953 | FIG 954 | PINEAPPLE 955 | BANANA 956 | JACKFRUIT 957 | CUSTARD APPLE 958 | POMEGRANATE 959 | HAY 960 | CARBONARA 961 | CHOCOLATE SAUCE 962 | DOUGH 963 | MEAT LOAF 964 | PIZZA 965 | POTPIE 966 | BURRITO 967 | RED WINE 968 | ESPRESSO 969 | CUP 970 | EGGNOG 971 | ALP 972 | BUBBLE 973 | CLIFF 974 | CORAL REEF 975 | GEYSER 976 | LAKESIDE 977 | PROMONTORY 978 | SANDBAR 979 | SEASHORE 980 | VALLEY 981 | VOLCANO 982 | BALLPLAYER 983 | GROOM 984 | SCUBA DIVER 985 | RAPESEED 986 | DAISY 987 | LADY SLIPPER 988 | CORN 989 | ACORN 990 | HIP 991 | BUCKEYE 992 | CORAL FUNGUS 993 | AGARIC 994 | GYROMITRA 995 | STINKHORN 996 | EARTHSTAR 997 | HEN-OF-THE-WOODS 998 | BOLETE 999 | EAR 1000 | TOILET TISSUE -------------------------------------------------------------------------------- /models/resnet50_ensemble/resnet_ensemble.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tritonclient.http as httpclient 3 | import torch 4 | from PIL import Image 5 | 6 | 7 | if __name__ == '__main__': 8 | triton_client = httpclient.InferenceServerClient(url='127.0.0.1:8000') 9 | 10 | image = Image.open('../resources/images/cat.jpg') 11 | image = np.asarray(image) 12 | image = np.expand_dims(image, axis=0) 13 | image = image.astype(np.float32) 14 | 15 | inputs = [] 16 | inputs.append(httpclient.InferInput('ENSEMBLE_INPUT_0', image.shape, "FP32")) 17 | inputs[0].set_data_from_numpy(image, binary_data=False) 18 | outputs = [] 19 | outputs.append(httpclient.InferRequestedOutput('ENSEMBLE_OUTPUT_0', binary_data=False, class_count=10)) 20 | 21 | results = triton_client.infer('resnet50_ensemble', inputs=inputs, outputs=outputs) 22 | output_data0 = results.as_numpy('ENSEMBLE_OUTPUT_0') 23 | print(output_data0.shape) 24 | print(output_data0) 25 | -------------------------------------------------------------------------------- /models/resnet50_pytorch/config.pbtxt: -------------------------------------------------------------------------------- 1 | name: "resnet50_pytorch" 2 | platform: "pytorch_libtorch" 3 | max_batch_size: 128 4 | input [ 5 | { 6 | name: "INPUT__0" 7 | data_type: TYPE_FP32 8 | dims: [ 3, -1, -1 ] 9 | } 10 | ] 11 | output [ 12 | { 13 | name: "OUTPUT__0" 14 | data_type: TYPE_FP32 15 | dims: [ 1000 ] 16 | label_filename: "labels.txt" 17 | } 18 | ] 19 | instance_group [ 20 | { 21 | count: 1 22 | kind: KIND_GPU 23 | } 24 | ] 25 | -------------------------------------------------------------------------------- /models/resnet50_pytorch/labels.txt: -------------------------------------------------------------------------------- 1 | TENCH 2 | GOLDFISH 3 | WHITE SHARK 4 | TIGER SHARK 5 | HAMMERHEAD SHARK 6 | ELECTRIC RAY 7 | STINGRAY 8 | ROOSTER 9 | HEN 10 | OSTRICH 11 | BRAMBLING 12 | GOLDFINCH 13 | HOUSE FINCH 14 | SNOWBIRD 15 | INDIGO FINCH 16 | ROBIN 17 | BULBUL 18 | JAY 19 | MAGPIE 20 | CHICKADEE 21 | WATER OUZEL 22 | KITE 23 | BALD EAGLE 24 | VULTURE 25 | GREAT GREY OWL 26 | FIRE SALAMANDER 27 | NEWT 28 | EFT 29 | SPOTTED SALAMANDER 30 | AXOLOTL 31 | BULL FROG 32 | TREE FROG 33 | TAILED FROG 34 | LOGGERHEAD 35 | LEATHERBACK TURTLE 36 | MUD TURTLE 37 | TERRAPIN 38 | BOX TURTLE 39 | BANDED GECKO 40 | COMMON IGUANA 41 | AMERICAN CHAMELEON 42 | WHIPTAIL 43 | AGAMA 44 | FRILLED LIZARD 45 | ALLIGATOR LIZARD 46 | GILA MONSTER 47 | GREEN LIZARD 48 | AFRICAN CHAMELEON 49 | KOMODO DRAGON 50 | AFRICAN CROCODILE 51 | AMERICAN ALLIGATOR 52 | TRICERATOPS 53 | THUNDER SNAKE 54 | RINGNECK SNAKE 55 | HOGNOSE SNAKE 56 | GREEN SNAKE 57 | KING SNAKE 58 | GARTER SNAKE 59 | WATER SNAKE 60 | VINE SNAKE 61 | NIGHT SNAKE 62 | BOA 63 | ROCK PYTHON 64 | COBRA 65 | GREEN MAMBA 66 | SEA SNAKE 67 | HORNED VIPER 68 | DIAMONDBACK 69 | SIDEWINDER 70 | TRILOBITE 71 | HARVESTMAN 72 | SCORPION 73 | GARDEN SPIDER 74 | BARN SPIDER 75 | GARDEN SPIDER 76 | BLACK WIDOW 77 | TARANTULA 78 | WOLF SPIDER 79 | TICK 80 | CENTIPEDE 81 | GROUSE 82 | PTARMIGAN 83 | RUFFED GROUSE 84 | PRAIRIE CHICKEN 85 | PEACOCK 86 | QUAIL 87 | PARTRIDGE 88 | AFRICAN GREY 89 | MACAW 90 | COCKATOO 91 | LORIKEET 92 | COUCAL 93 | BEE EATER 94 | HORNBILL 95 | HUMMINGBIRD 96 | JACAMAR 97 | TOUCAN 98 | DRAKE 99 | MERGANSER 100 | GOOSE 101 | BLACK SWAN 102 | TUSKER 103 | ECHIDNA 104 | PLATYPUS 105 | WALLABY 106 | KOALA 107 | WOMBAT 108 | JELLYFISH 109 | SEA ANEMONE 110 | BRAIN CORAL 111 | FLATWORM 112 | NEMATODE 113 | CONCH 114 | SNAIL 115 | SLUG 116 | SEA SLUG 117 | CHITON 118 | CHAMBERED NAUTILUS 119 | DUNGENESS CRAB 120 | ROCK CRAB 121 | FIDDLER CRAB 122 | KING CRAB 123 | AMERICAN LOBSTER 124 | SPINY LOBSTER 125 | CRAYFISH 126 | HERMIT CRAB 127 | ISOPOD 128 | WHITE STORK 129 | BLACK STORK 130 | SPOONBILL 131 | FLAMINGO 132 | LITTLE BLUE HERON 133 | AMERICAN EGRET 134 | BITTERN 135 | CRANE 136 | LIMPKIN 137 | EUROPEAN GALLINULE 138 | AMERICAN COOT 139 | BUSTARD 140 | RUDDY TURNSTONE 141 | RED-BACKED SANDPIPER 142 | REDSHANK 143 | DOWITCHER 144 | OYSTERCATCHER 145 | PELICAN 146 | KING PENGUIN 147 | ALBATROSS 148 | GREY WHALE 149 | KILLER WHALE 150 | DUGONG 151 | SEA LION 152 | CHIHUAHUA 153 | JAPANESE SPANIEL 154 | MALTESE DOG 155 | PEKINESE 156 | SHIH-TZU 157 | BLENHEIM SPANIEL 158 | PAPILLON 159 | TOY TERRIER 160 | RHODESIAN RIDGEBACK 161 | AFGHAN HOUND 162 | BASSET 163 | BEAGLE 164 | BLOODHOUND 165 | BLUETICK 166 | COONHOUND 167 | WALKER HOUND 168 | ENGLISH FOXHOUND 169 | REDBONE 170 | BORZOI 171 | IRISH WOLFHOUND 172 | ITALIAN GREYHOUND 173 | WHIPPET 174 | IBIZAN HOUND 175 | NORWEGIAN ELKHOUND 176 | OTTERHOUND 177 | SALUKI 178 | SCOTTISH DEERHOUND 179 | WEIMARANER 180 | STAFFORDSHIRE BULLTERRIER 181 | STAFFORDSHIRE TERRIER 182 | BEDLINGTON TERRIER 183 | BORDER TERRIER 184 | KERRY BLUE TERRIER 185 | IRISH TERRIER 186 | NORFOLK TERRIER 187 | NORWICH TERRIER 188 | YORKSHIRE TERRIER 189 | WIRE-HAIRED FOX TERRIER 190 | LAKELAND TERRIER 191 | SEALYHAM TERRIER 192 | AIREDALE 193 | CAIRN 194 | AUSTRALIAN TERRIER 195 | DANDIE DINMONT 196 | BOSTON BULL 197 | MINIATURE SCHNAUZER 198 | GIANT SCHNAUZER 199 | STANDARD SCHNAUZER 200 | SCOTCH TERRIER 201 | TIBETAN TERRIER 202 | SILKY TERRIER 203 | WHEATEN TERRIER 204 | WHITE TERRIER 205 | LHASA 206 | RETRIEVER 207 | CURLY-COATED RETRIEVER 208 | GOLDEN RETRIEVER 209 | LABRADOR RETRIEVER 210 | CHESAPEAKE BAY RETRIEVER 211 | SHORT-HAIRED POINTER 212 | VISLA 213 | ENGLISH SETTER 214 | IRISH SETTER 215 | GORDON SETTER 216 | BRITTANY SPANIEL 217 | CLUMBER 218 | ENGLISH SPRINGER 219 | WELSH SPRINGER SPANIEL 220 | COCKER SPANIEL 221 | SUSSEX SPANIEL 222 | IRISH WATERSPANIEL 223 | KUVASZ 224 | SCHIPPERKE 225 | GROENENDAEL 226 | MALINOIS 227 | BRIARD 228 | KELPIE 229 | KOMONDOR 230 | OLD ENGLISH SHEEPDOG 231 | SHETLAND SHEEPDOG 232 | COLLIE 233 | BORDER COLLIE 234 | BOUVIER DES FLANDRES 235 | ROTTWEILER 236 | GERMAN SHEPHERD 237 | DOBERMAN 238 | MINIATURE PINSCHER 239 | GREATER SWISS MOUNTAIN DOG 240 | BERNESE MOUNTAIN DOG 241 | APPENZELLER 242 | ENTLEBUCHER 243 | BOXER 244 | BULL MASTIFF 245 | TIBETAN MASTIFF 246 | FRENCH BULLDOG 247 | GREAT DANE 248 | SAINT BERNARD 249 | ESKIMO DOG 250 | MALAMUTE 251 | SIBERIAN HUSKY 252 | DALMATIAN 253 | AFFENPINSCHER 254 | BASENJI 255 | PUG 256 | LEONBERG 257 | NEWFOUNDLAND 258 | GREAT PYRENEES 259 | SAMOYED 260 | POMERANIAN 261 | CHOW 262 | KEESHOND 263 | BRABANCON GRIFFON 264 | PEMBROKE 265 | CARDIGAN 266 | TOY POODLE 267 | MINIATURE POODLE 268 | STANDARD POODLE 269 | MEXICAN HAIRLESS 270 | TIMBER WOLF 271 | WHITE WOLF 272 | RED WOLF 273 | COYOTE 274 | DINGO 275 | DHOLE 276 | AFRICAN HUNTING DOG 277 | HYENA 278 | RED FOX 279 | KIT FOX 280 | ARCTIC FOX 281 | GREY FOX 282 | TABBY 283 | TIGER CAT 284 | PERSIAN CAT 285 | SIAMESE CAT 286 | EGYPTIAN CAT 287 | COUGAR 288 | LYNX 289 | LEOPARD 290 | SNOW LEOPARD 291 | JAGUAR 292 | LION 293 | TIGER 294 | CHEETAH 295 | BROWN BEAR 296 | AMERICAN BLACK BEAR 297 | ICE BEAR 298 | SLOTH BEAR 299 | MONGOOSE 300 | MEERKAT 301 | TIGER BEETLE 302 | LADYBUG 303 | GROUND BEETLE 304 | LONG-HORNED BEETLE 305 | LEAF BEETLE 306 | DUNG BEETLE 307 | RHINOCEROS BEETLE 308 | WEEVIL 309 | FLY 310 | BEE 311 | ANT 312 | GRASSHOPPER 313 | CRICKET 314 | WALKING STICK 315 | COCKROACH 316 | MANTIS 317 | CICADA 318 | LEAFHOPPER 319 | LACEWING 320 | DRAGONFLY 321 | DAMSELFLY 322 | ADMIRAL 323 | RINGLET 324 | MONARCH 325 | CABBAGE BUTTERFLY 326 | SULPHUR BUTTERFLY 327 | LYCAENID 328 | STARFISH 329 | SEA URCHIN 330 | SEA CUCUMBER 331 | WOOD RABBIT 332 | HARE 333 | ANGORA 334 | HAMSTER 335 | PORCUPINE 336 | FOX SQUIRREL 337 | MARMOT 338 | BEAVER 339 | GUINEA PIG 340 | SORREL 341 | ZEBRA 342 | HOG 343 | WILD BOAR 344 | WARTHOG 345 | HIPPOPOTAMUS 346 | OX 347 | WATER BUFFALO 348 | BISON 349 | RAM 350 | BIGHORN 351 | IBEX 352 | HARTEBEEST 353 | IMPALA 354 | GAZELLE 355 | ARABIAN CAMEL 356 | LLAMA 357 | WEASEL 358 | MINK 359 | POLECAT 360 | BLACK-FOOTED FERRET 361 | OTTER 362 | SKUNK 363 | BADGER 364 | ARMADILLO 365 | THREE-TOED SLOTH 366 | ORANGUTAN 367 | GORILLA 368 | CHIMPANZEE 369 | GIBBON 370 | SIAMANG 371 | GUENON 372 | PATAS 373 | BABOON 374 | MACAQUE 375 | LANGUR 376 | COLOBUS 377 | PROBOSCIS MONKEY 378 | MARMOSET 379 | CAPUCHIN 380 | HOWLER MONKEY 381 | TITI 382 | SPIDER MONKEY 383 | SQUIRREL MONKEY 384 | MADAGASCAR CAT 385 | INDRI 386 | INDIAN ELEPHANT 387 | AFRICAN ELEPHANT 388 | LESSER PANDA 389 | GIANT PANDA 390 | BARRACOUTA 391 | EEL 392 | COHO 393 | ROCK BEAUTY 394 | ANEMONE FISH 395 | STURGEON 396 | GAR 397 | LIONFISH 398 | PUFFER 399 | ABACUS 400 | ABAYA 401 | ACADEMIC GOWN 402 | ACCORDION 403 | ACOUSTIC GUITAR 404 | AIRCRAFT CARRIER 405 | AIRLINER 406 | AIRSHIP 407 | ALTAR 408 | AMBULANCE 409 | AMPHIBIAN 410 | ANALOG CLOCK 411 | APIARY 412 | APRON 413 | ASHCAN 414 | ASSAULT RIFLE 415 | BACKPACK 416 | BAKERY 417 | BALANCE BEAM 418 | BALLOON 419 | BALLPOINT 420 | BAND AID 421 | BANJO 422 | BANNISTER 423 | BARBELL 424 | BARBER CHAIR 425 | BARBERSHOP 426 | BARN 427 | BAROMETER 428 | BARREL 429 | BARROW 430 | BASEBALL 431 | BASKETBALL 432 | BASSINET 433 | BASSOON 434 | BATHING CAP 435 | BATH TOWEL 436 | BATHTUB 437 | BEACH WAGON 438 | BEACON 439 | BEAKER 440 | BEARSKIN 441 | BEER BOTTLE 442 | BEER GLASS 443 | BELL COTE 444 | BIB 445 | BICYCLE-BUILT-FOR-TWO 446 | BIKINI 447 | BINDER 448 | BINOCULARS 449 | BIRDHOUSE 450 | BOATHOUSE 451 | BOBSLED 452 | BOLO TIE 453 | BONNET 454 | BOOKCASE 455 | BOOKSHOP 456 | BOTTLECAP 457 | BOW 458 | BOW TIE 459 | BRASS 460 | BRASSIERE 461 | BREAKWATER 462 | BREASTPLATE 463 | BROOM 464 | BUCKET 465 | BUCKLE 466 | BULLETPROOF VEST 467 | BULLET TRAIN 468 | BUTCHER SHOP 469 | CAB 470 | CALDRON 471 | CANDLE 472 | CANNON 473 | CANOE 474 | CAN OPENER 475 | CARDIGAN 476 | CAR MIRROR 477 | CAROUSEL 478 | CARPENTERS KIT 479 | CARTON 480 | CAR WHEEL 481 | CASH MACHINE 482 | CASSETTE 483 | CASSETTE PLAYER 484 | CASTLE 485 | CATAMARAN 486 | CD PLAYER 487 | CELLO 488 | CELLULAR TELEPHONE 489 | CHAIN 490 | CHAINLINK FENCE 491 | CHAIN MAIL 492 | CHAIN SAW 493 | CHEST 494 | CHIFFONIER 495 | CHIME 496 | CHINA CABINET 497 | CHRISTMAS STOCKING 498 | CHURCH 499 | CINEMA 500 | CLEAVER 501 | CLIFF DWELLING 502 | CLOAK 503 | CLOG 504 | COCKTAIL SHAKER 505 | COFFEE MUG 506 | COFFEEPOT 507 | COIL 508 | COMBINATION LOCK 509 | COMPUTER KEYBOARD 510 | CONFECTIONERY 511 | CONTAINER SHIP 512 | CONVERTIBLE 513 | CORKSCREW 514 | CORNET 515 | COWBOY BOOT 516 | COWBOY HAT 517 | CRADLE 518 | CRANE 519 | CRASH HELMET 520 | CRATE 521 | CRIB 522 | CROCK POT 523 | CROQUET BALL 524 | CRUTCH 525 | CUIRASS 526 | DAM 527 | DESK 528 | DESKTOP COMPUTER 529 | DIAL TELEPHONE 530 | DIAPER 531 | DIGITAL CLOCK 532 | DIGITAL WATCH 533 | DINING TABLE 534 | DISHRAG 535 | DISHWASHER 536 | DISK BRAKE 537 | DOCK 538 | DOGSLED 539 | DOME 540 | DOORMAT 541 | DRILLING PLATFORM 542 | DRUM 543 | DRUMSTICK 544 | DUMBBELL 545 | DUTCH OVEN 546 | ELECTRIC FAN 547 | ELECTRIC GUITAR 548 | ELECTRIC LOCOMOTIVE 549 | ENTERTAINMENT CENTER 550 | ENVELOPE 551 | ESPRESSO MAKER 552 | FACE POWDER 553 | FEATHER BOA 554 | FILE 555 | FIREBOAT 556 | FIRE ENGINE 557 | FIRE SCREEN 558 | FLAGPOLE 559 | FLUTE 560 | FOLDING CHAIR 561 | FOOTBALL HELMET 562 | FORKLIFT 563 | FOUNTAIN 564 | FOUNTAIN PEN 565 | FOUR-POSTER 566 | FREIGHT CAR 567 | FRENCH HORN 568 | FRYING PAN 569 | FUR COAT 570 | GARBAGE TRUCK 571 | GASMASK 572 | GAS PUMP 573 | GOBLET 574 | GO-KART 575 | GOLF BALL 576 | GOLFCART 577 | GONDOLA 578 | GONG 579 | GOWN 580 | GRAND PIANO 581 | GREENHOUSE 582 | GRILLE 583 | GROCERY STORE 584 | GUILLOTINE 585 | HAIR SLIDE 586 | HAIR SPRAY 587 | HALF TRACK 588 | HAMMER 589 | HAMPER 590 | HAND BLOWER 591 | HAND-HELD COMPUTER 592 | HANDKERCHIEF 593 | HARD DISC 594 | HARMONICA 595 | HARP 596 | HARVESTER 597 | HATCHET 598 | HOLSTER 599 | HOME THEATER 600 | HONEYCOMB 601 | HOOK 602 | HOOPSKIRT 603 | HORIZONTAL BAR 604 | HORSE CART 605 | HOURGLASS 606 | IPOD 607 | IRON 608 | JACK-O-LANTERN 609 | JEAN 610 | JEEP 611 | JERSEY 612 | JIGSAW PUZZLE 613 | JINRIKISHA 614 | JOYSTICK 615 | KIMONO 616 | KNEE PAD 617 | KNOT 618 | LAB COAT 619 | LADLE 620 | LAMPSHADE 621 | LAPTOP 622 | LAWN MOWER 623 | LENS CAP 624 | LETTER OPENER 625 | LIBRARY 626 | LIFEBOAT 627 | LIGHTER 628 | LIMOUSINE 629 | LINER 630 | LIPSTICK 631 | LOAFER 632 | LOTION 633 | LOUDSPEAKER 634 | LOUPE 635 | LUMBERMILL 636 | MAGNETIC COMPASS 637 | MAILBAG 638 | MAILBOX 639 | MAILLOT 640 | MAILLOT 641 | MANHOLE COVER 642 | MARACA 643 | MARIMBA 644 | MASK 645 | MATCHSTICK 646 | MAYPOLE 647 | MAZE 648 | MEASURING CUP 649 | MEDICINE CHEST 650 | MEGALITH 651 | MICROPHONE 652 | MICROWAVE 653 | MILITARY UNIFORM 654 | MILK CAN 655 | MINIBUS 656 | MINISKIRT 657 | MINIVAN 658 | MISSILE 659 | MITTEN 660 | MIXING BOWL 661 | MOBILE HOME 662 | MODEL T 663 | MODEM 664 | MONASTERY 665 | MONITOR 666 | MOPED 667 | MORTAR 668 | MORTARBOARD 669 | MOSQUE 670 | MOSQUITO NET 671 | MOTOR SCOOTER 672 | MOUNTAIN BIKE 673 | MOUNTAIN TENT 674 | MOUSE 675 | MOUSETRAP 676 | MOVING VAN 677 | MUZZLE 678 | NAIL 679 | NECK BRACE 680 | NECKLACE 681 | NIPPLE 682 | NOTEBOOK 683 | OBELISK 684 | OBOE 685 | OCARINA 686 | ODOMETER 687 | OIL FILTER 688 | ORGAN 689 | OSCILLOSCOPE 690 | OVERSKIRT 691 | OXCART 692 | OXYGEN MASK 693 | PACKET 694 | PADDLE 695 | PADDLEWHEEL 696 | PADLOCK 697 | PAINTBRUSH 698 | PAJAMA 699 | PALACE 700 | PANPIPE 701 | PAPER TOWEL 702 | PARACHUTE 703 | PARALLEL BARS 704 | PARK BENCH 705 | PARKING METER 706 | PASSENGER CAR 707 | PATIO 708 | PAY-PHONE 709 | PEDESTAL 710 | PENCIL BOX 711 | PENCIL SHARPENER 712 | PERFUME 713 | PETRI DISH 714 | PHOTOCOPIER 715 | PICK 716 | PICKELHAUBE 717 | PICKET FENCE 718 | PICKUP 719 | PIER 720 | PIGGY BANK 721 | PILL BOTTLE 722 | PILLOW 723 | PING-PONG BALL 724 | PINWHEEL 725 | PIRATE 726 | PITCHER 727 | PLANE 728 | PLANETARIUM 729 | PLASTIC BAG 730 | PLATE RACK 731 | PLOW 732 | PLUNGER 733 | POLAROID CAMERA 734 | POLE 735 | POLICE VAN 736 | PONCHO 737 | POOL TABLE 738 | POP BOTTLE 739 | POT 740 | POTTERS WHEEL 741 | POWER DRILL 742 | PRAYER RUG 743 | PRINTER 744 | PRISON 745 | PROJECTILE 746 | PROJECTOR 747 | PUCK 748 | PUNCHING BAG 749 | PURSE 750 | QUILL 751 | QUILT 752 | RACER 753 | RACKET 754 | RADIATOR 755 | RADIO 756 | RADIO TELESCOPE 757 | RAIN BARREL 758 | RECREATIONAL VEHICLE 759 | REEL 760 | REFLEX CAMERA 761 | REFRIGERATOR 762 | REMOTE CONTROL 763 | RESTAURANT 764 | REVOLVER 765 | RIFLE 766 | ROCKING CHAIR 767 | ROTISSERIE 768 | RUBBER ERASER 769 | RUGBY BALL 770 | RULE 771 | RUNNING SHOE 772 | SAFE 773 | SAFETY PIN 774 | SALTSHAKER 775 | SANDAL 776 | SARONG 777 | SAX 778 | SCABBARD 779 | SCALE 780 | SCHOOL BUS 781 | SCHOONER 782 | SCOREBOARD 783 | SCREEN 784 | SCREW 785 | SCREWDRIVER 786 | SEAT BELT 787 | SEWING MACHINE 788 | SHIELD 789 | SHOE SHOP 790 | SHOJI 791 | SHOPPING BASKET 792 | SHOPPING CART 793 | SHOVEL 794 | SHOWER CAP 795 | SHOWER CURTAIN 796 | SKI 797 | SKI MASK 798 | SLEEPING BAG 799 | SLIDE RULE 800 | SLIDING DOOR 801 | SLOT 802 | SNORKEL 803 | SNOWMOBILE 804 | SNOWPLOW 805 | SOAP DISPENSER 806 | SOCCER BALL 807 | SOCK 808 | SOLAR DISH 809 | SOMBRERO 810 | SOUP BOWL 811 | SPACE BAR 812 | SPACE HEATER 813 | SPACE SHUTTLE 814 | SPATULA 815 | SPEEDBOAT 816 | SPIDER WEB 817 | SPINDLE 818 | SPORTS CAR 819 | SPOTLIGHT 820 | STAGE 821 | STEAM LOCOMOTIVE 822 | STEEL ARCH BRIDGE 823 | STEEL DRUM 824 | STETHOSCOPE 825 | STOLE 826 | STONE WALL 827 | STOPWATCH 828 | STOVE 829 | STRAINER 830 | STREETCAR 831 | STRETCHER 832 | STUDIO COUCH 833 | STUPA 834 | SUBMARINE 835 | SUIT 836 | SUNDIAL 837 | SUNGLASS 838 | SUNGLASSES 839 | SUNSCREEN 840 | SUSPENSION BRIDGE 841 | SWAB 842 | SWEATSHIRT 843 | SWIMMING TRUNKS 844 | SWING 845 | SWITCH 846 | SYRINGE 847 | TABLE LAMP 848 | TANK 849 | TAPE PLAYER 850 | TEAPOT 851 | TEDDY 852 | TELEVISION 853 | TENNIS BALL 854 | THATCH 855 | THEATER CURTAIN 856 | THIMBLE 857 | THRESHER 858 | THRONE 859 | TILE ROOF 860 | TOASTER 861 | TOBACCO SHOP 862 | TOILET SEAT 863 | TORCH 864 | TOTEM POLE 865 | TOW TRUCK 866 | TOYSHOP 867 | TRACTOR 868 | TRAILER TRUCK 869 | TRAY 870 | TRENCH COAT 871 | TRICYCLE 872 | TRIMARAN 873 | TRIPOD 874 | TRIUMPHAL ARCH 875 | TROLLEYBUS 876 | TROMBONE 877 | TUB 878 | TURNSTILE 879 | TYPEWRITER KEYBOARD 880 | UMBRELLA 881 | UNICYCLE 882 | UPRIGHT 883 | VACUUM 884 | VASE 885 | VAULT 886 | VELVET 887 | VENDING MACHINE 888 | VESTMENT 889 | VIADUCT 890 | VIOLIN 891 | VOLLEYBALL 892 | WAFFLE IRON 893 | WALL CLOCK 894 | WALLET 895 | WARDROBE 896 | WARPLANE 897 | WASHBASIN 898 | WASHER 899 | WATER BOTTLE 900 | WATER JUG 901 | WATER TOWER 902 | WHISKEY JUG 903 | WHISTLE 904 | WIG 905 | WINDOW SCREEN 906 | WINDOW SHADE 907 | WINDSOR TIE 908 | WINE BOTTLE 909 | WING 910 | WOK 911 | WOODEN SPOON 912 | WOOL 913 | WORM FENCE 914 | WRECK 915 | YAWL 916 | YURT 917 | WEB SITE 918 | COMIC BOOK 919 | CROSSWORD PUZZLE 920 | STREET SIGN 921 | TRAFFIC LIGHT 922 | BOOK JACKET 923 | MENU 924 | PLATE 925 | GUACAMOLE 926 | CONSOMME 927 | HOT POT 928 | TRIFLE 929 | ICE CREAM 930 | ICE LOLLY 931 | FRENCH LOAF 932 | BAGEL 933 | PRETZEL 934 | CHEESEBURGER 935 | HOTDOG 936 | MASHED POTATO 937 | HEAD CABBAGE 938 | BROCCOLI 939 | CAULIFLOWER 940 | ZUCCHINI 941 | SPAGHETTI SQUASH 942 | ACORN SQUASH 943 | BUTTERNUT SQUASH 944 | CUCUMBER 945 | ARTICHOKE 946 | BELL PEPPER 947 | CARDOON 948 | MUSHROOM 949 | GRANNY SMITH 950 | STRAWBERRY 951 | ORANGE 952 | LEMON 953 | FIG 954 | PINEAPPLE 955 | BANANA 956 | JACKFRUIT 957 | CUSTARD APPLE 958 | POMEGRANATE 959 | HAY 960 | CARBONARA 961 | CHOCOLATE SAUCE 962 | DOUGH 963 | MEAT LOAF 964 | PIZZA 965 | POTPIE 966 | BURRITO 967 | RED WINE 968 | ESPRESSO 969 | CUP 970 | EGGNOG 971 | ALP 972 | BUBBLE 973 | CLIFF 974 | CORAL REEF 975 | GEYSER 976 | LAKESIDE 977 | PROMONTORY 978 | SANDBAR 979 | SEASHORE 980 | VALLEY 981 | VOLCANO 982 | BALLPLAYER 983 | GROOM 984 | SCUBA DIVER 985 | RAPESEED 986 | DAISY 987 | LADY SLIPPER 988 | CORN 989 | ACORN 990 | HIP 991 | BUCKEYE 992 | CORAL FUNGUS 993 | AGARIC 994 | GYROMITRA 995 | STINKHORN 996 | EARTHSTAR 997 | HEN-OF-THE-WOODS 998 | BOLETE 999 | EAR 1000 | TOILET TISSUE -------------------------------------------------------------------------------- /models/resnet50_pytorch/resnet_client.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tritonclient.http as httpclient 3 | import torch 4 | from PIL import Image 5 | 6 | 7 | if __name__ == '__main__': 8 | triton_client = httpclient.InferenceServerClient(url='172.17.0.2:8000') 9 | 10 | image = Image.open('../resources/images/cat.jpg') 11 | 12 | image = image.resize((224, 224), Image.ANTIALIAS) 13 | image = np.asarray(image) 14 | image = image / 255 15 | image = np.expand_dims(image, axis=0) 16 | image = np.transpose(image, axes=[0, 3, 1, 2]) 17 | image = image.astype(np.float32) 18 | 19 | inputs = [] 20 | inputs.append(httpclient.InferInput('INPUT__0', image.shape, "FP32")) 21 | inputs[0].set_data_from_numpy(image, binary_data=False) 22 | outputs = [] 23 | outputs.append(httpclient.InferRequestedOutput('OUTPUT__0', binary_data=False, class_count=3)) 24 | 25 | results = triton_client.infer('resnet50_pytorch', inputs=inputs, outputs=outputs) 26 | output_data0 = results.as_numpy('OUTPUT__0') 27 | print(output_data0.shape) 28 | print(output_data0) 29 | -------------------------------------------------------------------------------- /models/resnet50_pytorch/resnet_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision.models as models 3 | 4 | 5 | resnet50 = models.resnet50(pretrained=True) 6 | resnet50.eval() 7 | image = torch.randn(1, 3, 244, 244) 8 | resnet50_traced = torch.jit.trace(resnet50, image) 9 | resnet50(image) 10 | resnet50_traced.save('model.pt') 11 | -------------------------------------------------------------------------------- /quick/resnet50_pytorch/cat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zzk0/triton/9197448568c5ae964a1dc936624526e86ed05b4e/quick/resnet50_pytorch/cat.jpg -------------------------------------------------------------------------------- /quick/resnet50_pytorch/client_example.py: -------------------------------------------------------------------------------- 1 | import tritonclient.http as httpclient 2 | 3 | if __name__ == '__main__': 4 | triton_client = httpclient.InferenceServerClient(url='127.0.0.1:8000') 5 | 6 | model_repository_index = triton_client.get_model_repository_index() 7 | server_meta = triton_client.get_server_metadata() 8 | model_meta = triton_client.get_model_metadata('resnet50_pytorch') 9 | model_config = triton_client.get_model_config('resnet50_pytorch') 10 | statistics = triton_client.get_inference_statistics() 11 | shm_status = triton_client.get_cuda_shared_memory_status() 12 | sshm_status = triton_client.get_system_shared_memory_status() 13 | 14 | server_live = triton_client.is_server_live() 15 | server_ready = triton_client.is_server_ready() 16 | model_ready = triton_client.is_model_ready('resnet50_pytorch') 17 | 18 | # 启动命令: ./bin/tritonserver --model-store=/models --model-control-mode explicit --load-model resnet50_pytorch 19 | triton_client.unload_model('resnet50_pytorch') 20 | triton_client.load_model('resnet50_pytorch') 21 | 22 | # shared memory 23 | triton_client.register_system_shared_memory('a', 'b', 1024) 24 | sshm_status = triton_client.get_system_shared_memory_status() 25 | triton_client.unregister_system_shared_memory('a') 26 | 27 | triton_client.close() 28 | 29 | with httpclient.InferenceServerClient(url='127.0.0.1:8000'): 30 | pass 31 | 32 | -------------------------------------------------------------------------------- /quick/resnet50_pytorch/config.pbtxt: -------------------------------------------------------------------------------- 1 | name: "resnet50_pytorch" 2 | platform: "pytorch_libtorch" 3 | max_batch_size: 128 4 | input [ 5 | { 6 | name: "INPUT__0" 7 | data_type: TYPE_FP32 8 | dims: [ 3, 224, 224 ] 9 | } 10 | ] 11 | output [ 12 | { 13 | name: "OUTPUT__0" 14 | data_type: TYPE_FP32 15 | dims: [ 1000 ] 16 | label_filename: "labels.txt" 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /quick/resnet50_pytorch/labels.txt: -------------------------------------------------------------------------------- 1 | TENCH 2 | GOLDFISH 3 | WHITE SHARK 4 | TIGER SHARK 5 | HAMMERHEAD SHARK 6 | ELECTRIC RAY 7 | STINGRAY 8 | ROOSTER 9 | HEN 10 | OSTRICH 11 | BRAMBLING 12 | GOLDFINCH 13 | HOUSE FINCH 14 | SNOWBIRD 15 | INDIGO FINCH 16 | ROBIN 17 | BULBUL 18 | JAY 19 | MAGPIE 20 | CHICKADEE 21 | WATER OUZEL 22 | KITE 23 | BALD EAGLE 24 | VULTURE 25 | GREAT GREY OWL 26 | FIRE SALAMANDER 27 | NEWT 28 | EFT 29 | SPOTTED SALAMANDER 30 | AXOLOTL 31 | BULL FROG 32 | TREE FROG 33 | TAILED FROG 34 | LOGGERHEAD 35 | LEATHERBACK TURTLE 36 | MUD TURTLE 37 | TERRAPIN 38 | BOX TURTLE 39 | BANDED GECKO 40 | COMMON IGUANA 41 | AMERICAN CHAMELEON 42 | WHIPTAIL 43 | AGAMA 44 | FRILLED LIZARD 45 | ALLIGATOR LIZARD 46 | GILA MONSTER 47 | GREEN LIZARD 48 | AFRICAN CHAMELEON 49 | KOMODO DRAGON 50 | AFRICAN CROCODILE 51 | AMERICAN ALLIGATOR 52 | TRICERATOPS 53 | THUNDER SNAKE 54 | RINGNECK SNAKE 55 | HOGNOSE SNAKE 56 | GREEN SNAKE 57 | KING SNAKE 58 | GARTER SNAKE 59 | WATER SNAKE 60 | VINE SNAKE 61 | NIGHT SNAKE 62 | BOA 63 | ROCK PYTHON 64 | COBRA 65 | GREEN MAMBA 66 | SEA SNAKE 67 | HORNED VIPER 68 | DIAMONDBACK 69 | SIDEWINDER 70 | TRILOBITE 71 | HARVESTMAN 72 | SCORPION 73 | GARDEN SPIDER 74 | BARN SPIDER 75 | GARDEN SPIDER 76 | BLACK WIDOW 77 | TARANTULA 78 | WOLF SPIDER 79 | TICK 80 | CENTIPEDE 81 | GROUSE 82 | PTARMIGAN 83 | RUFFED GROUSE 84 | PRAIRIE CHICKEN 85 | PEACOCK 86 | QUAIL 87 | PARTRIDGE 88 | AFRICAN GREY 89 | MACAW 90 | COCKATOO 91 | LORIKEET 92 | COUCAL 93 | BEE EATER 94 | HORNBILL 95 | HUMMINGBIRD 96 | JACAMAR 97 | TOUCAN 98 | DRAKE 99 | MERGANSER 100 | GOOSE 101 | BLACK SWAN 102 | TUSKER 103 | ECHIDNA 104 | PLATYPUS 105 | WALLABY 106 | KOALA 107 | WOMBAT 108 | JELLYFISH 109 | SEA ANEMONE 110 | BRAIN CORAL 111 | FLATWORM 112 | NEMATODE 113 | CONCH 114 | SNAIL 115 | SLUG 116 | SEA SLUG 117 | CHITON 118 | CHAMBERED NAUTILUS 119 | DUNGENESS CRAB 120 | ROCK CRAB 121 | FIDDLER CRAB 122 | KING CRAB 123 | AMERICAN LOBSTER 124 | SPINY LOBSTER 125 | CRAYFISH 126 | HERMIT CRAB 127 | ISOPOD 128 | WHITE STORK 129 | BLACK STORK 130 | SPOONBILL 131 | FLAMINGO 132 | LITTLE BLUE HERON 133 | AMERICAN EGRET 134 | BITTERN 135 | CRANE 136 | LIMPKIN 137 | EUROPEAN GALLINULE 138 | AMERICAN COOT 139 | BUSTARD 140 | RUDDY TURNSTONE 141 | RED-BACKED SANDPIPER 142 | REDSHANK 143 | DOWITCHER 144 | OYSTERCATCHER 145 | PELICAN 146 | KING PENGUIN 147 | ALBATROSS 148 | GREY WHALE 149 | KILLER WHALE 150 | DUGONG 151 | SEA LION 152 | CHIHUAHUA 153 | JAPANESE SPANIEL 154 | MALTESE DOG 155 | PEKINESE 156 | SHIH-TZU 157 | BLENHEIM SPANIEL 158 | PAPILLON 159 | TOY TERRIER 160 | RHODESIAN RIDGEBACK 161 | AFGHAN HOUND 162 | BASSET 163 | BEAGLE 164 | BLOODHOUND 165 | BLUETICK 166 | COONHOUND 167 | WALKER HOUND 168 | ENGLISH FOXHOUND 169 | REDBONE 170 | BORZOI 171 | IRISH WOLFHOUND 172 | ITALIAN GREYHOUND 173 | WHIPPET 174 | IBIZAN HOUND 175 | NORWEGIAN ELKHOUND 176 | OTTERHOUND 177 | SALUKI 178 | SCOTTISH DEERHOUND 179 | WEIMARANER 180 | STAFFORDSHIRE BULLTERRIER 181 | STAFFORDSHIRE TERRIER 182 | BEDLINGTON TERRIER 183 | BORDER TERRIER 184 | KERRY BLUE TERRIER 185 | IRISH TERRIER 186 | NORFOLK TERRIER 187 | NORWICH TERRIER 188 | YORKSHIRE TERRIER 189 | WIRE-HAIRED FOX TERRIER 190 | LAKELAND TERRIER 191 | SEALYHAM TERRIER 192 | AIREDALE 193 | CAIRN 194 | AUSTRALIAN TERRIER 195 | DANDIE DINMONT 196 | BOSTON BULL 197 | MINIATURE SCHNAUZER 198 | GIANT SCHNAUZER 199 | STANDARD SCHNAUZER 200 | SCOTCH TERRIER 201 | TIBETAN TERRIER 202 | SILKY TERRIER 203 | WHEATEN TERRIER 204 | WHITE TERRIER 205 | LHASA 206 | RETRIEVER 207 | CURLY-COATED RETRIEVER 208 | GOLDEN RETRIEVER 209 | LABRADOR RETRIEVER 210 | CHESAPEAKE BAY RETRIEVER 211 | SHORT-HAIRED POINTER 212 | VISLA 213 | ENGLISH SETTER 214 | IRISH SETTER 215 | GORDON SETTER 216 | BRITTANY SPANIEL 217 | CLUMBER 218 | ENGLISH SPRINGER 219 | WELSH SPRINGER SPANIEL 220 | COCKER SPANIEL 221 | SUSSEX SPANIEL 222 | IRISH WATERSPANIEL 223 | KUVASZ 224 | SCHIPPERKE 225 | GROENENDAEL 226 | MALINOIS 227 | BRIARD 228 | KELPIE 229 | KOMONDOR 230 | OLD ENGLISH SHEEPDOG 231 | SHETLAND SHEEPDOG 232 | COLLIE 233 | BORDER COLLIE 234 | BOUVIER DES FLANDRES 235 | ROTTWEILER 236 | GERMAN SHEPHERD 237 | DOBERMAN 238 | MINIATURE PINSCHER 239 | GREATER SWISS MOUNTAIN DOG 240 | BERNESE MOUNTAIN DOG 241 | APPENZELLER 242 | ENTLEBUCHER 243 | BOXER 244 | BULL MASTIFF 245 | TIBETAN MASTIFF 246 | FRENCH BULLDOG 247 | GREAT DANE 248 | SAINT BERNARD 249 | ESKIMO DOG 250 | MALAMUTE 251 | SIBERIAN HUSKY 252 | DALMATIAN 253 | AFFENPINSCHER 254 | BASENJI 255 | PUG 256 | LEONBERG 257 | NEWFOUNDLAND 258 | GREAT PYRENEES 259 | SAMOYED 260 | POMERANIAN 261 | CHOW 262 | KEESHOND 263 | BRABANCON GRIFFON 264 | PEMBROKE 265 | CARDIGAN 266 | TOY POODLE 267 | MINIATURE POODLE 268 | STANDARD POODLE 269 | MEXICAN HAIRLESS 270 | TIMBER WOLF 271 | WHITE WOLF 272 | RED WOLF 273 | COYOTE 274 | DINGO 275 | DHOLE 276 | AFRICAN HUNTING DOG 277 | HYENA 278 | RED FOX 279 | KIT FOX 280 | ARCTIC FOX 281 | GREY FOX 282 | TABBY 283 | TIGER CAT 284 | PERSIAN CAT 285 | SIAMESE CAT 286 | EGYPTIAN CAT 287 | COUGAR 288 | LYNX 289 | LEOPARD 290 | SNOW LEOPARD 291 | JAGUAR 292 | LION 293 | TIGER 294 | CHEETAH 295 | BROWN BEAR 296 | AMERICAN BLACK BEAR 297 | ICE BEAR 298 | SLOTH BEAR 299 | MONGOOSE 300 | MEERKAT 301 | TIGER BEETLE 302 | LADYBUG 303 | GROUND BEETLE 304 | LONG-HORNED BEETLE 305 | LEAF BEETLE 306 | DUNG BEETLE 307 | RHINOCEROS BEETLE 308 | WEEVIL 309 | FLY 310 | BEE 311 | ANT 312 | GRASSHOPPER 313 | CRICKET 314 | WALKING STICK 315 | COCKROACH 316 | MANTIS 317 | CICADA 318 | LEAFHOPPER 319 | LACEWING 320 | DRAGONFLY 321 | DAMSELFLY 322 | ADMIRAL 323 | RINGLET 324 | MONARCH 325 | CABBAGE BUTTERFLY 326 | SULPHUR BUTTERFLY 327 | LYCAENID 328 | STARFISH 329 | SEA URCHIN 330 | SEA CUCUMBER 331 | WOOD RABBIT 332 | HARE 333 | ANGORA 334 | HAMSTER 335 | PORCUPINE 336 | FOX SQUIRREL 337 | MARMOT 338 | BEAVER 339 | GUINEA PIG 340 | SORREL 341 | ZEBRA 342 | HOG 343 | WILD BOAR 344 | WARTHOG 345 | HIPPOPOTAMUS 346 | OX 347 | WATER BUFFALO 348 | BISON 349 | RAM 350 | BIGHORN 351 | IBEX 352 | HARTEBEEST 353 | IMPALA 354 | GAZELLE 355 | ARABIAN CAMEL 356 | LLAMA 357 | WEASEL 358 | MINK 359 | POLECAT 360 | BLACK-FOOTED FERRET 361 | OTTER 362 | SKUNK 363 | BADGER 364 | ARMADILLO 365 | THREE-TOED SLOTH 366 | ORANGUTAN 367 | GORILLA 368 | CHIMPANZEE 369 | GIBBON 370 | SIAMANG 371 | GUENON 372 | PATAS 373 | BABOON 374 | MACAQUE 375 | LANGUR 376 | COLOBUS 377 | PROBOSCIS MONKEY 378 | MARMOSET 379 | CAPUCHIN 380 | HOWLER MONKEY 381 | TITI 382 | SPIDER MONKEY 383 | SQUIRREL MONKEY 384 | MADAGASCAR CAT 385 | INDRI 386 | INDIAN ELEPHANT 387 | AFRICAN ELEPHANT 388 | LESSER PANDA 389 | GIANT PANDA 390 | BARRACOUTA 391 | EEL 392 | COHO 393 | ROCK BEAUTY 394 | ANEMONE FISH 395 | STURGEON 396 | GAR 397 | LIONFISH 398 | PUFFER 399 | ABACUS 400 | ABAYA 401 | ACADEMIC GOWN 402 | ACCORDION 403 | ACOUSTIC GUITAR 404 | AIRCRAFT CARRIER 405 | AIRLINER 406 | AIRSHIP 407 | ALTAR 408 | AMBULANCE 409 | AMPHIBIAN 410 | ANALOG CLOCK 411 | APIARY 412 | APRON 413 | ASHCAN 414 | ASSAULT RIFLE 415 | BACKPACK 416 | BAKERY 417 | BALANCE BEAM 418 | BALLOON 419 | BALLPOINT 420 | BAND AID 421 | BANJO 422 | BANNISTER 423 | BARBELL 424 | BARBER CHAIR 425 | BARBERSHOP 426 | BARN 427 | BAROMETER 428 | BARREL 429 | BARROW 430 | BASEBALL 431 | BASKETBALL 432 | BASSINET 433 | BASSOON 434 | BATHING CAP 435 | BATH TOWEL 436 | BATHTUB 437 | BEACH WAGON 438 | BEACON 439 | BEAKER 440 | BEARSKIN 441 | BEER BOTTLE 442 | BEER GLASS 443 | BELL COTE 444 | BIB 445 | BICYCLE-BUILT-FOR-TWO 446 | BIKINI 447 | BINDER 448 | BINOCULARS 449 | BIRDHOUSE 450 | BOATHOUSE 451 | BOBSLED 452 | BOLO TIE 453 | BONNET 454 | BOOKCASE 455 | BOOKSHOP 456 | BOTTLECAP 457 | BOW 458 | BOW TIE 459 | BRASS 460 | BRASSIERE 461 | BREAKWATER 462 | BREASTPLATE 463 | BROOM 464 | BUCKET 465 | BUCKLE 466 | BULLETPROOF VEST 467 | BULLET TRAIN 468 | BUTCHER SHOP 469 | CAB 470 | CALDRON 471 | CANDLE 472 | CANNON 473 | CANOE 474 | CAN OPENER 475 | CARDIGAN 476 | CAR MIRROR 477 | CAROUSEL 478 | CARPENTERS KIT 479 | CARTON 480 | CAR WHEEL 481 | CASH MACHINE 482 | CASSETTE 483 | CASSETTE PLAYER 484 | CASTLE 485 | CATAMARAN 486 | CD PLAYER 487 | CELLO 488 | CELLULAR TELEPHONE 489 | CHAIN 490 | CHAINLINK FENCE 491 | CHAIN MAIL 492 | CHAIN SAW 493 | CHEST 494 | CHIFFONIER 495 | CHIME 496 | CHINA CABINET 497 | CHRISTMAS STOCKING 498 | CHURCH 499 | CINEMA 500 | CLEAVER 501 | CLIFF DWELLING 502 | CLOAK 503 | CLOG 504 | COCKTAIL SHAKER 505 | COFFEE MUG 506 | COFFEEPOT 507 | COIL 508 | COMBINATION LOCK 509 | COMPUTER KEYBOARD 510 | CONFECTIONERY 511 | CONTAINER SHIP 512 | CONVERTIBLE 513 | CORKSCREW 514 | CORNET 515 | COWBOY BOOT 516 | COWBOY HAT 517 | CRADLE 518 | CRANE 519 | CRASH HELMET 520 | CRATE 521 | CRIB 522 | CROCK POT 523 | CROQUET BALL 524 | CRUTCH 525 | CUIRASS 526 | DAM 527 | DESK 528 | DESKTOP COMPUTER 529 | DIAL TELEPHONE 530 | DIAPER 531 | DIGITAL CLOCK 532 | DIGITAL WATCH 533 | DINING TABLE 534 | DISHRAG 535 | DISHWASHER 536 | DISK BRAKE 537 | DOCK 538 | DOGSLED 539 | DOME 540 | DOORMAT 541 | DRILLING PLATFORM 542 | DRUM 543 | DRUMSTICK 544 | DUMBBELL 545 | DUTCH OVEN 546 | ELECTRIC FAN 547 | ELECTRIC GUITAR 548 | ELECTRIC LOCOMOTIVE 549 | ENTERTAINMENT CENTER 550 | ENVELOPE 551 | ESPRESSO MAKER 552 | FACE POWDER 553 | FEATHER BOA 554 | FILE 555 | FIREBOAT 556 | FIRE ENGINE 557 | FIRE SCREEN 558 | FLAGPOLE 559 | FLUTE 560 | FOLDING CHAIR 561 | FOOTBALL HELMET 562 | FORKLIFT 563 | FOUNTAIN 564 | FOUNTAIN PEN 565 | FOUR-POSTER 566 | FREIGHT CAR 567 | FRENCH HORN 568 | FRYING PAN 569 | FUR COAT 570 | GARBAGE TRUCK 571 | GASMASK 572 | GAS PUMP 573 | GOBLET 574 | GO-KART 575 | GOLF BALL 576 | GOLFCART 577 | GONDOLA 578 | GONG 579 | GOWN 580 | GRAND PIANO 581 | GREENHOUSE 582 | GRILLE 583 | GROCERY STORE 584 | GUILLOTINE 585 | HAIR SLIDE 586 | HAIR SPRAY 587 | HALF TRACK 588 | HAMMER 589 | HAMPER 590 | HAND BLOWER 591 | HAND-HELD COMPUTER 592 | HANDKERCHIEF 593 | HARD DISC 594 | HARMONICA 595 | HARP 596 | HARVESTER 597 | HATCHET 598 | HOLSTER 599 | HOME THEATER 600 | HONEYCOMB 601 | HOOK 602 | HOOPSKIRT 603 | HORIZONTAL BAR 604 | HORSE CART 605 | HOURGLASS 606 | IPOD 607 | IRON 608 | JACK-O-LANTERN 609 | JEAN 610 | JEEP 611 | JERSEY 612 | JIGSAW PUZZLE 613 | JINRIKISHA 614 | JOYSTICK 615 | KIMONO 616 | KNEE PAD 617 | KNOT 618 | LAB COAT 619 | LADLE 620 | LAMPSHADE 621 | LAPTOP 622 | LAWN MOWER 623 | LENS CAP 624 | LETTER OPENER 625 | LIBRARY 626 | LIFEBOAT 627 | LIGHTER 628 | LIMOUSINE 629 | LINER 630 | LIPSTICK 631 | LOAFER 632 | LOTION 633 | LOUDSPEAKER 634 | LOUPE 635 | LUMBERMILL 636 | MAGNETIC COMPASS 637 | MAILBAG 638 | MAILBOX 639 | MAILLOT 640 | MAILLOT 641 | MANHOLE COVER 642 | MARACA 643 | MARIMBA 644 | MASK 645 | MATCHSTICK 646 | MAYPOLE 647 | MAZE 648 | MEASURING CUP 649 | MEDICINE CHEST 650 | MEGALITH 651 | MICROPHONE 652 | MICROWAVE 653 | MILITARY UNIFORM 654 | MILK CAN 655 | MINIBUS 656 | MINISKIRT 657 | MINIVAN 658 | MISSILE 659 | MITTEN 660 | MIXING BOWL 661 | MOBILE HOME 662 | MODEL T 663 | MODEM 664 | MONASTERY 665 | MONITOR 666 | MOPED 667 | MORTAR 668 | MORTARBOARD 669 | MOSQUE 670 | MOSQUITO NET 671 | MOTOR SCOOTER 672 | MOUNTAIN BIKE 673 | MOUNTAIN TENT 674 | MOUSE 675 | MOUSETRAP 676 | MOVING VAN 677 | MUZZLE 678 | NAIL 679 | NECK BRACE 680 | NECKLACE 681 | NIPPLE 682 | NOTEBOOK 683 | OBELISK 684 | OBOE 685 | OCARINA 686 | ODOMETER 687 | OIL FILTER 688 | ORGAN 689 | OSCILLOSCOPE 690 | OVERSKIRT 691 | OXCART 692 | OXYGEN MASK 693 | PACKET 694 | PADDLE 695 | PADDLEWHEEL 696 | PADLOCK 697 | PAINTBRUSH 698 | PAJAMA 699 | PALACE 700 | PANPIPE 701 | PAPER TOWEL 702 | PARACHUTE 703 | PARALLEL BARS 704 | PARK BENCH 705 | PARKING METER 706 | PASSENGER CAR 707 | PATIO 708 | PAY-PHONE 709 | PEDESTAL 710 | PENCIL BOX 711 | PENCIL SHARPENER 712 | PERFUME 713 | PETRI DISH 714 | PHOTOCOPIER 715 | PICK 716 | PICKELHAUBE 717 | PICKET FENCE 718 | PICKUP 719 | PIER 720 | PIGGY BANK 721 | PILL BOTTLE 722 | PILLOW 723 | PING-PONG BALL 724 | PINWHEEL 725 | PIRATE 726 | PITCHER 727 | PLANE 728 | PLANETARIUM 729 | PLASTIC BAG 730 | PLATE RACK 731 | PLOW 732 | PLUNGER 733 | POLAROID CAMERA 734 | POLE 735 | POLICE VAN 736 | PONCHO 737 | POOL TABLE 738 | POP BOTTLE 739 | POT 740 | POTTERS WHEEL 741 | POWER DRILL 742 | PRAYER RUG 743 | PRINTER 744 | PRISON 745 | PROJECTILE 746 | PROJECTOR 747 | PUCK 748 | PUNCHING BAG 749 | PURSE 750 | QUILL 751 | QUILT 752 | RACER 753 | RACKET 754 | RADIATOR 755 | RADIO 756 | RADIO TELESCOPE 757 | RAIN BARREL 758 | RECREATIONAL VEHICLE 759 | REEL 760 | REFLEX CAMERA 761 | REFRIGERATOR 762 | REMOTE CONTROL 763 | RESTAURANT 764 | REVOLVER 765 | RIFLE 766 | ROCKING CHAIR 767 | ROTISSERIE 768 | RUBBER ERASER 769 | RUGBY BALL 770 | RULE 771 | RUNNING SHOE 772 | SAFE 773 | SAFETY PIN 774 | SALTSHAKER 775 | SANDAL 776 | SARONG 777 | SAX 778 | SCABBARD 779 | SCALE 780 | SCHOOL BUS 781 | SCHOONER 782 | SCOREBOARD 783 | SCREEN 784 | SCREW 785 | SCREWDRIVER 786 | SEAT BELT 787 | SEWING MACHINE 788 | SHIELD 789 | SHOE SHOP 790 | SHOJI 791 | SHOPPING BASKET 792 | SHOPPING CART 793 | SHOVEL 794 | SHOWER CAP 795 | SHOWER CURTAIN 796 | SKI 797 | SKI MASK 798 | SLEEPING BAG 799 | SLIDE RULE 800 | SLIDING DOOR 801 | SLOT 802 | SNORKEL 803 | SNOWMOBILE 804 | SNOWPLOW 805 | SOAP DISPENSER 806 | SOCCER BALL 807 | SOCK 808 | SOLAR DISH 809 | SOMBRERO 810 | SOUP BOWL 811 | SPACE BAR 812 | SPACE HEATER 813 | SPACE SHUTTLE 814 | SPATULA 815 | SPEEDBOAT 816 | SPIDER WEB 817 | SPINDLE 818 | SPORTS CAR 819 | SPOTLIGHT 820 | STAGE 821 | STEAM LOCOMOTIVE 822 | STEEL ARCH BRIDGE 823 | STEEL DRUM 824 | STETHOSCOPE 825 | STOLE 826 | STONE WALL 827 | STOPWATCH 828 | STOVE 829 | STRAINER 830 | STREETCAR 831 | STRETCHER 832 | STUDIO COUCH 833 | STUPA 834 | SUBMARINE 835 | SUIT 836 | SUNDIAL 837 | SUNGLASS 838 | SUNGLASSES 839 | SUNSCREEN 840 | SUSPENSION BRIDGE 841 | SWAB 842 | SWEATSHIRT 843 | SWIMMING TRUNKS 844 | SWING 845 | SWITCH 846 | SYRINGE 847 | TABLE LAMP 848 | TANK 849 | TAPE PLAYER 850 | TEAPOT 851 | TEDDY 852 | TELEVISION 853 | TENNIS BALL 854 | THATCH 855 | THEATER CURTAIN 856 | THIMBLE 857 | THRESHER 858 | THRONE 859 | TILE ROOF 860 | TOASTER 861 | TOBACCO SHOP 862 | TOILET SEAT 863 | TORCH 864 | TOTEM POLE 865 | TOW TRUCK 866 | TOYSHOP 867 | TRACTOR 868 | TRAILER TRUCK 869 | TRAY 870 | TRENCH COAT 871 | TRICYCLE 872 | TRIMARAN 873 | TRIPOD 874 | TRIUMPHAL ARCH 875 | TROLLEYBUS 876 | TROMBONE 877 | TUB 878 | TURNSTILE 879 | TYPEWRITER KEYBOARD 880 | UMBRELLA 881 | UNICYCLE 882 | UPRIGHT 883 | VACUUM 884 | VASE 885 | VAULT 886 | VELVET 887 | VENDING MACHINE 888 | VESTMENT 889 | VIADUCT 890 | VIOLIN 891 | VOLLEYBALL 892 | WAFFLE IRON 893 | WALL CLOCK 894 | WALLET 895 | WARDROBE 896 | WARPLANE 897 | WASHBASIN 898 | WASHER 899 | WATER BOTTLE 900 | WATER JUG 901 | WATER TOWER 902 | WHISKEY JUG 903 | WHISTLE 904 | WIG 905 | WINDOW SCREEN 906 | WINDOW SHADE 907 | WINDSOR TIE 908 | WINE BOTTLE 909 | WING 910 | WOK 911 | WOODEN SPOON 912 | WOOL 913 | WORM FENCE 914 | WRECK 915 | YAWL 916 | YURT 917 | WEB SITE 918 | COMIC BOOK 919 | CROSSWORD PUZZLE 920 | STREET SIGN 921 | TRAFFIC LIGHT 922 | BOOK JACKET 923 | MENU 924 | PLATE 925 | GUACAMOLE 926 | CONSOMME 927 | HOT POT 928 | TRIFLE 929 | ICE CREAM 930 | ICE LOLLY 931 | FRENCH LOAF 932 | BAGEL 933 | PRETZEL 934 | CHEESEBURGER 935 | HOTDOG 936 | MASHED POTATO 937 | HEAD CABBAGE 938 | BROCCOLI 939 | CAULIFLOWER 940 | ZUCCHINI 941 | SPAGHETTI SQUASH 942 | ACORN SQUASH 943 | BUTTERNUT SQUASH 944 | CUCUMBER 945 | ARTICHOKE 946 | BELL PEPPER 947 | CARDOON 948 | MUSHROOM 949 | GRANNY SMITH 950 | STRAWBERRY 951 | ORANGE 952 | LEMON 953 | FIG 954 | PINEAPPLE 955 | BANANA 956 | JACKFRUIT 957 | CUSTARD APPLE 958 | POMEGRANATE 959 | HAY 960 | CARBONARA 961 | CHOCOLATE SAUCE 962 | DOUGH 963 | MEAT LOAF 964 | PIZZA 965 | POTPIE 966 | BURRITO 967 | RED WINE 968 | ESPRESSO 969 | CUP 970 | EGGNOG 971 | ALP 972 | BUBBLE 973 | CLIFF 974 | CORAL REEF 975 | GEYSER 976 | LAKESIDE 977 | PROMONTORY 978 | SANDBAR 979 | SEASHORE 980 | VALLEY 981 | VOLCANO 982 | BALLPLAYER 983 | GROOM 984 | SCUBA DIVER 985 | RAPESEED 986 | DAISY 987 | LADY SLIPPER 988 | CORN 989 | ACORN 990 | HIP 991 | BUCKEYE 992 | CORAL FUNGUS 993 | AGARIC 994 | GYROMITRA 995 | STINKHORN 996 | EARTHSTAR 997 | HEN-OF-THE-WOODS 998 | BOLETE 999 | EAR 1000 | TOILET TISSUE -------------------------------------------------------------------------------- /quick/resnet50_pytorch/resnet_client.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tritonclient.http as httpclient 3 | import torch 4 | from PIL import Image 5 | 6 | 7 | if __name__ == '__main__': 8 | triton_client = httpclient.InferenceServerClient(url='127.0.0.1:8000') 9 | image = Image.open('./cat.jpg') 10 | 11 | image = image.resize((224, 224), Image.ANTIALIAS) 12 | image = np.asarray(image) 13 | image = image / 255 14 | image = np.expand_dims(image, axis=0) 15 | image = np.transpose(image, axes=[0, 3, 1, 2]) 16 | image = image.astype(np.float32) 17 | 18 | inputs = [] 19 | inputs.append(httpclient.InferInput('INPUT__0', image.shape, "FP32")) 20 | inputs[0].set_data_from_numpy(image, binary_data=False) 21 | outputs = [] 22 | outputs.append(httpclient.InferRequestedOutput('OUTPUT__0', binary_data=False, class_count=3)) # class_count 表示 topN 分类 23 | # outputs.append(httpclient.InferRequestedOutput('OUTPUT__0', binary_data=False)) 24 | 25 | results = triton_client.infer('resnet50_pytorch', inputs=inputs, outputs=outputs) 26 | output_data0 = results.as_numpy('OUTPUT__0') 27 | print(output_data0.shape) 28 | print(output_data0) 29 | -------------------------------------------------------------------------------- /quick/resnet50_pytorch/resnet_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision.models as models 3 | 4 | 5 | resnet50 = models.resnet50(pretrained=True) 6 | resnet50.eval() 7 | image = torch.randn(1, 3, 244, 244) 8 | resnet50_traced = torch.jit.trace(resnet50, image) 9 | resnet50(image) 10 | resnet50_traced.save('model.pt') 11 | -------------------------------------------------------------------------------- /rate/example_python_a/1/model.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | import triton_python_backend_utils as pb_utils 4 | 5 | 6 | class TritonPythonModel: 7 | 8 | def initialize(self, args): 9 | self.model_config = model_config = json.loads(args['model_config']) 10 | output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") 11 | output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") 12 | self.output0_dtype = pb_utils.triton_string_to_numpy(output0_config['data_type']) 13 | self.output1_dtype = pb_utils.triton_string_to_numpy(output1_config['data_type']) 14 | 15 | def execute(self, requests): 16 | time.sleep(10) 17 | print('model A executing...') 18 | output0_dtype = self.output0_dtype 19 | output1_dtype = self.output1_dtype 20 | responses = [] 21 | for request in requests: 22 | in_0 = pb_utils.get_input_tensor_by_name(request, 'INPUT0') 23 | in_1 = pb_utils.get_input_tensor_by_name(request, 'INPUT1') 24 | out_0, out_1 = (in_0.as_numpy() + in_1.as_numpy(), 25 | in_0.as_numpy() - in_1.as_numpy()) 26 | out_tensor_0 = pb_utils.Tensor('OUTPUT0', out_0.astype(output0_dtype)) 27 | out_tensor_1 = pb_utils.Tensor('OUTPUT1', out_1.astype(output1_dtype)) 28 | inference_response = pb_utils.InferenceResponse(output_tensors=[out_tensor_0, out_tensor_1]) 29 | responses.append(inference_response) 30 | return responses 31 | 32 | def finalize(self): 33 | print('Cleaning up...') 34 | -------------------------------------------------------------------------------- /rate/example_python_a/client.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import tritonclient.http as httpclient 4 | 5 | 6 | if __name__ == '__main__': 7 | triton_client = httpclient.InferenceServerClient(url='127.0.0.1:8000') 8 | 9 | inputs = [] 10 | inputs.append(httpclient.InferInput('INPUT0', [4], "FP32")) 11 | inputs.append(httpclient.InferInput('INPUT1', [4], "FP32")) 12 | input_data0 = np.random.randn(4).astype(np.float32) 13 | input_data1 = np.random.randn(4).astype(np.float32) 14 | inputs[0].set_data_from_numpy(input_data0, binary_data=False) 15 | inputs[1].set_data_from_numpy(input_data1, binary_data=False) 16 | outputs = [] 17 | outputs.append(httpclient.InferRequestedOutput('OUTPUT0', binary_data=False)) 18 | outputs.append(httpclient.InferRequestedOutput('OUTPUT1', binary_data=False)) 19 | 20 | t0 = time.time() 21 | results = triton_client.infer('example_python', inputs=inputs, outputs=outputs) 22 | t1 = time.time() 23 | print('inference time cost: {} ms'.format(1000 * (t1 - t0))) 24 | output_data0 = results.as_numpy('OUTPUT0') 25 | output_data1 = results.as_numpy('OUTPUT1') 26 | 27 | print(input_data0) 28 | print(input_data1) 29 | print(output_data0) 30 | print(output_data1) 31 | -------------------------------------------------------------------------------- /rate/example_python_a/config.pbtxt: -------------------------------------------------------------------------------- 1 | name: "example_python_a" 2 | backend: "python" 3 | input [ 4 | { 5 | name: "INPUT0" 6 | data_type: TYPE_FP32 7 | dims: [ 4 ] 8 | } 9 | ] 10 | input [ 11 | { 12 | name: "INPUT1" 13 | data_type: TYPE_FP32 14 | dims: [ 4 ] 15 | } 16 | ] 17 | output [ 18 | { 19 | name: "OUTPUT0" 20 | data_type: TYPE_FP32 21 | dims: [ 4 ] 22 | } 23 | ] 24 | output [ 25 | { 26 | name: "OUTPUT1" 27 | data_type: TYPE_FP32 28 | dims: [ 4 ] 29 | } 30 | ] 31 | 32 | instance_group [ 33 | { 34 | count: 2 35 | kind: KIND_CPU 36 | rate_limiter { 37 | resources [ 38 | { 39 | name: "R1" 40 | count: 10 41 | }, 42 | { 43 | name: "R2" 44 | count: 5 45 | }, 46 | { 47 | name: "R3" 48 | global: True 49 | count: 10 50 | } 51 | ] 52 | priority: 1 53 | } 54 | } 55 | ] 56 | -------------------------------------------------------------------------------- /rate/example_python_b/1/model.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | import triton_python_backend_utils as pb_utils 4 | 5 | 6 | class TritonPythonModel: 7 | 8 | def initialize(self, args): 9 | self.model_config = model_config = json.loads(args['model_config']) 10 | output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") 11 | output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") 12 | self.output0_dtype = pb_utils.triton_string_to_numpy(output0_config['data_type']) 13 | self.output1_dtype = pb_utils.triton_string_to_numpy(output1_config['data_type']) 14 | 15 | def execute(self, requests): 16 | time.sleep(10) 17 | print('model B executing...') 18 | output0_dtype = self.output0_dtype 19 | output1_dtype = self.output1_dtype 20 | responses = [] 21 | for request in requests: 22 | in_0 = pb_utils.get_input_tensor_by_name(request, 'INPUT0') 23 | in_1 = pb_utils.get_input_tensor_by_name(request, 'INPUT1') 24 | out_0, out_1 = (in_0.as_numpy() + in_1.as_numpy(), 25 | in_0.as_numpy() - in_1.as_numpy()) 26 | out_tensor_0 = pb_utils.Tensor('OUTPUT0', out_0.astype(output0_dtype)) 27 | out_tensor_1 = pb_utils.Tensor('OUTPUT1', out_1.astype(output1_dtype)) 28 | inference_response = pb_utils.InferenceResponse(output_tensors=[out_tensor_0, out_tensor_1]) 29 | responses.append(inference_response) 30 | return responses 31 | 32 | def finalize(self): 33 | print('Cleaning up...') 34 | -------------------------------------------------------------------------------- /rate/example_python_b/client.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import tritonclient.http as httpclient 4 | 5 | 6 | if __name__ == '__main__': 7 | triton_client = httpclient.InferenceServerClient(url='127.0.0.1:8000') 8 | 9 | inputs = [] 10 | inputs.append(httpclient.InferInput('INPUT0', [4], "FP32")) 11 | inputs.append(httpclient.InferInput('INPUT1', [4], "FP32")) 12 | input_data0 = np.random.randn(4).astype(np.float32) 13 | input_data1 = np.random.randn(4).astype(np.float32) 14 | inputs[0].set_data_from_numpy(input_data0, binary_data=False) 15 | inputs[1].set_data_from_numpy(input_data1, binary_data=False) 16 | outputs = [] 17 | outputs.append(httpclient.InferRequestedOutput('OUTPUT0', binary_data=False)) 18 | outputs.append(httpclient.InferRequestedOutput('OUTPUT1', binary_data=False)) 19 | 20 | t0 = time.time() 21 | results = triton_client.infer('example_python_b', inputs=inputs, outputs=outputs) 22 | t1 = time.time() 23 | print('inference time cost: {} ms'.format(1000 * (t1 - t0))) 24 | output_data0 = results.as_numpy('OUTPUT0') 25 | output_data1 = results.as_numpy('OUTPUT1') 26 | 27 | print(input_data0) 28 | print(input_data1) 29 | print(output_data0) 30 | print(output_data1) 31 | -------------------------------------------------------------------------------- /rate/example_python_b/config.pbtxt: -------------------------------------------------------------------------------- 1 | name: "example_python_b" 2 | backend: "python" 3 | input [ 4 | { 5 | name: "INPUT0" 6 | data_type: TYPE_FP32 7 | dims: [ 4 ] 8 | } 9 | ] 10 | input [ 11 | { 12 | name: "INPUT1" 13 | data_type: TYPE_FP32 14 | dims: [ 4 ] 15 | } 16 | ] 17 | output [ 18 | { 19 | name: "OUTPUT0" 20 | data_type: TYPE_FP32 21 | dims: [ 4 ] 22 | } 23 | ] 24 | output [ 25 | { 26 | name: "OUTPUT1" 27 | data_type: TYPE_FP32 28 | dims: [ 4 ] 29 | } 30 | ] 31 | 32 | instance_group [ 33 | { 34 | count: 2 35 | kind: KIND_CPU 36 | rate_limiter { 37 | resources [ 38 | { 39 | name: "R3" 40 | global: True 41 | count: 3 42 | } 43 | ] 44 | priority: 1 45 | } 46 | } 47 | ] 48 | -------------------------------------------------------------------------------- /rate/example_python_c/1/model.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | import triton_python_backend_utils as pb_utils 4 | 5 | 6 | class TritonPythonModel: 7 | 8 | def initialize(self, args): 9 | self.model_config = model_config = json.loads(args['model_config']) 10 | output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") 11 | output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") 12 | self.output0_dtype = pb_utils.triton_string_to_numpy(output0_config['data_type']) 13 | self.output1_dtype = pb_utils.triton_string_to_numpy(output1_config['data_type']) 14 | 15 | def execute(self, requests): 16 | time.sleep(10) 17 | print('model C executing...') 18 | output0_dtype = self.output0_dtype 19 | output1_dtype = self.output1_dtype 20 | responses = [] 21 | for request in requests: 22 | in_0 = pb_utils.get_input_tensor_by_name(request, 'INPUT0') 23 | in_1 = pb_utils.get_input_tensor_by_name(request, 'INPUT1') 24 | out_0, out_1 = (in_0.as_numpy() + in_1.as_numpy(), 25 | in_0.as_numpy() - in_1.as_numpy()) 26 | out_tensor_0 = pb_utils.Tensor('OUTPUT0', out_0.astype(output0_dtype)) 27 | out_tensor_1 = pb_utils.Tensor('OUTPUT1', out_1.astype(output1_dtype)) 28 | inference_response = pb_utils.InferenceResponse(output_tensors=[out_tensor_0, out_tensor_1]) 29 | responses.append(inference_response) 30 | return responses 31 | 32 | def finalize(self): 33 | print('Cleaning up...') 34 | -------------------------------------------------------------------------------- /rate/example_python_c/client.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import tritonclient.http as httpclient 4 | 5 | 6 | if __name__ == '__main__': 7 | triton_client = httpclient.InferenceServerClient(url='127.0.0.1:8000') 8 | 9 | inputs = [] 10 | inputs.append(httpclient.InferInput('INPUT0', [4], "FP32")) 11 | inputs.append(httpclient.InferInput('INPUT1', [4], "FP32")) 12 | input_data0 = np.random.randn(4).astype(np.float32) 13 | input_data1 = np.random.randn(4).astype(np.float32) 14 | inputs[0].set_data_from_numpy(input_data0, binary_data=False) 15 | inputs[1].set_data_from_numpy(input_data1, binary_data=False) 16 | outputs = [] 17 | outputs.append(httpclient.InferRequestedOutput('OUTPUT0', binary_data=False)) 18 | outputs.append(httpclient.InferRequestedOutput('OUTPUT1', binary_data=False)) 19 | 20 | t0 = time.time() 21 | results = triton_client.infer('example_python_c', inputs=inputs, outputs=outputs) 22 | t1 = time.time() 23 | print('inference time cost: {} ms'.format(1000 * (t1 - t0))) 24 | output_data0 = results.as_numpy('OUTPUT0') 25 | output_data1 = results.as_numpy('OUTPUT1') 26 | 27 | print(input_data0) 28 | print(input_data1) 29 | print(output_data0) 30 | print(output_data1) 31 | -------------------------------------------------------------------------------- /rate/example_python_c/config.pbtxt: -------------------------------------------------------------------------------- 1 | name: "example_python_c" 2 | backend: "python" 3 | input [ 4 | { 5 | name: "INPUT0" 6 | data_type: TYPE_FP32 7 | dims: [ 4 ] 8 | } 9 | ] 10 | input [ 11 | { 12 | name: "INPUT1" 13 | data_type: TYPE_FP32 14 | dims: [ 4 ] 15 | } 16 | ] 17 | output [ 18 | { 19 | name: "OUTPUT0" 20 | data_type: TYPE_FP32 21 | dims: [ 4 ] 22 | } 23 | ] 24 | output [ 25 | { 26 | name: "OUTPUT1" 27 | data_type: TYPE_FP32 28 | dims: [ 4 ] 29 | } 30 | ] 31 | 32 | instance_group [ 33 | { 34 | count: 2 35 | kind: KIND_CPU 36 | rate_limiter { 37 | resources [ 38 | { 39 | name: "R1" 40 | count: 5 41 | }, 42 | { 43 | name: "R2" 44 | count: 5 45 | } 46 | ] 47 | priority: 1 48 | } 49 | } 50 | ] 51 | -------------------------------------------------------------------------------- /rate/example_python_d/1/model.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | import triton_python_backend_utils as pb_utils 4 | 5 | 6 | class TritonPythonModel: 7 | 8 | def initialize(self, args): 9 | self.model_config = model_config = json.loads(args['model_config']) 10 | output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") 11 | output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") 12 | self.output0_dtype = pb_utils.triton_string_to_numpy(output0_config['data_type']) 13 | self.output1_dtype = pb_utils.triton_string_to_numpy(output1_config['data_type']) 14 | 15 | def execute(self, requests): 16 | time.sleep(10) 17 | print('model D executing...') 18 | output0_dtype = self.output0_dtype 19 | output1_dtype = self.output1_dtype 20 | responses = [] 21 | for request in requests: 22 | in_0 = pb_utils.get_input_tensor_by_name(request, 'INPUT0') 23 | in_1 = pb_utils.get_input_tensor_by_name(request, 'INPUT1') 24 | out_0, out_1 = (in_0.as_numpy() + in_1.as_numpy(), 25 | in_0.as_numpy() - in_1.as_numpy()) 26 | out_tensor_0 = pb_utils.Tensor('OUTPUT0', out_0.astype(output0_dtype)) 27 | out_tensor_1 = pb_utils.Tensor('OUTPUT1', out_1.astype(output1_dtype)) 28 | inference_response = pb_utils.InferenceResponse(output_tensors=[out_tensor_0, out_tensor_1]) 29 | responses.append(inference_response) 30 | return responses 31 | 32 | def finalize(self): 33 | print('Cleaning up...') 34 | -------------------------------------------------------------------------------- /rate/example_python_d/client.py: -------------------------------------------------------------------------------- 1 | import time 2 | import numpy as np 3 | import tritonclient.http as httpclient 4 | 5 | 6 | if __name__ == '__main__': 7 | triton_client = httpclient.InferenceServerClient(url='127.0.0.1:8000') 8 | 9 | inputs = [] 10 | inputs.append(httpclient.InferInput('INPUT0', [4], "FP32")) 11 | inputs.append(httpclient.InferInput('INPUT1', [4], "FP32")) 12 | input_data0 = np.random.randn(4).astype(np.float32) 13 | input_data1 = np.random.randn(4).astype(np.float32) 14 | inputs[0].set_data_from_numpy(input_data0, binary_data=False) 15 | inputs[1].set_data_from_numpy(input_data1, binary_data=False) 16 | outputs = [] 17 | outputs.append(httpclient.InferRequestedOutput('OUTPUT0', binary_data=False)) 18 | outputs.append(httpclient.InferRequestedOutput('OUTPUT1', binary_data=False)) 19 | 20 | t0 = time.time() 21 | results = triton_client.infer('example_python_d', inputs=inputs, outputs=outputs) 22 | t1 = time.time() 23 | print('inference time cost: {} ms'.format(1000 * (t1 - t0))) 24 | output_data0 = results.as_numpy('OUTPUT0') 25 | output_data1 = results.as_numpy('OUTPUT1') 26 | 27 | print(input_data0) 28 | print(input_data1) 29 | print(output_data0) 30 | print(output_data1) 31 | -------------------------------------------------------------------------------- /rate/example_python_d/config.pbtxt: -------------------------------------------------------------------------------- 1 | name: "example_python_d" 2 | backend: "python" 3 | input [ 4 | { 5 | name: "INPUT0" 6 | data_type: TYPE_FP32 7 | dims: [ 4 ] 8 | } 9 | ] 10 | input [ 11 | { 12 | name: "INPUT1" 13 | data_type: TYPE_FP32 14 | dims: [ 4 ] 15 | } 16 | ] 17 | output [ 18 | { 19 | name: "OUTPUT0" 20 | data_type: TYPE_FP32 21 | dims: [ 4 ] 22 | } 23 | ] 24 | output [ 25 | { 26 | name: "OUTPUT1" 27 | data_type: TYPE_FP32 28 | dims: [ 4 ] 29 | } 30 | ] 31 | 32 | instance_group [ 33 | { 34 | count: 2 35 | kind: KIND_CPU 36 | rate_limiter { 37 | resources [ 38 | { 39 | name: "R1" 40 | count: 4 41 | }, 42 | { 43 | name: "R2" 44 | count: 6 45 | } 46 | ] 47 | priority: 10 48 | } 49 | } 50 | ] --------------------------------------------------------------------------------