├── .gitattributes ├── .gitignore ├── Hackathon2021 └── README.md ├── Hackathon2022 ├── HackathonGuide.md ├── README.md └── code │ ├── .clang-format │ ├── .gitignore │ ├── .style.yapf │ ├── AttentionPlugin │ ├── AttentionPlugin.cu │ ├── AttentionPlugin.h │ ├── Makefile │ ├── attentionKernels.cu │ ├── attentionKernels.h │ ├── layerKernels.cu │ ├── layerKernels.h │ ├── testAttentionPlugin.py │ ├── utils.cu │ └── utils.h │ ├── LayerNormPlugin │ ├── LayerNormPlugin.cu │ ├── LayerNormPlugin.h │ ├── Makefile │ └── testLayerNormPlugin.py │ ├── MaskPlugin-1output │ ├── Makefile │ ├── MaskPlugin.cu │ ├── MaskPlugin.h │ └── testMaskPlugin.py │ ├── MaskPlugin-2output │ ├── Makefile │ ├── MaskPlugin.cu │ ├── MaskPlugin.h │ └── testMaskPlugin.py │ ├── README.md │ ├── build.sh │ ├── calibrator.py │ ├── createFakeIOData.py │ ├── decoder-surgeonV3.py │ ├── decoder.onnx │ ├── decoder.py │ ├── encoder-surgeonV5.py │ ├── encoder-surgeonV6.py │ ├── encoder.onnx │ ├── encoder.py │ ├── requirements.txt │ └── testEncoderAndDecoder.py ├── Hackathon2023 ├── README.md ├── controlnet │ ├── DDIM_scheduler.py │ ├── HACKATHON.md │ ├── LICENSE │ ├── README.md │ ├── annotator │ │ ├── canny │ │ │ └── __init__.py │ │ ├── ckpts │ │ │ └── ckpts.txt │ │ ├── hed │ │ │ └── __init__.py │ │ ├── midas │ │ │ ├── LICENSE │ │ │ ├── __init__.py │ │ │ ├── api.py │ │ │ ├── midas │ │ │ │ ├── __init__.py │ │ │ │ ├── base_model.py │ │ │ │ ├── blocks.py │ │ │ │ ├── dpt_depth.py │ │ │ │ ├── midas_net.py │ │ │ │ ├── midas_net_custom.py │ │ │ │ ├── transforms.py │ │ │ │ └── vit.py │ │ │ └── utils.py │ │ ├── mlsd │ │ │ ├── LICENSE │ │ │ ├── __init__.py │ │ │ ├── models │ │ │ │ ├── mbv2_mlsd_large.py │ │ │ │ └── mbv2_mlsd_tiny.py │ │ │ └── utils.py │ │ ├── openpose │ │ │ ├── LICENSE │ │ │ ├── __init__.py │ │ │ ├── body.py │ │ │ ├── hand.py │ │ │ ├── model.py │ │ │ └── util.py │ │ ├── uniformer │ │ │ ├── LICENSE │ │ │ ├── __init__.py │ │ │ ├── configs │ │ │ │ └── _base_ │ │ │ │ │ ├── datasets │ │ │ │ │ ├── ade20k.py │ │ │ │ │ ├── chase_db1.py │ │ │ │ │ ├── cityscapes.py │ │ │ │ │ ├── cityscapes_769x769.py │ │ │ │ │ ├── drive.py │ │ │ │ │ ├── hrf.py │ │ │ │ │ ├── pascal_context.py │ │ │ │ │ ├── pascal_context_59.py │ │ │ │ │ ├── pascal_voc12.py │ │ │ │ │ ├── pascal_voc12_aug.py │ │ │ │ │ └── stare.py │ │ │ │ │ ├── default_runtime.py │ │ │ │ │ ├── models │ │ │ │ │ ├── ann_r50-d8.py │ │ │ │ │ ├── apcnet_r50-d8.py │ │ │ │ │ ├── ccnet_r50-d8.py │ │ │ │ │ ├── cgnet.py │ │ │ │ │ ├── danet_r50-d8.py │ │ │ │ │ ├── deeplabv3_r50-d8.py │ │ │ │ │ ├── deeplabv3_unet_s5-d16.py │ │ │ │ │ ├── deeplabv3plus_r50-d8.py │ │ │ │ │ ├── dmnet_r50-d8.py │ │ │ │ │ ├── dnl_r50-d8.py │ │ │ │ │ ├── emanet_r50-d8.py │ │ │ │ │ ├── encnet_r50-d8.py │ │ │ │ │ ├── fast_scnn.py │ │ │ │ │ ├── fcn_hr18.py │ │ │ │ │ ├── fcn_r50-d8.py │ │ │ │ │ ├── fcn_unet_s5-d16.py │ │ │ │ │ ├── fpn_r50.py │ │ │ │ │ ├── fpn_uniformer.py │ │ │ │ │ ├── gcnet_r50-d8.py │ │ │ │ │ ├── lraspp_m-v3-d8.py │ │ │ │ │ ├── nonlocal_r50-d8.py │ │ │ │ │ ├── ocrnet_hr18.py │ │ │ │ │ ├── ocrnet_r50-d8.py │ │ │ │ │ ├── pointrend_r50.py │ │ │ │ │ ├── psanet_r50-d8.py │ │ │ │ │ ├── pspnet_r50-d8.py │ │ │ │ │ ├── pspnet_unet_s5-d16.py │ │ │ │ │ ├── upernet_r50.py │ │ │ │ │ └── upernet_uniformer.py │ │ │ │ │ └── schedules │ │ │ │ │ ├── schedule_160k.py │ │ │ │ │ ├── schedule_20k.py │ │ │ │ │ ├── schedule_40k.py │ │ │ │ │ └── schedule_80k.py │ │ │ ├── exp │ │ │ │ └── upernet_global_small │ │ │ │ │ ├── config.py │ │ │ │ │ ├── run.sh │ │ │ │ │ ├── test.sh │ │ │ │ │ ├── test_config_g.py │ │ │ │ │ ├── test_config_h32.py │ │ │ │ │ └── test_config_w32.py │ │ │ ├── mmcv │ │ │ │ ├── __init__.py │ │ │ │ ├── arraymisc │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── quantization.py │ │ │ │ ├── cnn │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── alexnet.py │ │ │ │ │ ├── bricks │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── activation.py │ │ │ │ │ │ ├── context_block.py │ │ │ │ │ │ ├── conv.py │ │ │ │ │ │ ├── conv2d_adaptive_padding.py │ │ │ │ │ │ ├── conv_module.py │ │ │ │ │ │ ├── conv_ws.py │ │ │ │ │ │ ├── depthwise_separable_conv_module.py │ │ │ │ │ │ ├── drop.py │ │ │ │ │ │ ├── generalized_attention.py │ │ │ │ │ │ ├── hsigmoid.py │ │ │ │ │ │ ├── hswish.py │ │ │ │ │ │ ├── non_local.py │ │ │ │ │ │ ├── norm.py │ │ │ │ │ │ ├── padding.py │ │ │ │ │ │ ├── plugin.py │ │ │ │ │ │ ├── registry.py │ │ │ │ │ │ ├── scale.py │ │ │ │ │ │ ├── swish.py │ │ │ │ │ │ ├── transformer.py │ │ │ │ │ │ ├── upsample.py │ │ │ │ │ │ └── wrappers.py │ │ │ │ │ ├── builder.py │ │ │ │ │ ├── resnet.py │ │ │ │ │ ├── utils │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── flops_counter.py │ │ │ │ │ │ ├── fuse_conv_bn.py │ │ │ │ │ │ ├── sync_bn.py │ │ │ │ │ │ └── weight_init.py │ │ │ │ │ └── vgg.py │ │ │ │ ├── engine │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── test.py │ │ │ │ ├── fileio │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── file_client.py │ │ │ │ │ ├── handlers │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── base.py │ │ │ │ │ │ ├── json_handler.py │ │ │ │ │ │ ├── pickle_handler.py │ │ │ │ │ │ └── yaml_handler.py │ │ │ │ │ ├── io.py │ │ │ │ │ └── parse.py │ │ │ │ ├── image │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── colorspace.py │ │ │ │ │ ├── geometric.py │ │ │ │ │ ├── io.py │ │ │ │ │ ├── misc.py │ │ │ │ │ └── photometric.py │ │ │ │ ├── model_zoo │ │ │ │ │ ├── deprecated.json │ │ │ │ │ ├── mmcls.json │ │ │ │ │ └── open_mmlab.json │ │ │ │ ├── ops │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── assign_score_withk.py │ │ │ │ │ ├── ball_query.py │ │ │ │ │ ├── bbox.py │ │ │ │ │ ├── border_align.py │ │ │ │ │ ├── box_iou_rotated.py │ │ │ │ │ ├── carafe.py │ │ │ │ │ ├── cc_attention.py │ │ │ │ │ ├── contour_expand.py │ │ │ │ │ ├── corner_pool.py │ │ │ │ │ ├── correlation.py │ │ │ │ │ ├── deform_conv.py │ │ │ │ │ ├── deform_roi_pool.py │ │ │ │ │ ├── deprecated_wrappers.py │ │ │ │ │ ├── focal_loss.py │ │ │ │ │ ├── furthest_point_sample.py │ │ │ │ │ ├── fused_bias_leakyrelu.py │ │ │ │ │ ├── gather_points.py │ │ │ │ │ ├── group_points.py │ │ │ │ │ ├── info.py │ │ │ │ │ ├── iou3d.py │ │ │ │ │ ├── knn.py │ │ │ │ │ ├── masked_conv.py │ │ │ │ │ ├── merge_cells.py │ │ │ │ │ ├── modulated_deform_conv.py │ │ │ │ │ ├── multi_scale_deform_attn.py │ │ │ │ │ ├── nms.py │ │ │ │ │ ├── pixel_group.py │ │ │ │ │ ├── point_sample.py │ │ │ │ │ ├── points_in_boxes.py │ │ │ │ │ ├── points_sampler.py │ │ │ │ │ ├── psa_mask.py │ │ │ │ │ ├── roi_align.py │ │ │ │ │ ├── roi_align_rotated.py │ │ │ │ │ ├── roi_pool.py │ │ │ │ │ ├── roiaware_pool3d.py │ │ │ │ │ ├── roipoint_pool3d.py │ │ │ │ │ ├── saconv.py │ │ │ │ │ ├── scatter_points.py │ │ │ │ │ ├── sync_bn.py │ │ │ │ │ ├── three_interpolate.py │ │ │ │ │ ├── three_nn.py │ │ │ │ │ ├── tin_shift.py │ │ │ │ │ ├── upfirdn2d.py │ │ │ │ │ └── voxelize.py │ │ │ │ ├── parallel │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── _functions.py │ │ │ │ │ ├── collate.py │ │ │ │ │ ├── data_container.py │ │ │ │ │ ├── data_parallel.py │ │ │ │ │ ├── distributed.py │ │ │ │ │ ├── distributed_deprecated.py │ │ │ │ │ ├── registry.py │ │ │ │ │ ├── scatter_gather.py │ │ │ │ │ └── utils.py │ │ │ │ ├── runner │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── base_module.py │ │ │ │ │ ├── base_runner.py │ │ │ │ │ ├── builder.py │ │ │ │ │ ├── checkpoint.py │ │ │ │ │ ├── default_constructor.py │ │ │ │ │ ├── dist_utils.py │ │ │ │ │ ├── epoch_based_runner.py │ │ │ │ │ ├── fp16_utils.py │ │ │ │ │ ├── hooks │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── checkpoint.py │ │ │ │ │ │ ├── closure.py │ │ │ │ │ │ ├── ema.py │ │ │ │ │ │ ├── evaluation.py │ │ │ │ │ │ ├── hook.py │ │ │ │ │ │ ├── iter_timer.py │ │ │ │ │ │ ├── logger │ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ │ ├── base.py │ │ │ │ │ │ │ ├── dvclive.py │ │ │ │ │ │ │ ├── mlflow.py │ │ │ │ │ │ │ ├── neptune.py │ │ │ │ │ │ │ ├── pavi.py │ │ │ │ │ │ │ ├── tensorboard.py │ │ │ │ │ │ │ ├── text.py │ │ │ │ │ │ │ └── wandb.py │ │ │ │ │ │ ├── lr_updater.py │ │ │ │ │ │ ├── memory.py │ │ │ │ │ │ ├── momentum_updater.py │ │ │ │ │ │ ├── optimizer.py │ │ │ │ │ │ ├── profiler.py │ │ │ │ │ │ ├── sampler_seed.py │ │ │ │ │ │ └── sync_buffer.py │ │ │ │ │ ├── iter_based_runner.py │ │ │ │ │ ├── log_buffer.py │ │ │ │ │ ├── optimizer │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── builder.py │ │ │ │ │ │ └── default_constructor.py │ │ │ │ │ ├── priority.py │ │ │ │ │ └── utils.py │ │ │ │ ├── utils │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── config.py │ │ │ │ │ ├── env.py │ │ │ │ │ ├── ext_loader.py │ │ │ │ │ ├── logging.py │ │ │ │ │ ├── misc.py │ │ │ │ │ ├── parrots_jit.py │ │ │ │ │ ├── parrots_wrapper.py │ │ │ │ │ ├── path.py │ │ │ │ │ ├── progressbar.py │ │ │ │ │ ├── registry.py │ │ │ │ │ ├── testing.py │ │ │ │ │ ├── timer.py │ │ │ │ │ ├── trace.py │ │ │ │ │ └── version_utils.py │ │ │ │ ├── version.py │ │ │ │ ├── video │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── io.py │ │ │ │ │ ├── optflow.py │ │ │ │ │ └── processing.py │ │ │ │ └── visualization │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── color.py │ │ │ │ │ ├── image.py │ │ │ │ │ └── optflow.py │ │ │ ├── mmcv_custom │ │ │ │ ├── __init__.py │ │ │ │ └── checkpoint.py │ │ │ └── mmseg │ │ │ │ ├── apis │ │ │ │ ├── __init__.py │ │ │ │ ├── inference.py │ │ │ │ ├── test.py │ │ │ │ └── train.py │ │ │ │ ├── core │ │ │ │ ├── __init__.py │ │ │ │ ├── evaluation │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── class_names.py │ │ │ │ │ ├── eval_hooks.py │ │ │ │ │ └── metrics.py │ │ │ │ ├── seg │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── builder.py │ │ │ │ │ └── sampler │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── base_pixel_sampler.py │ │ │ │ │ │ └── ohem_pixel_sampler.py │ │ │ │ └── utils │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── misc.py │ │ │ │ ├── datasets │ │ │ │ ├── __init__.py │ │ │ │ ├── ade.py │ │ │ │ ├── builder.py │ │ │ │ ├── chase_db1.py │ │ │ │ ├── cityscapes.py │ │ │ │ ├── custom.py │ │ │ │ ├── dataset_wrappers.py │ │ │ │ ├── drive.py │ │ │ │ ├── hrf.py │ │ │ │ ├── pascal_context.py │ │ │ │ ├── pipelines │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── compose.py │ │ │ │ │ ├── formating.py │ │ │ │ │ ├── loading.py │ │ │ │ │ ├── test_time_aug.py │ │ │ │ │ └── transforms.py │ │ │ │ ├── stare.py │ │ │ │ └── voc.py │ │ │ │ ├── models │ │ │ │ ├── __init__.py │ │ │ │ ├── backbones │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── cgnet.py │ │ │ │ │ ├── fast_scnn.py │ │ │ │ │ ├── hrnet.py │ │ │ │ │ ├── mobilenet_v2.py │ │ │ │ │ ├── mobilenet_v3.py │ │ │ │ │ ├── resnest.py │ │ │ │ │ ├── resnet.py │ │ │ │ │ ├── resnext.py │ │ │ │ │ ├── unet.py │ │ │ │ │ ├── uniformer.py │ │ │ │ │ └── vit.py │ │ │ │ ├── builder.py │ │ │ │ ├── decode_heads │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── ann_head.py │ │ │ │ │ ├── apc_head.py │ │ │ │ │ ├── aspp_head.py │ │ │ │ │ ├── cascade_decode_head.py │ │ │ │ │ ├── cc_head.py │ │ │ │ │ ├── da_head.py │ │ │ │ │ ├── decode_head.py │ │ │ │ │ ├── dm_head.py │ │ │ │ │ ├── dnl_head.py │ │ │ │ │ ├── ema_head.py │ │ │ │ │ ├── enc_head.py │ │ │ │ │ ├── fcn_head.py │ │ │ │ │ ├── fpn_head.py │ │ │ │ │ ├── gc_head.py │ │ │ │ │ ├── lraspp_head.py │ │ │ │ │ ├── nl_head.py │ │ │ │ │ ├── ocr_head.py │ │ │ │ │ ├── point_head.py │ │ │ │ │ ├── psa_head.py │ │ │ │ │ ├── psp_head.py │ │ │ │ │ ├── sep_aspp_head.py │ │ │ │ │ ├── sep_fcn_head.py │ │ │ │ │ └── uper_head.py │ │ │ │ ├── losses │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── accuracy.py │ │ │ │ │ ├── cross_entropy_loss.py │ │ │ │ │ ├── dice_loss.py │ │ │ │ │ ├── lovasz_loss.py │ │ │ │ │ └── utils.py │ │ │ │ ├── necks │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── fpn.py │ │ │ │ │ └── multilevel_neck.py │ │ │ │ ├── segmentors │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── base.py │ │ │ │ │ ├── cascade_encoder_decoder.py │ │ │ │ │ └── encoder_decoder.py │ │ │ │ └── utils │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── drop.py │ │ │ │ │ ├── inverted_residual.py │ │ │ │ │ ├── make_divisible.py │ │ │ │ │ ├── res_layer.py │ │ │ │ │ ├── se_layer.py │ │ │ │ │ ├── self_attention_block.py │ │ │ │ │ ├── up_conv_block.py │ │ │ │ │ └── weight_init.py │ │ │ │ ├── ops │ │ │ │ ├── __init__.py │ │ │ │ ├── encoding.py │ │ │ │ └── wrappers.py │ │ │ │ └── utils │ │ │ │ ├── __init__.py │ │ │ │ ├── collect_env.py │ │ │ │ └── logger.py │ │ └── util.py │ ├── canny2image_TRT.py │ ├── cldm │ │ ├── cldm.py │ │ ├── ddim_hacked.py │ │ ├── hack.py │ │ ├── logger.py │ │ └── model.py │ ├── compute_score.py │ ├── config.py │ ├── docs │ │ ├── annotator.md │ │ ├── faq.md │ │ ├── low_vram.md │ │ └── train.md │ ├── environment.yaml │ ├── export_onnx.py │ ├── font │ │ └── DejaVuSans.ttf │ ├── gradio_annotator.py │ ├── gradio_canny2image.py │ ├── gradio_depth2image.py │ ├── gradio_fake_scribble2image.py │ ├── gradio_hed2image.py │ ├── gradio_hough2image.py │ ├── gradio_normal2image.py │ ├── gradio_pose2image.py │ ├── gradio_scribble2image.py │ ├── gradio_scribble2image_interactive.py │ ├── gradio_seg2image.py │ ├── infer_cudagraph.py │ ├── ldm │ │ ├── data │ │ │ ├── __init__.py │ │ │ └── util.py │ │ ├── models │ │ │ ├── autoencoder.py │ │ │ └── diffusion │ │ │ │ ├── __init__.py │ │ │ │ ├── ddim.py │ │ │ │ ├── ddpm.py │ │ │ │ ├── dpm_solver │ │ │ │ ├── __init__.py │ │ │ │ ├── dpm_solver.py │ │ │ │ └── sampler.py │ │ │ │ ├── plms.py │ │ │ │ └── sampling_util.py │ │ ├── modules │ │ │ ├── attention.py │ │ │ ├── diffusionmodules │ │ │ │ ├── __init__.py │ │ │ │ ├── model.py │ │ │ │ ├── openaimodel.py │ │ │ │ ├── upscaling.py │ │ │ │ └── util.py │ │ │ ├── distributions │ │ │ │ ├── __init__.py │ │ │ │ └── distributions.py │ │ │ ├── ema.py │ │ │ ├── encoders │ │ │ │ ├── __init__.py │ │ │ │ └── modules.py │ │ │ ├── image_degradation │ │ │ │ ├── __init__.py │ │ │ │ ├── bsrgan.py │ │ │ │ ├── bsrgan_light.py │ │ │ │ ├── utils │ │ │ │ │ └── test.png │ │ │ │ └── utils_image.py │ │ │ └── midas │ │ │ │ ├── __init__.py │ │ │ │ ├── api.py │ │ │ │ ├── midas │ │ │ │ ├── __init__.py │ │ │ │ ├── base_model.py │ │ │ │ ├── blocks.py │ │ │ │ ├── dpt_depth.py │ │ │ │ ├── midas_net.py │ │ │ │ ├── midas_net_custom.py │ │ │ │ ├── transforms.py │ │ │ │ └── vit.py │ │ │ │ └── utils.py │ │ └── util.py │ ├── models │ │ ├── cldm_v15.yaml │ │ └── cldm_v21.yaml │ ├── plugin │ │ ├── CMakeLists.txt │ │ ├── common │ │ │ ├── bertCommon.h │ │ │ ├── checkMacrosPlugin.cpp │ │ │ ├── checkMacrosPlugin.h │ │ │ ├── common.cuh │ │ │ ├── cub_helper.h │ │ │ ├── cudaDriverWrapper.cpp │ │ │ ├── cudaDriverWrapper.h │ │ │ ├── dimsHelpers.h │ │ │ ├── half.h │ │ │ ├── plugin.cpp │ │ │ ├── plugin.h │ │ │ ├── reducedMathPlugin.cpp │ │ │ └── serialize.hpp │ │ └── groupNormPlugin │ │ │ ├── groupNormKernel.cu │ │ │ ├── groupNormKernel.h │ │ │ ├── groupNormPlugin.cpp │ │ │ ├── groupNormPlugin.h │ │ │ └── groupNormPluginCommon.h │ ├── preprocess.sh │ ├── share.py │ ├── surgeon_graph.py │ ├── test_imgs │ │ ├── bag.png │ │ ├── bag_scribble.png │ │ ├── bird.png │ │ ├── boy.png │ │ ├── building.png │ │ ├── building2.png │ │ ├── cute_robot.png │ │ ├── cyber.png │ │ ├── dog.png │ │ ├── dog2.png │ │ ├── house.png │ │ ├── house_line.png │ │ ├── human.png │ │ ├── human_line.png │ │ ├── man.png │ │ ├── old.png │ │ ├── pose1.png │ │ ├── pose2.png │ │ ├── room.png │ │ ├── sd.png │ │ ├── shose.png │ │ ├── toy.png │ │ ├── user_1.png │ │ └── user_3.png │ ├── tool_add_control.py │ ├── tool_add_control_sd21.py │ ├── tool_transfer_control.py │ ├── tutorial_dataset.py │ ├── tutorial_dataset_test.py │ ├── tutorial_train.py │ └── tutorial_train_sd21.py ├── final_round.md ├── preliminary_round.md └── questions.md ├── LICENSE ├── README.md ├── cookbook ├── .clang-format ├── .gitattributes ├── .gitignore ├── .pre-commit-config.yaml ├── .style.yapf ├── 00-Data │ ├── README.md │ ├── data │ │ └── .gitignore │ ├── extract_MNIST.py │ ├── get_model_part1.py │ ├── get_model_part2.py │ ├── model │ │ └── .gitkeep │ └── unit_test.sh ├── 01-SimpleDemo │ ├── README.md │ ├── TensorRT-10 │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.cpp │ │ ├── main.h │ │ ├── main_cookbook_flavor.py │ │ ├── main_numpy.py │ │ ├── main_pytorch.py │ │ └── unit_test.sh │ ├── TensorRT-6 │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.cpp │ │ └── main.py │ ├── TensorRT-7 │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.cpp │ │ └── main.py │ ├── TensorRT-8.0 │ │ ├── Makefile │ │ ├── README.md │ │ ├── main-cuda.py │ │ ├── main-cudart.py │ │ └── main.cpp │ ├── TensorRT-8.6 │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.cpp │ │ └── main.py │ ├── build-README.py │ └── unit_test.sh ├── 02-API │ ├── Builder │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── BuilderConfig │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── CudaEngine │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── ExecutionContext │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── HostMemory │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── Layer │ │ ├── Activation │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Assertion │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Cast │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Concatenation │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Condition │ │ │ ├── README.md │ │ │ └── unit_test.sh │ │ ├── Constant │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Convolution │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Cumulative │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Deconvolution │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Dequantize │ │ │ ├── README.md │ │ │ └── unit_test.sh │ │ ├── DynamicQuantize │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Einsum │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Elementwise │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Fill │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Gather │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── GridSample │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Identity │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── IfConditionStructure │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── IfConditionalInput │ │ │ ├── README.md │ │ │ └── unit_test.sh │ │ ├── IfConditionalOutput │ │ │ ├── README.md │ │ │ └── unit_test.sh │ │ ├── Iterator │ │ │ ├── README.md │ │ │ └── unit_test.sh │ │ ├── LRN │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── LoopOutput │ │ │ ├── README.md │ │ │ └── unit_test.sh │ │ ├── LoopStructure │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── MatrixMultiply │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── NMS │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── NonZero │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Normalization │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── OneHot │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Padding │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── ParametricReLU │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Plugin │ │ │ ├── README.md │ │ │ └── unit_test.sh │ │ ├── PluginV2 │ │ │ ├── README.md │ │ │ └── unit_test.sh │ │ ├── PluginV3 │ │ │ ├── README.md │ │ │ └── unit_test.sh │ │ ├── Pooling │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── QDQStructure │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Quantize │ │ │ ├── README.md │ │ │ └── unit_test.sh │ │ ├── README.md │ │ ├── RaggedSoftMax │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Recurrence │ │ │ ├── README.md │ │ │ └── unit_test.sh │ │ ├── Reduce │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Resize │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── ReverseSequence │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Scale │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Scatter │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Select │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Shape │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Shuffle │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Slice │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Softmax │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Squeeze │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── TopK │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── TripLimit │ │ │ ├── README.md │ │ │ └── unit_test.sh │ │ ├── Unary │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── Unsqueeze │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ ├── main.py │ │ ├── template │ │ │ ├── README.md │ │ │ ├── main.py │ │ │ └── unit_test.sh │ │ └── unit_test.sh │ ├── Network │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── ONNXParser │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── OptimizationProfile │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── README.md │ ├── Runtime │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── Tensor │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── build-README.py │ └── unit_test.sh ├── 03-Workflow │ ├── Paddlepaddle-ONNX-TensorRT │ │ ├── .gitignore │ │ ├── README.md │ │ ├── main.py │ │ ├── requirements.txt │ │ └── unit_test.sh │ ├── README.md │ ├── TensorFlow2-ONNX-TensorRT │ │ ├── .gitignore │ │ ├── README.md │ │ ├── main.py │ │ ├── requirements.txt │ │ └── unit_test.sh │ ├── build-README.py │ ├── pyTorch-ONNX-TensorRT │ │ ├── C++ │ │ │ ├── Makefile │ │ │ └── main.cpp │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── pyTorch-TensorRT │ │ ├── C++ │ │ │ ├── Makefile │ │ │ └── main.cpp │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ └── unit_test.sh ├── 04-Feature │ ├── AlgorithmSelector │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── AuxStream │ │ ├── README.md │ │ ├── main.py │ │ ├── max_aux_streams=0.png │ │ ├── max_aux_streams=1.png │ │ ├── max_aux_streams=2.png │ │ └── unit_test.sh │ ├── BuilderOptimizationLevel │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── DebugTensor │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── EngineInspector │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── ErrorRecorder │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── ExecutionContextAllocationStrategy │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── GPUAllocator │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── HardwareCompatibility │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── LabeledDimension │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── LeanAndDispatchRuntime │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── Logger │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── OutputAllocator │ │ ├── README.md │ │ └── unit_test.sh │ ├── Profiler │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── ProfilingVerbosity │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── ProgressMonitor │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── README.md │ ├── Refit │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── Safety │ │ ├── README.md │ │ └── unit_test.sh │ ├── SerializationConfig │ │ ├── README.md │ │ └── unit_test.sh │ ├── Sparsity │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── TimingCache │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── VersionCompatibility │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── WeightStreaming │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── build-README.py │ └── unit_test.sh ├── 05-Plugin │ ├── APIs-V2-deprecated │ │ ├── AddScalarPlugin.cu │ │ ├── AddScalarPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.cpp │ │ ├── main.py │ │ └── unit_test.sh │ ├── APIs │ │ ├── AddScalarPlugin.cu │ │ ├── AddScalarPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.cpp │ │ ├── main.py │ │ └── unit_test.sh │ ├── BasicExample-V2-deprecated │ │ ├── AddScalarPlugin.cu │ │ ├── AddScalarPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.cpp │ │ ├── main.py │ │ └── unit_test.sh │ ├── BasicExample │ │ ├── AddScalarPlugin.cu │ │ ├── AddScalarPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.cpp │ │ ├── main.py │ │ └── unit_test.sh │ ├── DataDependentShape │ │ ├── Makefile │ │ ├── PushLeftPlugin.cu │ │ ├── PushLeftPlugin.h │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── IdentityPlugin │ │ ├── IdentityPlugin.cu │ │ ├── IdentityPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── InPlacePlugin │ │ ├── AddScalarPlugin.cu │ │ ├── AddScalarPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── MultiVersion │ │ ├── AddScalarPlugin.cu │ │ ├── AddScalarPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── ONNXParserWithPlugin │ │ ├── AddScalarPlugin.cu │ │ ├── AddScalarPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── PassHostData │ │ ├── Makefile │ │ ├── PassHostDataPlugin.cu │ │ ├── PassHostDataPlugin.h │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── PluginInsideEngine-C++ │ │ ├── AddScalarPlugin.cu │ │ ├── AddScalarPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.cpp │ │ └── unit_test.sh │ ├── PluginInsideEngine-Python │ │ ├── AddScalarPlugin.cu │ │ ├── AddScalarPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── PythonPlugin │ │ ├── README.md │ │ ├── add_scalar_cuda_python-V2-deprecated.py │ │ ├── add_scalar_cuda_python.py │ │ ├── add_scalar_cupy.py │ │ ├── add_scalar_numba.py │ │ ├── add_scalar_torch.py │ │ ├── add_scalar_triton.py │ │ └── unit_test.sh │ ├── QuickDeployablePlugin │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── README.md │ ├── ShapeInputTensor │ │ ├── Makefile │ │ ├── MyReshapePlugin.cu │ │ ├── MyReshapePlugin.h │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── Tactic+TimingCache │ │ ├── AddScalarPlugin.cu │ │ ├── AddScalarPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── UseCuBLAS-V2-deprecated │ │ ├── CuBLASGemmPlugin.cu │ │ ├── CuBLASGemmPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── cuBLASAlone.cu │ │ ├── main.py │ │ └── unit_test.sh │ ├── UseCuBLAS │ │ ├── CuBLASGemmPlugin.cu │ │ ├── CuBLASGemmPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.py │ │ ├── unit_test.sh │ │ └── useCuBLASAlone.cpp │ ├── UseFP16 │ │ ├── AddScalarPlugin.cu │ │ ├── AddScalarPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── UseINT8-PTQ-V2-deprecated │ │ ├── AddScalarPlugin.cu │ │ ├── AddScalarPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.py │ │ ├── model.int8cache │ │ └── unit_test.sh │ ├── UseINT8-PTQ-error │ │ ├── AddScalarPlugin.cu │ │ ├── AddScalarPlugin.h │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.py │ │ ├── model.int8cache │ │ └── unit_test.sh │ ├── build-README.py │ └── unit_test.sh ├── 06-DLFrameworkTRT │ ├── README.md │ ├── Torch-TensorRT │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── build-README.py │ └── unit_test.sh ├── 07-Tool │ ├── CheckTorchOperator │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── ContextPrinter │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── EnginePrinter │ │ ├── README.md │ │ ├── main.py │ │ ├── main.sh │ │ └── unit_test.sh │ ├── FP16Tuning │ │ ├── .gitignore │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── ListAPIs │ │ ├── .gitignore │ │ ├── README.md │ │ ├── log-main.py.log │ │ ├── main.py │ │ ├── result-TesnorRT-10.0.1.log │ │ ├── result-TesnorRT-10.1.0.log │ │ ├── result-TesnorRT-10.10.0.31.log │ │ ├── result-TesnorRT-10.11.0.33.log │ │ ├── result-TesnorRT-10.2.0.post1.log │ │ ├── result-TesnorRT-10.3.0.log │ │ ├── result-TesnorRT-10.4.0.log │ │ ├── result-TesnorRT-10.5.0.log │ │ ├── result-TesnorRT-10.6.0.post1.log │ │ ├── result-TesnorRT-10.7.0.log │ │ ├── result-TesnorRT-10.8.0.43.log │ │ ├── result-TesnorRT-10.9.0.34.log │ │ ├── result-TesnorRT-8.6.3.log │ │ ├── result-TesnorRT-9.3.0.post12.dev1.log │ │ └── unit_test.sh │ ├── Netron │ │ ├── README.md │ │ └── unit_test.sh │ ├── NetworkPrinter │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.cpp │ │ ├── main.py │ │ └── unit_test.sh │ ├── NetworkSerialization │ │ ├── C++-TODO │ │ │ ├── Makefile │ │ │ ├── Makefile.inc │ │ │ ├── NetworkInspector.hpp │ │ │ ├── cnpy.cpp │ │ │ ├── cnpy.h │ │ │ ├── cookbookHelper.hpp │ │ │ ├── json.hpp │ │ │ ├── json_fwd.hpp │ │ │ └── main.cpp │ │ ├── README.md │ │ ├── TRT-8-version │ │ │ ├── NetworkInspector.py │ │ │ ├── NetworkRebuilder.py │ │ │ ├── UnitTestTemplate.py │ │ │ ├── UnitTestTemplate.py.txt │ │ │ ├── main.py │ │ │ └── note.md │ │ ├── main.py │ │ ├── unit_test.sh │ │ ├── unit_test_cases.py │ │ └── unit_tests.py │ ├── NsightSystems │ │ ├── Help-analyze.txt │ │ ├── Help-cancel.txt │ │ ├── Help-export.txt │ │ ├── Help-launch.txt │ │ ├── Help-nvprof.txt │ │ ├── Help-profile.txt │ │ ├── Help-recipe.txt │ │ ├── Help-service.txt │ │ ├── Help-sessions-list.txt │ │ ├── Help-shutdown.txt │ │ ├── Help-stats.txt │ │ ├── Help-stop.txt │ │ ├── Help.txt │ │ ├── README.md │ │ ├── main.sh │ │ └── unit_test.sh │ ├── ONNX-TODO │ │ ├── README.md │ │ └── unit_test.sh │ ├── OnnxGraphSurgeon │ │ ├── 01-create_model.py │ │ ├── 02-add_node.py │ │ ├── 03-remove_node.py │ │ ├── 04-replace_node.py │ │ ├── 05-print_graph.py │ │ ├── 06-fold.py │ │ ├── 07-shape_operation_and_simplify.py │ │ ├── 08-isolate_subgraph.py │ │ ├── 09-build_model_with_API.py │ │ ├── 10-advance_API.py │ │ ├── API │ │ │ ├── graphAPI.py │ │ │ └── gsAPI-TODO.py │ │ ├── README.md │ │ ├── main.sh │ │ └── unit_test.sh │ ├── OnnxWeightProcess │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── Onnxruntime │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── Polygraphy │ │ ├── API │ │ │ ├── README.md │ │ │ ├── gs_workflow.py │ │ │ ├── main.py │ │ │ ├── main.sh │ │ │ └── unit_test.sh │ │ ├── Check │ │ │ ├── Help-check-lint.txt │ │ │ ├── Help-check.txt │ │ │ ├── README.md │ │ │ ├── main.sh │ │ │ └── unit_test.sh │ │ ├── Convert │ │ │ ├── Help-convert.txt │ │ │ ├── README.md │ │ │ ├── data_loader.py │ │ │ ├── main.sh │ │ │ └── unit_test.sh │ │ ├── Data │ │ │ ├── Help-data-to-input.txt │ │ │ ├── Help-data.txt │ │ │ ├── README.md │ │ │ ├── main.sh │ │ │ └── unit_test.sh │ │ ├── Debug │ │ │ ├── Help-debug-build.txt │ │ │ ├── Help-debug-precision.txt │ │ │ ├── Help-debug-reduce.txt │ │ │ ├── Help-debug-repeat.txt │ │ │ ├── Help-debug.txt │ │ │ ├── README.md │ │ │ ├── main.sh │ │ │ └── unit_test.sh │ │ ├── Help.txt │ │ ├── Inspect │ │ │ ├── .gitignore │ │ │ ├── Help-inspect-capability.txt │ │ │ ├── Help-inspect-data.txt │ │ │ ├── Help-inspect-diff-tactics.txt │ │ │ ├── Help-inspect-model.txt │ │ │ ├── Help-inspect-sparsity.txt │ │ │ ├── Help-inspect-tactics.txt │ │ │ ├── Help-inspect.txt │ │ │ ├── README.md │ │ │ ├── main.sh │ │ │ └── unit_test.sh │ │ ├── Plugin-Unfini │ │ │ ├── Help-plugin-list.txt │ │ │ ├── Help-plugin-match.txt │ │ │ ├── Help-plugin-replace.txt │ │ │ ├── Help-plugin.txt │ │ │ ├── README.md │ │ │ ├── main.sh │ │ │ ├── match_and_replace_plug │ │ │ │ ├── README.md │ │ │ │ └── plugins │ │ │ │ │ └── toyPlugin │ │ │ │ │ └── pattern.py │ │ │ └── unit_test.sh │ │ ├── README.md │ │ ├── Run │ │ │ ├── .gitignore │ │ │ ├── Help-run.txt │ │ │ ├── README.md │ │ │ ├── main.sh │ │ │ └── unit_test.sh │ │ ├── Surgeon │ │ │ ├── Help-surgeon-extract.txt │ │ │ ├── Help-surgeon-insert.txt │ │ │ ├── Help-surgeon-prune.txt │ │ │ ├── Help-surgeon-sanitize.txt │ │ │ ├── Help-surgeon.txt │ │ │ ├── README.md │ │ │ ├── main.sh │ │ │ └── unit_test.sh │ │ ├── Template │ │ │ ├── .gitignore │ │ │ ├── Help-template-onnx-gs.txt │ │ │ ├── Help-template-trt-config.txt │ │ │ ├── Help-template-trt-network.txt │ │ │ ├── Help-template.txt │ │ │ ├── README.md │ │ │ ├── main.sh │ │ │ └── unit_test.sh │ │ └── unit_test.sh │ ├── PolygraphyExtensionTrtexec │ │ ├── README.md │ │ └── unit_test.sh │ ├── README.md │ ├── TRTEngineExplorer │ │ ├── README.md │ │ └── unit_test.sh │ ├── TriPy │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── build-README.py │ ├── nvtx │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.cpp │ │ ├── main.png │ │ ├── main.py │ │ └── unit_test.sh │ ├── trtexec │ │ ├── Help.txt │ │ ├── README.md │ │ ├── main.sh │ │ └── unit_test.sh │ └── unit_test.sh ├── 08-Advance │ ├── C++StaticCompilation │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.cpp │ │ └── unit_test.sh │ ├── CudaGraph │ │ ├── Makefile │ │ ├── README.md │ │ ├── main copy.py │ │ ├── main.cpp │ │ ├── main.py │ │ └── unit_test.sh │ ├── DataFormat │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── EmptyTensor-Unfini │ │ ├── README.md │ │ └── unit_test.sh │ ├── MultiContext │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── MultiOptimizationProfile │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── MultiStream │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── README.md │ ├── StreamAndAsync │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.cpp │ │ ├── main.py │ │ └── unit_test.sh │ ├── Subgraph-Unfini │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── build-README.py │ └── unit_test.sh ├── 09-TRTLLM │ ├── README.md │ ├── build-README.py │ └── unit_test.sh ├── 98-Uncategorized │ ├── DeviceInfo │ │ ├── README.md │ │ ├── main.py │ │ ├── output │ │ │ ├── 3070.txt │ │ │ ├── 3080.txt │ │ │ ├── 3090.txt │ │ │ ├── 4090.txt │ │ │ ├── 5000-Ada.txt │ │ │ ├── 6000-Ada.txt │ │ │ ├── A10.txt │ │ │ ├── A100-PCIE-40GB.txt │ │ │ ├── A100-PCIE-80GB.txt │ │ │ ├── A100-SXM4-80GB.txt │ │ │ ├── A16.txt │ │ │ ├── A2.txt │ │ │ ├── A30.txt │ │ │ ├── A40.txt │ │ │ ├── A6000.txt │ │ │ ├── A800.txt │ │ │ ├── B100.txt │ │ │ ├── B200.txt │ │ │ ├── GP100.txt │ │ │ ├── H100 HBM3.txt │ │ │ ├── H100 PCIe.txt │ │ │ ├── H200.txt │ │ │ ├── L2.txt │ │ │ ├── L20.txt │ │ │ ├── L4.txt │ │ │ ├── L40.txt │ │ │ ├── L40S.txt │ │ │ ├── P100-PCIE.txt │ │ │ ├── RTX-8000.txt │ │ │ ├── T4.txt │ │ │ ├── V100-PCIE-32GB.txt │ │ │ └── V100-SXM3-32GB.txt │ │ └── unit_test.sh │ ├── LibraryInfo │ │ ├── README.md │ │ ├── main.py │ │ └── unit_test.sh │ ├── NpyAndNpz │ │ ├── Makefile │ │ ├── README.md │ │ ├── cnpy.h │ │ ├── get_data.py │ │ ├── main.cpp │ │ └── unit_test.sh │ ├── Number │ │ ├── README.md │ │ ├── build_number_md.py │ │ ├── build_number_picture.py │ │ ├── output │ │ │ ├── BF16.md │ │ │ ├── DataType.png │ │ │ ├── FP16.md │ │ │ ├── FP32.md │ │ │ ├── FP4e0m3(NVFP4).md │ │ │ ├── FP4e2m1(MXFP4).md │ │ │ ├── FP64.md │ │ │ ├── FP6e2m3.md │ │ │ ├── FP6e3m2.md │ │ │ ├── FP8e4m3.md │ │ │ ├── FP8e5m2.md │ │ │ ├── FP8e8m0.md │ │ │ ├── Integer.md │ │ │ └── TF32.md │ │ └── unit_test.sh │ ├── README.md │ ├── build-README.py │ └── unit_test.sh ├── 99-Todo │ └── README.md ├── README.md ├── build-README.py ├── copyright.py ├── include │ ├── Makefile.inc │ ├── calibrator.cpp │ ├── calibrator.h │ ├── cnpy.cpp │ ├── cnpy.h │ ├── cookbookHelper.cu │ └── cookbookHelper.cuh ├── requirement-dev.txt ├── requirements.txt ├── setup.py ├── tensorrt_cookbook │ ├── __init__.py │ ├── utils.py │ ├── utils_class.py │ ├── utils_cookbook.py │ ├── utils_function.py │ ├── utils_network.py │ ├── utils_network_serialization.py │ ├── utils_onnx.py │ └── version.py └── unit_test.sh └── old ├── Makefile ├── README.md ├── include ├── Logger.h ├── TrtLite.h └── Utils.h ├── plugins ├── AddPlugin.cu ├── AddPlugin.h ├── AddPluginDyn.cu ├── AddPluginDyn.h ├── GatherND.cu ├── GatherND.h ├── GridSamplerPlugin.cpp ├── OnehotPlugin.cu ├── OnehotPlugin.h ├── ScatterND.cu └── ScatterND.h ├── python ├── app_GatherND_plugin.py ├── app_Onehot_plugin.py ├── app_Onehot_plugin_TF.py ├── app_ScatterND_plugin.py ├── app_basic.py ├── app_decode_inference.py ├── app_onnx_custom.py ├── app_onnx_resnet50.py ├── app_plugin.py ├── trt_lite.py └── trt_lite2.py └── samples ├── AppBasic.cpp ├── AppDynamicShape.cpp ├── AppInt8.cpp ├── AppLoadRefit.cpp ├── AppMultiContext.cpp ├── AppOnnx.cpp ├── AppPlugin.cpp ├── AppPluginDynamicShape.cpp ├── AppShapeLayer.cpp └── AppThroughput.cpp /.gitattributes: -------------------------------------------------------------------------------- 1 | Hackathon2022/code/encoder.onnx filter=lfs diff=lfs merge=lfs -text 2 | Hackathon2022/code/decoder.onnx filter=lfs diff=lfs merge=lfs -text 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | Notes.txt 2 | compile_commands.json 3 | old/build/* 4 | old/python/*.onnx 5 | old/python/*.trt 6 | old/python/*.qdrep 7 | old/python/*.pb 8 | .cache 9 | .vscode/ 10 | __pycache__ 11 | -------------------------------------------------------------------------------- /Hackathon2022/code/.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | -------------------------------------------------------------------------------- /Hackathon2022/code/AttentionPlugin/layerKernels.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "utils.h" 4 | 5 | template 6 | __global__ void transpose102_addBias(T *dst, T *q_u, T *q_v, const T *src, const T *bias, const T *bias_u, const T *bias_v, const int batch_size, const int time1, const int off0, const int i_off1, const int o_off1, const int off2); 7 | 8 | template 9 | __global__ void add_ac_bd_mask(T *dst, const T *ac, const T *bd, const int *mask, const T sqrt_d_k, const int off, const int offset, const int time2); 10 | 11 | template 12 | __global__ void transpose102_res(T *dst, T *src, T *res, T *bias, const int off0, const int i_off1, const int o_off1, const int off2, const int time1); 13 | 14 | template 15 | void invokeSoftmax(T *dst, T *src, const int batch_size, const int head_num, const int time1, const int time2, cudaStream_t stream); 16 | -------------------------------------------------------------------------------- /Hackathon2022/code/README.md: -------------------------------------------------------------------------------- 1 | # TensorRT Hackathon 2022 初赛试题参考实现 2 | 3 | ## 运行方法 4 | + 使用 docker image：nvcr.io/nvidia/tensorrt:22.07-py3 5 | ```shell 6 | pip install -r requirements.txt 7 | chmod +x ./build.sh 8 | ./build.sh 9 | ``` 10 | -------------------------------------------------------------------------------- /Hackathon2022/code/decoder.onnx: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:28b1826c62cf8ecef353be4aa79f1ae87bc60304f42d4e367be9f89f513b8c5c 3 | size 51761485 4 | -------------------------------------------------------------------------------- /Hackathon2022/code/encoder.onnx: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:7efefc1dcb1d79f313e246a8a2b8fe8e70d12a67298b8681c51c95b05cc06aa4 3 | size 143432736 4 | -------------------------------------------------------------------------------- /Hackathon2022/code/requirements.txt: -------------------------------------------------------------------------------- 1 | nvidia-pyindex 2 | onnx 3 | onnxruntime-gpu 4 | onnx_graphsurgeon --index-url https://pypi.ngc.nvidia.com 5 | polygraphy 6 | cuda-python 7 | colored 8 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | This is a [ControlNet 1.0](https://github.com/lllyasviel/ControlNet) canny2image pipeline optimized by NVIDIA TensorRT. 4 | 5 | # Setup 6 | ## Environment 7 | ```registry.cn-hangzhou.aliyuncs.com/trt-hackathon/trt-hackathon:v2``` 8 | ## run 9 | Export torch models and convert them to TensorRT engine. 10 | 11 | ```bash preprocess.sh``` 12 | 13 | Generate images. 14 | 15 | ```python3 compute_score.py``` 16 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/canny/__init__.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | 3 | 4 | class CannyDetector: 5 | 6 | def __call__(self, img, low_threshold, high_threshold): 7 | return cv2.Canny(img, low_threshold, high_threshold) 8 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/ckpts/ckpts.txt: -------------------------------------------------------------------------------- 1 | Weights here. 2 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/midas/midas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/annotator/midas/midas/__init__.py -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/midas/midas/base_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class BaseModel(torch.nn.Module): 5 | 6 | def load(self, path): 7 | """Load model from file. 8 | 9 | Args: 10 | path (str): file path 11 | """ 12 | parameters = torch.load(path, map_location=torch.device('cpu')) 13 | 14 | if "optimizer" in parameters: 15 | parameters = parameters["model"] 16 | 17 | self.load_state_dict(parameters) 18 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/configs/_base_/datasets/pascal_voc12_aug.py: -------------------------------------------------------------------------------- 1 | _base_ = './pascal_voc12.py' 2 | # dataset settings 3 | data = dict(train=dict(ann_dir=['SegmentationClass', 'SegmentationClassAug'], 4 | split=[ 5 | 'ImageSets/Segmentation/train.txt', 6 | 'ImageSets/Segmentation/aug.txt' 7 | ])) 8 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/configs/_base_/default_runtime.py: -------------------------------------------------------------------------------- 1 | # yapf:disable 2 | log_config = dict( 3 | interval=50, 4 | hooks=[ 5 | dict(type='TextLoggerHook', by_epoch=False), 6 | # dict(type='TensorboardLoggerHook') 7 | ]) 8 | # yapf:enable 9 | dist_params = dict(backend='nccl') 10 | log_level = 'INFO' 11 | load_from = None 12 | resume_from = None 13 | workflow = [('train', 1)] 14 | cudnn_benchmark = True 15 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/configs/_base_/schedules/schedule_160k.py: -------------------------------------------------------------------------------- 1 | # optimizer 2 | optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) 3 | optimizer_config = dict() 4 | # learning policy 5 | lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) 6 | # runtime settings 7 | runner = dict(type='IterBasedRunner', max_iters=160000) 8 | checkpoint_config = dict(by_epoch=False, interval=16000) 9 | evaluation = dict(interval=16000, metric='mIoU') 10 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/configs/_base_/schedules/schedule_20k.py: -------------------------------------------------------------------------------- 1 | # optimizer 2 | optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) 3 | optimizer_config = dict() 4 | # learning policy 5 | lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) 6 | # runtime settings 7 | runner = dict(type='IterBasedRunner', max_iters=20000) 8 | checkpoint_config = dict(by_epoch=False, interval=2000) 9 | evaluation = dict(interval=2000, metric='mIoU') 10 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/configs/_base_/schedules/schedule_40k.py: -------------------------------------------------------------------------------- 1 | # optimizer 2 | optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) 3 | optimizer_config = dict() 4 | # learning policy 5 | lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) 6 | # runtime settings 7 | runner = dict(type='IterBasedRunner', max_iters=40000) 8 | checkpoint_config = dict(by_epoch=False, interval=4000) 9 | evaluation = dict(interval=4000, metric='mIoU') 10 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/configs/_base_/schedules/schedule_80k.py: -------------------------------------------------------------------------------- 1 | # optimizer 2 | optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) 3 | optimizer_config = dict() 4 | # learning policy 5 | lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) 6 | # runtime settings 7 | runner = dict(type='IterBasedRunner', max_iters=80000) 8 | checkpoint_config = dict(by_epoch=False, interval=8000) 9 | evaluation = dict(interval=8000, metric='mIoU') 10 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/exp/upernet_global_small/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | work_path=$(dirname $0) 4 | PYTHONPATH="$(dirname $0)/../../":$PYTHONPATH \ 5 | python -m torch.distributed.launch --nproc_per_node=8 \ 6 | tools/train.py ${work_path}/config.py \ 7 | --launcher pytorch \ 8 | --options model.backbone.pretrained_path='your_model_path/uniformer_small_in1k.pth' \ 9 | --work-dir ${work_path}/ckpt \ 10 | 2>&1 | tee -a ${work_path}/log.txt 11 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/exp/upernet_global_small/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | work_path=$(dirname $0) 4 | PYTHONPATH="$(dirname $0)/../../":$PYTHONPATH \ 5 | python -m torch.distributed.launch --nproc_per_node=8 \ 6 | tools/test.py ${work_path}/test_config_h32.py \ 7 | ${work_path}/ckpt/latest.pth \ 8 | --launcher pytorch \ 9 | --eval mIoU \ 10 | 2>&1 | tee -a ${work_path}/log.txt 11 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | # flake8: noqa 3 | 4 | # The following modules are not imported to this level, so mmcv may be used 5 | # without PyTorch. 6 | # - runner 7 | # - parallel 8 | # - op 9 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/arraymisc/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .quantization import dequantize, quantize 3 | 4 | __all__ = ['quantize', 'dequantize'] 5 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/cnn/bricks/hswish.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import torch.nn as nn 3 | 4 | from .registry import ACTIVATION_LAYERS 5 | 6 | 7 | @ACTIVATION_LAYERS.register_module() 8 | class HSwish(nn.Module): 9 | """Hard Swish Module. 10 | 11 | This module applies the hard swish function: 12 | 13 | .. math:: 14 | Hswish(x) = x * ReLU6(x + 3) / 6 15 | 16 | Args: 17 | inplace (bool): can optionally do the operation in-place. 18 | Default: False. 19 | 20 | Returns: 21 | Tensor: The output tensor. 22 | """ 23 | 24 | def __init__(self, inplace=False): 25 | super(HSwish, self).__init__() 26 | self.act = nn.ReLU6(inplace) 27 | 28 | def forward(self, x): 29 | return x * self.act(x + 3) / 6 30 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/cnn/bricks/registry.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from annotator.uniformer.mmcv.utils import Registry 3 | 4 | CONV_LAYERS = Registry('conv layer') 5 | NORM_LAYERS = Registry('norm layer') 6 | ACTIVATION_LAYERS = Registry('activation layer') 7 | PADDING_LAYERS = Registry('padding layer') 8 | UPSAMPLE_LAYERS = Registry('upsample layer') 9 | PLUGIN_LAYERS = Registry('plugin layer') 10 | 11 | DROPOUT_LAYERS = Registry('drop out layers') 12 | POSITIONAL_ENCODING = Registry('position encoding') 13 | ATTENTION = Registry('attention') 14 | FEEDFORWARD_NETWORK = Registry('feed-forward Network') 15 | TRANSFORMER_LAYER = Registry('transformerLayer') 16 | TRANSFORMER_LAYER_SEQUENCE = Registry('transformer-layers sequence') 17 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/cnn/bricks/scale.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import torch 3 | import torch.nn as nn 4 | 5 | 6 | class Scale(nn.Module): 7 | """A learnable scale parameter. 8 | 9 | This layer scales the input by a learnable factor. It multiplies a 10 | learnable scale parameter of shape (1,) with input of any shape. 11 | 12 | Args: 13 | scale (float): Initial value of scale factor. Default: 1.0 14 | """ 15 | 16 | def __init__(self, scale=1.0): 17 | super(Scale, self).__init__() 18 | self.scale = nn.Parameter(torch.tensor(scale, dtype=torch.float)) 19 | 20 | def forward(self, x): 21 | return x * self.scale 22 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/cnn/bricks/swish.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import torch 3 | import torch.nn as nn 4 | 5 | from .registry import ACTIVATION_LAYERS 6 | 7 | 8 | @ACTIVATION_LAYERS.register_module() 9 | class Swish(nn.Module): 10 | """Swish Module. 11 | 12 | This module applies the swish function: 13 | 14 | .. math:: 15 | Swish(x) = x * Sigmoid(x) 16 | 17 | Returns: 18 | Tensor: The output tensor. 19 | """ 20 | 21 | def __init__(self): 22 | super(Swish, self).__init__() 23 | 24 | def forward(self, x): 25 | return x * torch.sigmoid(x) 26 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/engine/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .test import (collect_results_cpu, collect_results_gpu, multi_gpu_test, 3 | single_gpu_test) 4 | 5 | __all__ = [ 6 | 'collect_results_cpu', 'collect_results_gpu', 'multi_gpu_test', 7 | 'single_gpu_test' 8 | ] 9 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/fileio/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .file_client import BaseStorageBackend, FileClient 3 | from .handlers import BaseFileHandler, JsonHandler, PickleHandler, YamlHandler 4 | from .io import dump, load, register_handler 5 | from .parse import dict_from_file, list_from_file 6 | 7 | __all__ = [ 8 | 'BaseStorageBackend', 'FileClient', 'load', 'dump', 'register_handler', 9 | 'BaseFileHandler', 'JsonHandler', 'PickleHandler', 'YamlHandler', 10 | 'list_from_file', 'dict_from_file' 11 | ] 12 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/fileio/handlers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .base import BaseFileHandler 3 | from .json_handler import JsonHandler 4 | from .pickle_handler import PickleHandler 5 | from .yaml_handler import YamlHandler 6 | 7 | __all__ = ['BaseFileHandler', 'JsonHandler', 'PickleHandler', 'YamlHandler'] 8 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/fileio/handlers/yaml_handler.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import yaml 3 | 4 | try: 5 | from yaml import CLoader as Loader, CDumper as Dumper 6 | except ImportError: 7 | from yaml import Loader, Dumper 8 | 9 | from .base import BaseFileHandler # isort:skip 10 | 11 | 12 | class YamlHandler(BaseFileHandler): 13 | 14 | def load_from_fileobj(self, file, **kwargs): 15 | kwargs.setdefault('Loader', Loader) 16 | return yaml.load(file, **kwargs) 17 | 18 | def dump_to_fileobj(self, obj, file, **kwargs): 19 | kwargs.setdefault('Dumper', Dumper) 20 | yaml.dump(obj, file, **kwargs) 21 | 22 | def dump_to_str(self, obj, **kwargs): 23 | kwargs.setdefault('Dumper', Dumper) 24 | return yaml.dump(obj, **kwargs) 25 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/model_zoo/deprecated.json: -------------------------------------------------------------------------------- 1 | { 2 | "resnet50_caffe": "detectron/resnet50_caffe", 3 | "resnet50_caffe_bgr": "detectron2/resnet50_caffe_bgr", 4 | "resnet101_caffe": "detectron/resnet101_caffe", 5 | "resnet101_caffe_bgr": "detectron2/resnet101_caffe_bgr" 6 | } 7 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/parallel/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .collate import collate 3 | from .data_container import DataContainer 4 | from .data_parallel import MMDataParallel 5 | from .distributed import MMDistributedDataParallel 6 | from .registry import MODULE_WRAPPERS 7 | from .scatter_gather import scatter, scatter_kwargs 8 | from .utils import is_module_wrapper 9 | 10 | __all__ = [ 11 | 'collate', 'DataContainer', 'MMDataParallel', 'MMDistributedDataParallel', 12 | 'scatter', 'scatter_kwargs', 'is_module_wrapper', 'MODULE_WRAPPERS' 13 | ] 14 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/parallel/registry.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from torch.nn.parallel import DataParallel, DistributedDataParallel 3 | 4 | from annotator.uniformer.mmcv.utils import Registry 5 | 6 | MODULE_WRAPPERS = Registry('module wrapper') 7 | MODULE_WRAPPERS.register_module(module=DataParallel) 8 | MODULE_WRAPPERS.register_module(module=DistributedDataParallel) 9 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/parallel/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .registry import MODULE_WRAPPERS 3 | 4 | 5 | def is_module_wrapper(module): 6 | """Check if a module is a module wrapper. 7 | 8 | The following 3 modules in MMCV (and their subclasses) are regarded as 9 | module wrappers: DataParallel, DistributedDataParallel, 10 | MMDistributedDataParallel (the deprecated version). You may add you own 11 | module wrapper by registering it to mmcv.parallel.MODULE_WRAPPERS. 12 | 13 | Args: 14 | module (nn.Module): The module to be checked. 15 | 16 | Returns: 17 | bool: True if the input module is a module wrapper. 18 | """ 19 | module_wrappers = tuple(MODULE_WRAPPERS.module_dict.values()) 20 | return isinstance(module, module_wrappers) 21 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/runner/builder.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import copy 3 | 4 | from ..utils import Registry 5 | 6 | RUNNERS = Registry('runner') 7 | RUNNER_BUILDERS = Registry('runner builder') 8 | 9 | 10 | def build_runner_constructor(cfg): 11 | return RUNNER_BUILDERS.build(cfg) 12 | 13 | 14 | def build_runner(cfg, default_args=None): 15 | runner_cfg = copy.deepcopy(cfg) 16 | constructor_type = runner_cfg.pop('constructor', 17 | 'DefaultRunnerConstructor') 18 | runner_constructor = build_runner_constructor( 19 | dict(type=constructor_type, 20 | runner_cfg=runner_cfg, 21 | default_args=default_args)) 22 | runner = runner_constructor() 23 | return runner 24 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/runner/hooks/closure.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .hook import HOOKS, Hook 3 | 4 | 5 | @HOOKS.register_module() 6 | class ClosureHook(Hook): 7 | 8 | def __init__(self, fn_name, fn): 9 | assert hasattr(self, fn_name) 10 | assert callable(fn) 11 | setattr(self, fn_name, fn) 12 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/runner/hooks/iter_timer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import time 3 | 4 | from .hook import HOOKS, Hook 5 | 6 | 7 | @HOOKS.register_module() 8 | class IterTimerHook(Hook): 9 | 10 | def before_epoch(self, runner): 11 | self.t = time.time() 12 | 13 | def before_iter(self, runner): 14 | runner.log_buffer.update({'data_time': time.time() - self.t}) 15 | 16 | def after_iter(self, runner): 17 | runner.log_buffer.update({'time': time.time() - self.t}) 18 | self.t = time.time() 19 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/runner/hooks/logger/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .base import LoggerHook 3 | from .dvclive import DvcliveLoggerHook 4 | from .mlflow import MlflowLoggerHook 5 | from .neptune import NeptuneLoggerHook 6 | from .pavi import PaviLoggerHook 7 | from .tensorboard import TensorboardLoggerHook 8 | from .text import TextLoggerHook 9 | from .wandb import WandbLoggerHook 10 | 11 | __all__ = [ 12 | 'LoggerHook', 'MlflowLoggerHook', 'PaviLoggerHook', 13 | 'TensorboardLoggerHook', 'TextLoggerHook', 'WandbLoggerHook', 14 | 'NeptuneLoggerHook', 'DvcliveLoggerHook' 15 | ] 16 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/runner/hooks/memory.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import torch 3 | 4 | from .hook import HOOKS, Hook 5 | 6 | 7 | @HOOKS.register_module() 8 | class EmptyCacheHook(Hook): 9 | 10 | def __init__(self, before_epoch=False, after_epoch=True, after_iter=False): 11 | self._before_epoch = before_epoch 12 | self._after_epoch = after_epoch 13 | self._after_iter = after_iter 14 | 15 | def after_iter(self, runner): 16 | if self._after_iter: 17 | torch.cuda.empty_cache() 18 | 19 | def before_epoch(self, runner): 20 | if self._before_epoch: 21 | torch.cuda.empty_cache() 22 | 23 | def after_epoch(self, runner): 24 | if self._after_epoch: 25 | torch.cuda.empty_cache() 26 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/runner/hooks/sampler_seed.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .hook import HOOKS, Hook 3 | 4 | 5 | @HOOKS.register_module() 6 | class DistSamplerSeedHook(Hook): 7 | """Data-loading sampler for distributed training. 8 | 9 | When distributed training, it is only useful in conjunction with 10 | :obj:`EpochBasedRunner`, while :obj:`IterBasedRunner` achieves the same 11 | purpose with :obj:`IterLoader`. 12 | """ 13 | 14 | def before_epoch(self, runner): 15 | if hasattr(runner.data_loader.sampler, 'set_epoch'): 16 | # in case the data loader uses `SequentialSampler` in Pytorch 17 | runner.data_loader.sampler.set_epoch(runner.epoch) 18 | elif hasattr(runner.data_loader.batch_sampler.sampler, 'set_epoch'): 19 | # batch sampler in pytorch warps the sampler as its attributes. 20 | runner.data_loader.batch_sampler.sampler.set_epoch(runner.epoch) 21 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/runner/hooks/sync_buffer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from ..dist_utils import allreduce_params 3 | from .hook import HOOKS, Hook 4 | 5 | 6 | @HOOKS.register_module() 7 | class SyncBuffersHook(Hook): 8 | """Synchronize model buffers such as running_mean and running_var in BN at 9 | the end of each epoch. 10 | 11 | Args: 12 | distributed (bool): Whether distributed training is used. It is 13 | effective only for distributed training. Defaults to True. 14 | """ 15 | 16 | def __init__(self, distributed=True): 17 | self.distributed = distributed 18 | 19 | def after_epoch(self, runner): 20 | """All-reduce model buffers at the end of each epoch.""" 21 | if self.distributed: 22 | allreduce_params(runner.model.buffers()) 23 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/runner/optimizer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .builder import (OPTIMIZER_BUILDERS, OPTIMIZERS, build_optimizer, 3 | build_optimizer_constructor) 4 | from .default_constructor import DefaultOptimizerConstructor 5 | 6 | __all__ = [ 7 | 'OPTIMIZER_BUILDERS', 'OPTIMIZERS', 'DefaultOptimizerConstructor', 8 | 'build_optimizer', 'build_optimizer_constructor' 9 | ] 10 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/utils/trace.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | import torch 4 | 5 | from annotator.uniformer.mmcv.utils import digit_version 6 | 7 | 8 | def is_jit_tracing() -> bool: 9 | if (torch.__version__ != 'parrots' 10 | and digit_version(torch.__version__) >= digit_version('1.6.0')): 11 | on_trace = torch.jit.is_tracing() 12 | # In PyTorch 1.6, torch.jit.is_tracing has a bug. 13 | # Refers to https://github.com/pytorch/pytorch/issues/42448 14 | if isinstance(on_trace, bool): 15 | return on_trace 16 | else: 17 | return torch._C._is_tracing() 18 | else: 19 | warnings.warn( 20 | 'torch.jit.is_tracing is only supported after v1.6.0. ' 21 | 'Therefore is_tracing returns False automatically. Please ' 22 | 'set on_trace manually if you are using trace.', UserWarning) 23 | return False 24 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/video/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .io import Cache, VideoReader, frames2video 3 | from .optflow import (dequantize_flow, flow_from_bytes, flow_warp, flowread, 4 | flowwrite, quantize_flow, sparse_flow_from_bytes) 5 | from .processing import concat_video, convert_video, cut_video, resize_video 6 | 7 | __all__ = [ 8 | 'Cache', 'VideoReader', 'frames2video', 'convert_video', 'resize_video', 9 | 'cut_video', 'concat_video', 'flowread', 'flowwrite', 'quantize_flow', 10 | 'dequantize_flow', 'flow_warp', 'flow_from_bytes', 'sparse_flow_from_bytes' 11 | ] 12 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv/visualization/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .color import Color, color_val 3 | from .image import imshow, imshow_bboxes, imshow_det_bboxes 4 | from .optflow import flow2rgb, flowshow, make_color_wheel 5 | 6 | __all__ = [ 7 | 'Color', 'color_val', 'imshow', 'imshow_bboxes', 'imshow_det_bboxes', 8 | 'flowshow', 'flow2rgb', 'make_color_wheel' 9 | ] 10 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmcv_custom/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .checkpoint import load_checkpoint 4 | 5 | __all__ = ['load_checkpoint'] 6 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/apis/__init__.py: -------------------------------------------------------------------------------- 1 | from .inference import inference_segmentor, init_segmentor, show_result_pyplot 2 | from .test import multi_gpu_test, single_gpu_test 3 | from .train import get_root_logger, set_random_seed, train_segmentor 4 | 5 | __all__ = [ 6 | 'get_root_logger', 'set_random_seed', 'train_segmentor', 'init_segmentor', 7 | 'inference_segmentor', 'multi_gpu_test', 'single_gpu_test', 8 | 'show_result_pyplot' 9 | ] 10 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/core/__init__.py: -------------------------------------------------------------------------------- 1 | from .evaluation import * # noqa: F401, F403 2 | from .seg import * # noqa: F401, F403 3 | from .utils import * # noqa: F401, F403 4 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/core/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | from .class_names import get_classes, get_palette 2 | from .eval_hooks import DistEvalHook, EvalHook 3 | from .metrics import eval_metrics, mean_dice, mean_fscore, mean_iou 4 | 5 | __all__ = [ 6 | 'EvalHook', 'DistEvalHook', 'mean_dice', 'mean_iou', 'mean_fscore', 7 | 'eval_metrics', 'get_classes', 'get_palette' 8 | ] 9 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/core/seg/__init__.py: -------------------------------------------------------------------------------- 1 | from .builder import build_pixel_sampler 2 | from .sampler import BasePixelSampler, OHEMPixelSampler 3 | 4 | __all__ = ['build_pixel_sampler', 'BasePixelSampler', 'OHEMPixelSampler'] 5 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/core/seg/builder.py: -------------------------------------------------------------------------------- 1 | from annotator.uniformer.mmcv.utils import Registry, build_from_cfg 2 | 3 | PIXEL_SAMPLERS = Registry('pixel sampler') 4 | 5 | 6 | def build_pixel_sampler(cfg, **default_args): 7 | """Build pixel sampler for segmentation map.""" 8 | return build_from_cfg(cfg, PIXEL_SAMPLERS, default_args) 9 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/core/seg/sampler/__init__.py: -------------------------------------------------------------------------------- 1 | from .base_pixel_sampler import BasePixelSampler 2 | from .ohem_pixel_sampler import OHEMPixelSampler 3 | 4 | __all__ = ['BasePixelSampler', 'OHEMPixelSampler'] 5 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/core/seg/sampler/base_pixel_sampler.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | 3 | 4 | class BasePixelSampler(metaclass=ABCMeta): 5 | """Base class of pixel sampler.""" 6 | 7 | def __init__(self, **kwargs): 8 | pass 9 | 10 | @abstractmethod 11 | def sample(self, seg_logit, seg_label): 12 | """Placeholder for sample function.""" 13 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/core/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .misc import add_prefix 2 | 3 | __all__ = ['add_prefix'] 4 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/core/utils/misc.py: -------------------------------------------------------------------------------- 1 | def add_prefix(inputs, prefix): 2 | """Add prefix for dict. 3 | 4 | Args: 5 | inputs (dict): The input dict with str keys. 6 | prefix (str): The prefix to add. 7 | 8 | Returns: 9 | 10 | dict: The dict with keys updated with ``prefix``. 11 | """ 12 | 13 | outputs = dict() 14 | for name, value in inputs.items(): 15 | outputs[f'{prefix}.{name}'] = value 16 | 17 | return outputs 18 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | from .ade import ADE20KDataset 2 | from .builder import DATASETS, PIPELINES, build_dataloader, build_dataset 3 | from .chase_db1 import ChaseDB1Dataset 4 | from .cityscapes import CityscapesDataset 5 | from .custom import CustomDataset 6 | from .dataset_wrappers import ConcatDataset, RepeatDataset 7 | from .drive import DRIVEDataset 8 | from .hrf import HRFDataset 9 | from .pascal_context import PascalContextDataset, PascalContextDataset59 10 | from .stare import STAREDataset 11 | from .voc import PascalVOCDataset 12 | 13 | __all__ = [ 14 | 'CustomDataset', 'build_dataloader', 'ConcatDataset', 'RepeatDataset', 15 | 'DATASETS', 'build_dataset', 'PIPELINES', 'CityscapesDataset', 16 | 'PascalVOCDataset', 'ADE20KDataset', 'PascalContextDataset', 17 | 'PascalContextDataset59', 'ChaseDB1Dataset', 'DRIVEDataset', 'HRFDataset', 18 | 'STAREDataset' 19 | ] 20 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/datasets/hrf.py: -------------------------------------------------------------------------------- 1 | import os.path as osp 2 | 3 | from .builder import DATASETS 4 | from .custom import CustomDataset 5 | 6 | 7 | @DATASETS.register_module() 8 | class HRFDataset(CustomDataset): 9 | """HRF dataset. 10 | 11 | In segmentation map annotation for HRF, 0 stands for background, which is 12 | included in 2 categories. ``reduce_zero_label`` is fixed to False. The 13 | ``img_suffix`` is fixed to '.png' and ``seg_map_suffix`` is fixed to 14 | '.png'. 15 | """ 16 | 17 | CLASSES = ('background', 'vessel') 18 | 19 | PALETTE = [[120, 120, 120], [6, 230, 230]] 20 | 21 | def __init__(self, **kwargs): 22 | super(HRFDataset, self).__init__(img_suffix='.png', 23 | seg_map_suffix='.png', 24 | reduce_zero_label=False, 25 | **kwargs) 26 | assert osp.exists(self.img_dir) 27 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/datasets/pipelines/__init__.py: -------------------------------------------------------------------------------- 1 | from .compose import Compose 2 | from .formating import (Collect, ImageToTensor, ToDataContainer, ToTensor, 3 | Transpose, to_tensor) 4 | from .loading import LoadAnnotations, LoadImageFromFile 5 | from .test_time_aug import MultiScaleFlipAug 6 | from .transforms import (CLAHE, AdjustGamma, Normalize, Pad, 7 | PhotoMetricDistortion, RandomCrop, RandomFlip, 8 | RandomRotate, Rerange, Resize, RGB2Gray, SegRescale) 9 | 10 | __all__ = [ 11 | 'Compose', 'to_tensor', 'ToTensor', 'ImageToTensor', 'ToDataContainer', 12 | 'Transpose', 'Collect', 'LoadAnnotations', 'LoadImageFromFile', 13 | 'MultiScaleFlipAug', 'Resize', 'RandomFlip', 'Pad', 'RandomCrop', 14 | 'Normalize', 'SegRescale', 'PhotoMetricDistortion', 'RandomRotate', 15 | 'AdjustGamma', 'CLAHE', 'Rerange', 'RGB2Gray' 16 | ] 17 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .backbones import * # noqa: F401,F403 2 | from .builder import (BACKBONES, HEADS, LOSSES, SEGMENTORS, build_backbone, 3 | build_head, build_loss, build_segmentor) 4 | from .decode_heads import * # noqa: F401,F403 5 | from .losses import * # noqa: F401,F403 6 | from .necks import * # noqa: F401,F403 7 | from .segmentors import * # noqa: F401,F403 8 | 9 | __all__ = [ 10 | 'BACKBONES', 'HEADS', 'LOSSES', 'SEGMENTORS', 'build_backbone', 11 | 'build_head', 'build_loss', 'build_segmentor' 12 | ] 13 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/models/backbones/__init__.py: -------------------------------------------------------------------------------- 1 | from .cgnet import CGNet 2 | # from .fast_scnn import FastSCNN 3 | from .hrnet import HRNet 4 | from .mobilenet_v2 import MobileNetV2 5 | from .mobilenet_v3 import MobileNetV3 6 | from .resnest import ResNeSt 7 | from .resnet import ResNet, ResNetV1c, ResNetV1d 8 | from .resnext import ResNeXt 9 | from .unet import UNet 10 | from .vit import VisionTransformer 11 | from .uniformer import UniFormer 12 | 13 | __all__ = [ 14 | 'ResNet', 'ResNetV1c', 'ResNetV1d', 'ResNeXt', 'HRNet', 'ResNeSt', 15 | 'MobileNetV2', 'UNet', 'CGNet', 'MobileNetV3', 'VisionTransformer', 16 | 'UniFormer' 17 | ] 18 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/models/losses/__init__.py: -------------------------------------------------------------------------------- 1 | from .accuracy import Accuracy, accuracy 2 | from .cross_entropy_loss import (CrossEntropyLoss, binary_cross_entropy, 3 | cross_entropy, mask_cross_entropy) 4 | from .dice_loss import DiceLoss 5 | from .lovasz_loss import LovaszLoss 6 | from .utils import reduce_loss, weight_reduce_loss, weighted_loss 7 | 8 | __all__ = [ 9 | 'accuracy', 'Accuracy', 'cross_entropy', 'binary_cross_entropy', 10 | 'mask_cross_entropy', 'CrossEntropyLoss', 'reduce_loss', 11 | 'weight_reduce_loss', 'weighted_loss', 'LovaszLoss', 'DiceLoss' 12 | ] 13 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/models/necks/__init__.py: -------------------------------------------------------------------------------- 1 | from .fpn import FPN 2 | from .multilevel_neck import MultiLevelNeck 3 | 4 | __all__ = ['FPN', 'MultiLevelNeck'] 5 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/models/segmentors/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import BaseSegmentor 2 | from .cascade_encoder_decoder import CascadeEncoderDecoder 3 | from .encoder_decoder import EncoderDecoder 4 | 5 | __all__ = ['BaseSegmentor', 'EncoderDecoder', 'CascadeEncoderDecoder'] 6 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/models/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .drop import DropPath 2 | from .inverted_residual import InvertedResidual, InvertedResidualV3 3 | from .make_divisible import make_divisible 4 | from .res_layer import ResLayer 5 | from .se_layer import SELayer 6 | from .self_attention_block import SelfAttentionBlock 7 | from .up_conv_block import UpConvBlock 8 | from .weight_init import trunc_normal_ 9 | 10 | __all__ = [ 11 | 'ResLayer', 'SelfAttentionBlock', 'make_divisible', 'InvertedResidual', 12 | 'UpConvBlock', 'InvertedResidualV3', 'SELayer', 'DropPath', 'trunc_normal_' 13 | ] 14 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/ops/__init__.py: -------------------------------------------------------------------------------- 1 | from .encoding import Encoding 2 | from .wrappers import Upsample, resize 3 | 4 | __all__ = ['Upsample', 'resize', 'Encoding'] 5 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .collect_env import collect_env 2 | from .logger import get_root_logger 3 | 4 | __all__ = ['get_root_logger', 'collect_env'] 5 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/annotator/uniformer/mmseg/utils/collect_env.py: -------------------------------------------------------------------------------- 1 | from annotator.uniformer.mmcv.utils import collect_env as collect_base_env 2 | from annotator.uniformer.mmcv.utils import get_git_hash 3 | 4 | import annotator.uniformer.mmseg as mmseg 5 | 6 | 7 | def collect_env(): 8 | """Collect the information of the running environments.""" 9 | env_info = collect_base_env() 10 | env_info['MMSegmentation'] = f'{mmseg.__version__}+{get_git_hash()[:7]}' 11 | 12 | return env_info 13 | 14 | 15 | if __name__ == '__main__': 16 | for name, val in collect_env().items(): 17 | print('{}: {}'.format(name, val)) 18 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/config.py: -------------------------------------------------------------------------------- 1 | save_memory = False 2 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/docs/faq.md: -------------------------------------------------------------------------------- 1 | # FAQs 2 | 3 | **Q:** If the weight of a conv layer is zero, the gradient will also be zero, and the network will not learn anything. Why "zero convolution" works? 4 | 5 | **A:** This is wrong. Let us consider a very simple 6 | 7 | $$y=wx+b$$ 8 | 9 | and we have 10 | 11 | $$\partial y/\partial w=x, \partial y/\partial x=w, \partial y/\partial b=1$$ 12 | 13 | and if $w=0$ and $x \neq 0$, then 14 | 15 | $$\partial y/\partial w \neq 0, \partial y/\partial x=0, \partial y/\partial b\neq 0$$ 16 | 17 | which means as long as $x \neq 0$, one gradient descent iteration will make $w$ non-zero. Then 18 | 19 | $$\partial y/\partial x\neq 0$$ 20 | 21 | so that the zero convolutions will progressively become a common conv layer with non-zero weights. 22 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/docs/low_vram.md: -------------------------------------------------------------------------------- 1 | # Enable Low VRAM Mode 2 | 3 | If you are using 8GB GPU card (or if you want larger batch size), please open "config.py", and then set 4 | 5 | ```python 6 | save_memory = True 7 | ``` 8 | 9 | This feature is still being tested - not all graphics cards are guaranteed to succeed. 10 | 11 | But it should be neat as I can diffuse at a batch size of 12 now. 12 | 13 | (prompt "man") 14 | 15 | ![p](../github_page/ram12.jpg) 16 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/environment.yaml: -------------------------------------------------------------------------------- 1 | name: control 2 | channels: 3 | - pytorch 4 | - defaults 5 | dependencies: 6 | - python=3.8.5 7 | - pip=20.3 8 | - cudatoolkit=11.3 9 | - pytorch=1.12.1 10 | - torchvision=0.13.1 11 | - numpy=1.23.1 12 | - pip: 13 | - gradio==3.16.2 14 | - albumentations==1.3.0 15 | - opencv-contrib-python==4.3.0.36 16 | - imageio==2.9.0 17 | - imageio-ffmpeg==0.4.2 18 | - pytorch-lightning==1.5.0 19 | - omegaconf==2.1.1 20 | - test-tube>=0.7.5 21 | - streamlit==1.12.1 22 | - einops==0.3.0 23 | - transformers==4.19.2 24 | - webdataset==0.2.5 25 | - kornia==0.6 26 | - open_clip_torch==2.0.2 27 | - invisible-watermark>=0.1.5 28 | - streamlit-drawable-canvas==0.8.0 29 | - torchmetrics==0.6.0 30 | - timm==0.6.12 31 | - addict==2.4.0 32 | - yapf==0.32.0 33 | - prettytable==3.6.0 34 | - safetensors==0.2.7 35 | - basicsr==1.4.2 36 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/font/DejaVuSans.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/font/DejaVuSans.ttf -------------------------------------------------------------------------------- /Hackathon2023/controlnet/ldm/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/ldm/data/__init__.py -------------------------------------------------------------------------------- /Hackathon2023/controlnet/ldm/data/util.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from ldm.modules.midas.api import load_midas_transform 4 | 5 | 6 | class AddMiDaS(object): 7 | 8 | def __init__(self, model_type): 9 | super().__init__() 10 | self.transform = load_midas_transform(model_type) 11 | 12 | def pt2np(self, x): 13 | x = ((x + 1.0) * .5).detach().cpu().numpy() 14 | return x 15 | 16 | def np2pt(self, x): 17 | x = torch.from_numpy(x) * 2 - 1. 18 | return x 19 | 20 | def __call__(self, sample): 21 | # sample['jpg'] is tensor hwc in [-1, 1] at this point 22 | x = self.pt2np(sample['jpg']) 23 | x = self.transform({"image": x})["image"] 24 | sample['midas_in'] = x 25 | return sample 26 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/ldm/models/diffusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/ldm/models/diffusion/__init__.py -------------------------------------------------------------------------------- /Hackathon2023/controlnet/ldm/models/diffusion/dpm_solver/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/ldm/models/diffusion/dpm_solver/__init__.py -------------------------------------------------------------------------------- /Hackathon2023/controlnet/ldm/models/diffusion/sampling_util.py: -------------------------------------------------------------------------------- 1 | def append_dims(x, target_dims): 2 | """Appends dimensions to the end of a tensor until it has target_dims dimensions. 3 | From https://github.com/crowsonkb/k-diffusion/blob/master/k_diffusion/utils.py""" 4 | dims_to_append = target_dims - x.ndim 5 | if dims_to_append < 0: 6 | raise ValueError( 7 | f'input has {x.ndim} dims but target_dims is {target_dims}, which is less' 8 | ) 9 | return x[(..., ) + (None, ) * dims_to_append] 10 | 11 | 12 | def norm_thresholding(x0, value): 13 | s = append_dims( 14 | x0.pow(2).flatten(1).mean(1).sqrt().clamp(min=value), x0.ndim) 15 | return x0 * (value / s) 16 | 17 | 18 | def spatial_norm_thresholding(x0, value): 19 | # b c h w 20 | s = x0.pow(2).mean(1, keepdim=True).sqrt().clamp(min=value) 21 | return x0 * (value / s) 22 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/ldm/modules/diffusionmodules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/ldm/modules/diffusionmodules/__init__.py -------------------------------------------------------------------------------- /Hackathon2023/controlnet/ldm/modules/distributions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/ldm/modules/distributions/__init__.py -------------------------------------------------------------------------------- /Hackathon2023/controlnet/ldm/modules/encoders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/ldm/modules/encoders/__init__.py -------------------------------------------------------------------------------- /Hackathon2023/controlnet/ldm/modules/image_degradation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/ldm/modules/image_degradation/__init__.py -------------------------------------------------------------------------------- /Hackathon2023/controlnet/ldm/modules/image_degradation/utils/test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/ldm/modules/image_degradation/utils/test.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/ldm/modules/midas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/ldm/modules/midas/__init__.py -------------------------------------------------------------------------------- /Hackathon2023/controlnet/ldm/modules/midas/midas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/ldm/modules/midas/midas/__init__.py -------------------------------------------------------------------------------- /Hackathon2023/controlnet/ldm/modules/midas/midas/base_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class BaseModel(torch.nn.Module): 5 | 6 | def load(self, path): 7 | """Load model from file. 8 | 9 | Args: 10 | path (str): file path 11 | """ 12 | parameters = torch.load(path, map_location=torch.device('cpu')) 13 | 14 | if "optimizer" in parameters: 15 | parameters = parameters["model"] 16 | 17 | self.load_state_dict(parameters) 18 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/preprocess.sh: -------------------------------------------------------------------------------- 1 | echo "preprocess" 2 | cd plugin && mkdir build && cd build && cmake .. && make -j5 3 | cd ../../ && python3 export_onnx.py 4 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/share.py: -------------------------------------------------------------------------------- 1 | import config 2 | from cldm.hack import disable_verbosity, enable_sliced_attention 3 | 4 | disable_verbosity() 5 | 6 | if config.save_memory: 7 | enable_sliced_attention() 8 | -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/bag.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/bag.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/bag_scribble.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/bag_scribble.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/bird.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/bird.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/boy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/boy.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/building.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/building.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/building2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/building2.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/cute_robot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/cute_robot.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/cyber.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/cyber.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/dog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/dog.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/dog2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/dog2.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/house.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/house.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/house_line.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/house_line.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/human.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/human.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/human_line.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/human_line.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/man.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/man.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/old.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/old.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/pose1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/pose1.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/pose2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/pose2.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/room.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/room.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/sd.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/sd.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/shose.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/shose.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/toy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/toy.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/user_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/user_1.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/test_imgs/user_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/Hackathon2023/controlnet/test_imgs/user_3.png -------------------------------------------------------------------------------- /Hackathon2023/controlnet/tutorial_dataset_test.py: -------------------------------------------------------------------------------- 1 | from tutorial_dataset import MyDataset 2 | 3 | dataset = MyDataset() 4 | print(len(dataset)) 5 | 6 | item = dataset[1234] 7 | jpg = item['jpg'] 8 | txt = item['txt'] 9 | hint = item['hint'] 10 | print(txt) 11 | print(jpg.shape) 12 | print(hint.shape) 13 | -------------------------------------------------------------------------------- /cookbook/.gitattributes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/cookbook/.gitattributes -------------------------------------------------------------------------------- /cookbook/.gitignore: -------------------------------------------------------------------------------- 1 | # common 2 | __pycache__/ 3 | .hypothesis/ 4 | .vscode/ 5 | *.log 6 | *.gz 7 | *.yaml 8 | 9 | # TensorRT 10 | *.d 11 | *.exe 12 | *.Int8Cache 13 | *.json 14 | *.lock 15 | *.o 16 | *.raw 17 | *.so 18 | *.Tacitc 19 | *.TimingCache 20 | *.trt 21 | *.trt* 22 | 23 | # Nsight Systems 24 | *.nsys-rep 25 | *.qdrep 26 | 27 | # numpy 28 | *.npy 29 | *.npz 30 | 31 | # ML framework 32 | *.onnx 33 | *.onnx.weight 34 | *.pth 35 | *.ts 36 | *.pkl 37 | #*.pb 38 | 39 | # Packages 40 | build/ 41 | dist/ 42 | tensorrt_cookbook.egg-info/ 43 | -------------------------------------------------------------------------------- /cookbook/00-Data/data/.gitignore: -------------------------------------------------------------------------------- 1 | test/ 2 | train/ 3 | -------------------------------------------------------------------------------- /cookbook/00-Data/model/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/cookbook/00-Data/model/.gitkeep -------------------------------------------------------------------------------- /cookbook/01-SimpleDemo/README.md: -------------------------------------------------------------------------------- 1 | # 01-SimpleDemo 2 | 3 | + Simple stand-alone examples of using TensorRT to build a network and do inference. 4 | 5 | + We have equivalent implementations in Python and C++. 6 | 7 | + Now only newest TensorRT-10 is recommended. 8 | 9 | ## TensorRT-10 10 | 11 | + Basic example of using TensorRT-10. 12 | 13 | ## TensorRT-6 14 | 15 | + Basic example of using TensorRT-6. 16 | 17 | ## TensorRT-7 18 | 19 | + Basic example of using TensorRT-7. 20 | 21 | ## TensorRT-8.0 22 | 23 | + Basic example of using TensorRT-8.0. 24 | 25 | ## TensorRT-8.6 26 | 27 | + Basic example of using TensorRT-8.6. 28 | -------------------------------------------------------------------------------- /cookbook/01-SimpleDemo/TensorRT-10/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | all: main.exe 4 | 5 | main.exe: main.o cookbookHelper.o 6 | $(COMMAND_EXE) 7 | 8 | main.o: main.cpp 9 | $(COMMAND_O) 10 | 11 | .PHONY: test 12 | test: 13 | make clean; make all -j; rm -rf *.trt 14 | ./main.exe > log-main.exe.log 15 | 16 | .PHONY: clean 17 | clean: 18 | rm -rf *.d *.o *.exe *.trt 19 | -------------------------------------------------------------------------------- /cookbook/01-SimpleDemo/TensorRT-10/README.md: -------------------------------------------------------------------------------- 1 | # TensorRT-10 2 | 3 | + Basic example of using TensorRT-10. 4 | 5 | + We have totally 4 equivalent implementations. 6 | 7 | + For Python workflow, here are two equivalent choices for buffer management, using apckage numpy or torch respectively. 8 | 9 | ```bash 10 | python3 main_numpy.py 11 | 12 | python3 main_torch.py 13 | ``` 14 | 15 | + For C++ workflow, we need to build an executive file and then run it. 16 | 17 | ```bash 18 | make clean && make 19 | 20 | ./main.exe 21 | ``` 22 | 23 | + The one more example uses some code wrappers, we'd better to get used to it since all the other examples in cookbook is using it. 24 | 25 | ```bash 26 | python3 main_cookbook_flavor.py 27 | ``` 28 | -------------------------------------------------------------------------------- /cookbook/01-SimpleDemo/TensorRT-10/main.h: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * SPDX-License-Identifier: Apache-2.0 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | const std::string fuck {"FUCK3"}; 19 | -------------------------------------------------------------------------------- /cookbook/01-SimpleDemo/TensorRT-6/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | all: main.exe 4 | 5 | main.exe: main.o cookbookHelper.o 6 | $(COMMAND_EXE) 7 | 8 | main.o: main.cpp 9 | $(COMMAND_O) 10 | 11 | .PHONY: test 12 | test: 13 | make clean; make all -j; rm -rf *.trt 14 | ./main.exe > log-main.exe.log 15 | 16 | .PHONY: clean 17 | clean: 18 | rm -rf *.d *.o *.exe *.trt 19 | -------------------------------------------------------------------------------- /cookbook/01-SimpleDemo/TensorRT-6/README.md: -------------------------------------------------------------------------------- 1 | # TensorRT-6 2 | 3 | + Basic example of using TensorRT-6. 4 | 5 | + Only for commemoration. 6 | 7 | + Steps to run. 8 | 9 | ```bash 10 | make test 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/01-SimpleDemo/TensorRT-7/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | all: main.exe 4 | 5 | main.exe: main.o cookbookHelper.o 6 | $(COMMAND_EXE) 7 | 8 | main.o: main.cpp 9 | $(COMMAND_O) 10 | 11 | .PHONY: test 12 | test: 13 | make clean; make all -j; rm -rf *.trt 14 | ./main.exe > log-main.exe.log 15 | 16 | .PHONY: clean 17 | clean: 18 | rm -rf *.d *.o *.exe *.trt 19 | -------------------------------------------------------------------------------- /cookbook/01-SimpleDemo/TensorRT-7/README.md: -------------------------------------------------------------------------------- 1 | # TensorRT-7 2 | 3 | + Basic example of using TensorRT-7. 4 | 5 | + Only for commemoration. 6 | 7 | + Steps to run. 8 | 9 | ```bash 10 | make test 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/01-SimpleDemo/TensorRT-8.0/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | all: main.exe 4 | 5 | main.exe: main.o cookbookHelper.o 6 | $(COMMAND_EXE) 7 | 8 | main.o: main.cpp 9 | $(COMMAND_O) 10 | 11 | .PHONY: test 12 | test: 13 | make clean; make all -j; rm -rf *.trt 14 | ./main.exe > log-main.exe.log 15 | 16 | .PHONY: clean 17 | clean: 18 | rm -rf *.d *.o *.exe *.trt 19 | -------------------------------------------------------------------------------- /cookbook/01-SimpleDemo/TensorRT-8.0/README.md: -------------------------------------------------------------------------------- 1 | # TensorRT-8.0 2 | 3 | + Basic example of using TensorRT-8.0. 4 | 5 | + Only for commemoration. 6 | 7 | + Steps to run. 8 | 9 | ```bash 10 | make test 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/01-SimpleDemo/TensorRT-8.6/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | all: main.exe 4 | 5 | main.exe: main.o cookbookHelper.o 6 | $(COMMAND_EXE) 7 | 8 | main.o: main.cpp 9 | $(COMMAND_O) 10 | 11 | .PHONY: test 12 | test: 13 | make clean; make all -j; rm -rf *.trt 14 | ./main.exe > log-main.exe.log 15 | 16 | .PHONY: clean 17 | clean: 18 | rm -rf *.d *.o *.exe *.trt 19 | -------------------------------------------------------------------------------- /cookbook/01-SimpleDemo/TensorRT-8.6/README.md: -------------------------------------------------------------------------------- 1 | # TensorRT-8.6 2 | 3 | + Basic example of using TensorRT-8.6. 4 | 5 | + Only for commemoration. 6 | 7 | + Steps to run. 8 | 9 | ```bash 10 | make test 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/02-API/Builder/README.md: -------------------------------------------------------------------------------- 1 | # Builder 2 | 3 | + Simple usage of Builder's API. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/02-API/Builder/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/BuilderConfig/README.md: -------------------------------------------------------------------------------- 1 | # BuilderConfig 2 | 3 | + Simple usage of BuilderConfig's APIs. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/02-API/CudaEngine/README.md: -------------------------------------------------------------------------------- 1 | # CudaEngine 2 | 3 | + Simple usage of CudaEngine's API. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/02-API/ExecutionContext/README.md: -------------------------------------------------------------------------------- 1 | # ExecutionContext 2 | 3 | + Simple usage of ExecutionContext's API. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/02-API/HostMemory/README.md: -------------------------------------------------------------------------------- 1 | # HostMemory 2 | 3 | + Simple usage of HostMemory's API. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/02-API/HostMemory/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Assertion/README.md: -------------------------------------------------------------------------------- 1 | # Assertion Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Condition of input tensor 10 | 11 | | Name | Range | 12 | | :----------------------: | :----: | 13 | | Rank of condition tensor | 0 or 1 | 14 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Cast/README.md: -------------------------------------------------------------------------------- 1 | # Cast Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + `layer.get_output(0).dtype = XXX` should be used in some conversions or for network input / output tensors. 10 | 11 | + Refer to `../Identity` for more examples of casting data types 12 | 13 | + `kFP8`, `kINT4`, `kFP4` are not supported as output data type in cast Layer. 14 | 15 | + Alternative values of `trt.DataType` 16 | 17 | | Name | Alias | 18 | | :---: | :----------: | 19 | | FLOAT | trt.float32 | 20 | | HALF | trt.float16 | 21 | | INT8 | trt.int8 | 22 | | INT32 | trt.in32 | 23 | | BOOL | trt.bool | 24 | | UINT8 | trt.uint8 | 25 | | FP8 | trt.fp8 | 26 | | BF16 | trt.bfloat16 | 27 | | INT64 | trt.int64 | 28 | | INT4 | trt.int4 | 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Cast/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Concatenation/README.md: -------------------------------------------------------------------------------- 1 | # Concatenation Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Ranges of parameters 10 | 11 | | Name | Range | 12 | | :------------------------: | :---: | 13 | | Max number of input tensor | 10000 | 14 | 15 | + Default values of parameters 16 | 17 | | Name | Comment | 18 | | :--: | :------------------------: | 19 | | axis | $max\left(0, n - 3\right)$ | 20 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Condition/README.md: -------------------------------------------------------------------------------- 1 | # Condition Layer 2 | 3 | + Refer to `../IfConditionStructure`. 4 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Condition/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Constant/README.md: -------------------------------------------------------------------------------- 1 | # Constant Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + **np.ascontiguousarray()** must be used while converting `np.array` to `trt.Weights` 10 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Cumulative/README.md: -------------------------------------------------------------------------------- 1 | # Cumulative Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Dequantize/README.md: -------------------------------------------------------------------------------- 1 | # Dequantize Layer 2 | 3 | + Refer to `../QDQStructure`. 4 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Dequantize/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/DynamicQuantize/README.md: -------------------------------------------------------------------------------- 1 | # Dynamic quantizatize Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Einsum/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Fill/README.md: -------------------------------------------------------------------------------- 1 | # Fill Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Random seed is not supported in TensorRT yet. 10 | 11 | + Alternative values of `trt.FillOperation`. 12 | 13 | | Name | Comment | 14 | | :------------: | :-----: | 15 | | LINSPACE | | 16 | | RANDOM_NORMAL | | 17 | | RANDOM_UNIFORM | | 18 | 19 | + Default values of parameters 20 | 21 | | Name | Comment | 22 | | :---: | :----------------: | 23 | | alpha | 0 | 24 | | beta | 1 | 25 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Fill/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Gather/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Identity/README.md: -------------------------------------------------------------------------------- 1 | # Identity Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/IfConditionStructure/README.md: -------------------------------------------------------------------------------- 1 | # IfCondition structure 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + The IfCondition structure contains usage of `ConditionLayer`, `IfConditionalInputLayer` and `IfConditionalOutputLayer`. 10 | 11 | + The output tensor of the IfCondition structure is from `IfConditionOutputLayer`, while member function `get_output()` is also provided in `ConditionLayer`, `IfConditionInputLayer` and `IfConditionConditionLayer`, but their output are None. 12 | 13 | + Ranges of parameters 14 | 15 | | Name | Range | 16 | | :---------------------------: | :---------------: | 17 | | Rank of condition tensor | 0 | 18 | | Data type of condition tensor | trt.DataType.BOOL | 19 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/IfConditionalInput/README.md: -------------------------------------------------------------------------------- 1 | # Condition Input Layer 2 | 3 | + Refer to `../IfConditionStructure`. 4 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/IfConditionalInput/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/IfConditionalOutput/README.md: -------------------------------------------------------------------------------- 1 | # Condition Output Layer 2 | 3 | + Refer to `../IfConditionStructure`. 4 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/IfConditionalOutput/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Iterator/README.md: -------------------------------------------------------------------------------- 1 | # Iterator Layer 2 | 3 | + Refer to `../ConditionStructure`. 4 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Iterator/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/LRN/README.md: -------------------------------------------------------------------------------- 1 | # LRN Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Range of the parameter 10 | 11 | | Name | Range | 12 | | :---------: | :-------------------------: | 13 | | window_size | [3,15], must be even number | 14 | | alpha | [-1e20, 1e20] | 15 | | beta | [0.01, 1e5f] | 16 | | k | [1e-5, 1e10] | 17 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/LRN/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/LoopOutput/README.md: -------------------------------------------------------------------------------- 1 | # Loop Output Layer 2 | 3 | + Refer to `../LoopStructure`. 4 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/LoopOutput/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/MatrixMultiply/README.md: -------------------------------------------------------------------------------- 1 | # Matrix Multiply Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Alternative values of `trt.MatrixOperation` 10 | | Name | Comment | 11 | | :-----------------: | :---------------------------------------------------: | 12 | | NONE | no transpose operation | 13 | | VECTOR | the input tensor is vector (do not broadcast) | 14 | | TRANSPOSE | transpose the matrix before multiplication | 15 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/NMS/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/NonZero/README.md: -------------------------------------------------------------------------------- 1 | # NonZero Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Output Allocator MUST be used for Data-Dependent-Shape mode like this example. 10 | + In TensorRT-8.6 ~ TensorRT-9.X, `context.get_tensor_shape()` returns real output shape after the enqueue in DDS mode, so we can use it to get real output shape. 11 | + But this API always return shape with -1 in DDS mode since TensorRT-10, so we must use Output Allocator to get the real output shape. 12 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Normalization/README.md: -------------------------------------------------------------------------------- 1 | # Normalization Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Computation process: 10 | 11 | $$ 12 | T3 = \frac{T0 - \bold{mean}\left(T0,axes\right)}{\sqrt{\bold{var}\left(T0,axes\right)+\epsilon}} \times T1 + T2 13 | $$ 14 | 15 | + Layer Normalization 16 | 17 | + Shape of T0, T3: [$d_0, d_1, d_2, d_3, \dots$] 18 | + Shape of T1, T2: [$a_0, a_1, a_2, a_3, \dots$], $a_i == d_i$ if 1 << i is set as normalization axes, or ai == 1 if not set. 19 | 20 | + Group Normalization 21 | 22 | + Shape of T0, T3: [N, C, H, W, ...] 23 | + Shape of T1, T2: [1, G, 1, 1, ...], G is the number of groups, s.t. C % G == 0 24 | 25 | + Instance Normalization 26 | 27 | + Shape of T0, T3: [N, C, H, W, ...] 28 | + Shape of T1, T2: [1, C, 1, 1, ...] 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/OneHot/README.md: -------------------------------------------------------------------------------- 1 | # One Hot Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/OneHot/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Padding/README.md: -------------------------------------------------------------------------------- 1 | # paddingNd Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + IPaddingLayer is deprecated since TensorRT 8.2 and use slice layer instead. 10 | 11 | + Only padding or cropping on the last two dimensions are supported. 12 | 13 | + Ranges of parameters 14 | 15 | | Name | Range | 16 | | :------------------: | :-----------: | 17 | | Rank of input tensor | $\ge 4$ | 18 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/ParametricReLU/README.md: -------------------------------------------------------------------------------- 1 | # Parametric ReLU Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Shape of slope layer should be able to broadcast to the input tensor. 10 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Plugin/README.md: -------------------------------------------------------------------------------- 1 | # Plugin Layer 2 | 3 | + Refer to `05-Plugin/BasicExample-V2-deprecated`. 4 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Plugin/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/PluginV2/README.md: -------------------------------------------------------------------------------- 1 | # PluginV2 Layer 2 | 3 | + Refer to `05-Plugin/BasicExample-V2-deprecated`. 4 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/PluginV2/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/PluginV3/README.md: -------------------------------------------------------------------------------- 1 | # PluginV3 Layer 2 | 3 | + Refer to `05-Plugin`. 4 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/PluginV3/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/QDQStructure/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | # python3 main.py # TODO: BUG fix 23 | 24 | echo "Finish `basename $(pwd)`" 25 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Quantize/README.md: -------------------------------------------------------------------------------- 1 | # Quantize Layer 2 | 3 | + Refer to `../QDQStructure`. 4 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Quantize/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/README.md: -------------------------------------------------------------------------------- 1 | # Layer 2 | 3 | + Information of the layers are listed in [**Operators Document**](https://docs.nvidia.com/deeplearning/tensorrt/operators/docs/), here we just provide some more examples. 4 | 5 | + Steps to run examples 6 | 7 | + `main.py` contains the common APIs for all layers. 8 | 9 | ```bash 10 | python3 testAllLayer.py 11 | ``` 12 | 13 | + All scripts `*/main.py` are independently executable, we can go into the directory and follow the README.md inside. 14 | 15 | ## TODO 16 | 17 | + Add information of argument constrain, argument default value, deprecated APIs. 18 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Recurrence/README.md: -------------------------------------------------------------------------------- 1 | # Recurrence Layer 2 | 3 | + Refer to `../LoopStructure`. 4 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Recurrence/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Reduce/README.md: -------------------------------------------------------------------------------- 1 | # Reduce Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + More than one axis can be set at same time, for example: `axes=(1<<2)+(1<<3)` 10 | 11 | + Alternative values of `trt.ReduceOperation` 12 | 13 | | Name | Comment | initialization values (float/int) | 14 | | :--: | :-----: | :-------------------------------: | 15 | | SUM | sum | 0 | 16 | | PROD | product | 1 | 17 | | AVG | average | $-\infty$ / INT_MIN / -128 | 18 | | MAX | maximum | $\infty$ / INT_MAX / 127 | 19 | | MIN | minimum | NaN / 0 / -128 | 20 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Reduce/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Resize/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/ReverseSequence/README.md: -------------------------------------------------------------------------------- 1 | # Reverse Sequence Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Scale/README.md: -------------------------------------------------------------------------------- 1 | # Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Alternative values of `trt.ScaleMode` 10 | | Name | Comment | 11 | | :--------------: | :-------------------------: | 12 | | ELEMENTWISE | | 13 | | UNIFORM | | 14 | | CHANNEL | | 15 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Scale/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Select/README.md: -------------------------------------------------------------------------------- 1 | # Select Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Select/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Shape/README.md: -------------------------------------------------------------------------------- 1 | # Shape Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Shape/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Shuffle/README.md: -------------------------------------------------------------------------------- 1 | # Shuffle Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Slice/README.md: -------------------------------------------------------------------------------- 1 | # Slice Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Alternative values of `trt.SampleMode` 10 | | Name | Comment | 11 | | :--------------: | :-----------------------------------------------: | 12 | | DEFAULT | Error if load element out of bound | 13 | | WRAP | data[i] = data[i%w] | 14 | | CLAMP | data[i] = data[w-1] | 15 | | FILL | data[i] = fillValue | 16 | | REFLECT | data[i] = data[(w-1-i%w)*(i/w%2)+(i%w)*(1-i/w%2)] | 17 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Slice/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Softmax/README.md: -------------------------------------------------------------------------------- 1 | # Softmax Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Axes can not be set more than one, for example, `axes=(1<<2)+(1<<3)`. 10 | 11 | + Default values of parameters 12 | 13 | | Name | Comment | 14 | | :----------: | :----------------: | 15 | | axes | 1 << $max\left{0, Rank\left(tensor\right)-3\right} | 16 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Squeeze/README.md: -------------------------------------------------------------------------------- 1 | # Squeeze Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/TopK/README.md: -------------------------------------------------------------------------------- 1 | # TopK Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Entry with smaller index will be selected if they own the same value. 10 | 11 | + Axes can not be set more than one, for example, `axes=(1<<2)+(1<<3)`. 12 | 13 | + Alternative values of `trt.TopKOperation` 14 | 15 | | Name | Comment | 16 | | :--: | :---------------------: | 17 | | MAX | Get values from maximum | 18 | | MIN | Get values from minimum | 19 | 20 | + Ranges of parameters 21 | 22 | | Name | Range | 23 | | :--: | :--------: | 24 | | K | $\le 3840$ | 25 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/TopK/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/TripLimit/README.md: -------------------------------------------------------------------------------- 1 | # Trip Limit Layer 2 | 3 | + Refer to `../LoopStructure`. 4 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/TripLimit/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Unary/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/Unsqueeze/README.md: -------------------------------------------------------------------------------- 1 | # Squeeze Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/Layer/template/README.md: -------------------------------------------------------------------------------- 1 | # Layer 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Alternative values of `trt.` 10 | + 11 | | Name | Comment | 12 | | :-----------------: | :-----: | 13 | 14 | 15 | + Ranges of parameters 16 | 17 | | Name | Range | 18 | | :------------------: | :-----------: | 19 | 20 | 21 | + Default values of parameters 22 | 23 | | Name | Comment | 24 | | :----------: | :----------------: | 25 | -------------------------------------------------------------------------------- /cookbook/02-API/Network/README.md: -------------------------------------------------------------------------------- 1 | # Network 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/Network/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/ONNXParser/README.md: -------------------------------------------------------------------------------- 1 | # ONNX Parser 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/ONNXParser/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/OptimizationProfile/README.md: -------------------------------------------------------------------------------- 1 | # Optimization-Profile 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Refer to `class TRTWrapperV1` and `class TRTWrapperShapeInput` in file `include/utils_class.py` to see a complete usage of Optimization-Profile. 10 | + Refer to file `02-API/ExecutionContext/main.py` to see a short usage example of Optimization-Profile. 11 | -------------------------------------------------------------------------------- /cookbook/02-API/README.md: -------------------------------------------------------------------------------- 1 | # 02-API 2 | 3 | + Examples of APIs in TensorRT shown in Python, since those are mostly one-to-one correspondence to C++. 4 | 5 | ## Builder 6 | 7 | + Simple usage of Builder's API. 8 | 9 | ## BuilderConfig 10 | 11 | + Simple usage of BuilderConfig's APIs. 12 | 13 | ## CudaEngine 14 | 15 | + Simple usage of CudaEngine's API. 16 | 17 | ## ExecutionContext 18 | 19 | + Simple usage of ExecutionContext's API. 20 | 21 | ## HostMemory 22 | 23 | + Simple usage of HostMemory's API. 24 | 25 | ## Layer 26 | 27 | + Information of the layers are listed in [**Operators Document**](https://docs.nvidia.com/deeplearning/tensorrt/operators/docs/), here we just provide some more examples. 28 | 29 | ## Network 30 | 31 | + Steps to run. 32 | 33 | ## ONNX Parser 34 | 35 | + Steps to run. 36 | 37 | ## Optimization-Profile 38 | 39 | + Steps to run. 40 | 41 | ## Runtime 42 | 43 | + Steps to run. 44 | 45 | ## Tensor 46 | 47 | + Steps to run. 48 | -------------------------------------------------------------------------------- /cookbook/02-API/Runtime/README.md: -------------------------------------------------------------------------------- 1 | # Runtime 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/Runtime/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/02-API/Tensor/README.md: -------------------------------------------------------------------------------- 1 | # Tensor 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/02-API/Tensor/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/03-Workflow/Paddlepaddle-ONNX-TensorRT/.gitignore: -------------------------------------------------------------------------------- 1 | paddle_model_temp_dir/ 2 | -------------------------------------------------------------------------------- /cookbook/03-Workflow/Paddlepaddle-ONNX-TensorRT/README.md: -------------------------------------------------------------------------------- 1 | # Workflow of Paddlepaddle -> ONNX -> TensorRT 2 | 3 | + A workflow of: export trained model from Paddlepaddle to ONNX, parse ONNX in TensorRT, build TensorRT engine and do inference. 4 | 5 | + Need package `paddlepaddle` and `paddle2onnx` which are not included in the docker image. 6 | 7 | + Steps to run. 8 | 9 | ```bash 10 | python3 main.py 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/03-Workflow/Paddlepaddle-ONNX-TensorRT/requirements.txt: -------------------------------------------------------------------------------- 1 | paddlepaddle 2 | paddle2onnx 3 | -------------------------------------------------------------------------------- /cookbook/03-Workflow/README.md: -------------------------------------------------------------------------------- 1 | # 03-Workflow 2 | 3 | + Common workflow of using TensorRT from DL frameworks. 4 | 5 | ## Workflow of pyTorch -> ONNX -> TensorRT 6 | 7 | + Steps to run. 8 | 9 | ## Workflow of pyTorch -> TensorRT 10 | 11 | + Steps to run. 12 | -------------------------------------------------------------------------------- /cookbook/03-Workflow/TensorFlow2-ONNX-TensorRT/.gitignore: -------------------------------------------------------------------------------- 1 | pbModel/ 2 | -------------------------------------------------------------------------------- /cookbook/03-Workflow/TensorFlow2-ONNX-TensorRT/README.md: -------------------------------------------------------------------------------- 1 | # Workflow of TensorFlow2 -> ONNX -> TensorRT 2 | 3 | + A workflow of: export trained model from TensorFlow2 to ONNX, parse ONNX in TensorRT, build TensorRT engine and do inference. 4 | 5 | + Need package `tensorflow` which are not included in the docker image. 6 | 7 | + Steps to run. 8 | 9 | ```bash 10 | python3 main.py 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/03-Workflow/TensorFlow2-ONNX-TensorRT/requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow 2 | tf2onnx 3 | -------------------------------------------------------------------------------- /cookbook/03-Workflow/build-README.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from tensorrt_cookbook import build_readme 17 | 18 | outline = \ 19 | """ 20 | + Common workflow of using TensorRT from DL frameworks. 21 | """ 22 | 23 | build_readme(__file__, outline) 24 | -------------------------------------------------------------------------------- /cookbook/03-Workflow/pyTorch-ONNX-TensorRT/C++/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | MORE_LIB = -lnvonnxparser -lz 4 | 5 | all: main.exe 6 | 7 | main.exe: main.o cookbookHelper.o cnpy.o calibrator.o 8 | $(COMMAND_EXE) $(MORE_LIB) 9 | 10 | main.o: main.cpp 11 | $(COMMAND_O) 12 | 13 | .PHONY: test 14 | test: 15 | make clean; make all -j; rm -rf *.trt *.*.Int8Cache 16 | ./main.exe > log-main.exe.log 17 | 18 | .PHONY: clean 19 | clean: 20 | rm -rf *.d *.o *.exe *.trt *.Int8Cache 21 | -------------------------------------------------------------------------------- /cookbook/03-Workflow/pyTorch-ONNX-TensorRT/README.md: -------------------------------------------------------------------------------- 1 | # Workflow of pyTorch -> ONNX -> TensorRT 2 | 3 | + A workflow of: export trained model from pyTorch to ONNX, parse ONNX in TensorRT, build TensorRT engine and do inference. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | 11 | + The process of build and export a ONNX model is inside `00-Data/get_model_part1.py`. 12 | 13 | + Here are some independent cases: 14 | + Normal: use FP32 to work (original ONNX is in FP32 mode). 15 | + FP16: use FP16 to work. 16 | + INT8-PTQ: use INT8-PTQ to work, it shows the usage of calibrator as well. 17 | + INT8-QAT: use INT8-QAT to work, we need a INT8-QAT ONNX as input. 18 | 19 | + Here is a equivalent INT8-PTQ workflow in `C++` directory, the key point is the calibrator, comparing to the example in 01-SimpleDemo 20 | 21 | ```bash 22 | cd C++ 23 | make clean && make 24 | ./main.exe 25 | ``` 26 | -------------------------------------------------------------------------------- /cookbook/03-Workflow/pyTorch-TensorRT/C++/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | MORE_LIB = -lz 4 | 5 | all: main.exe 6 | 7 | main.exe: main.o cookbookHelper.o cnpy.o calibrator.o 8 | $(COMMAND_EXE) $(MORE_LIB) 9 | 10 | main.o: main.cpp 11 | $(COMMAND_O) 12 | 13 | .PHONY: test 14 | test: 15 | make clean; make all -j; rm -rf *.trt *.*.Int8Cache 16 | ./main.exe > log-main.exe.log 17 | 18 | .PHONY: clean 19 | clean: 20 | rm -rf *.d *.o *.exe *.trt *.Int8Cache 21 | -------------------------------------------------------------------------------- /cookbook/03-Workflow/pyTorch-TensorRT/README.md: -------------------------------------------------------------------------------- 1 | # Workflow of pyTorch -> TensorRT 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Here is a workflow of: 10 | + export weights of a trained model from pyTorch to NPZ file. 11 | + Build TensorRT engine with the weights and do inference 12 | 13 | + We need to run `00-Data/get_model.py` firstly to get related weights files as input. 14 | 15 | + Here are some independent cases: 16 | + Normal: use FP32 to work (original ONNX is in FP32 mode). 17 | + FP16: use FP16 to work. 18 | + INT8-PTQ: use INT8-PTQ to work, it shows the usage of calibrator as well. 19 | 20 | + Here is a equivalent INT8-PTQ workflow in `C++` directory, the key point is the calibrator, comparing to the example in 01-SimpleDemo 21 | 22 | ```bash 23 | cd C++ 24 | make clean && make 25 | ./main.exe 26 | ``` 27 | -------------------------------------------------------------------------------- /cookbook/04-Feature/AlgorithmSelector/README.md: -------------------------------------------------------------------------------- 1 | # Algorithm Selector 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/AuxStream/README.md: -------------------------------------------------------------------------------- 1 | # Aux Stream 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/AuxStream/max_aux_streams=0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/cookbook/04-Feature/AuxStream/max_aux_streams=0.png -------------------------------------------------------------------------------- /cookbook/04-Feature/AuxStream/max_aux_streams=1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/cookbook/04-Feature/AuxStream/max_aux_streams=1.png -------------------------------------------------------------------------------- /cookbook/04-Feature/AuxStream/max_aux_streams=2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/cookbook/04-Feature/AuxStream/max_aux_streams=2.png -------------------------------------------------------------------------------- /cookbook/04-Feature/BuilderOptimizationLevel/README.md: -------------------------------------------------------------------------------- 1 | # Builder Optimization Level 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/DebugTensor/README.md: -------------------------------------------------------------------------------- 1 | # Debug Tensor 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/EngineInspector/README.md: -------------------------------------------------------------------------------- 1 | # Engine Inspector 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/ErrorRecorder/README.md: -------------------------------------------------------------------------------- 1 | # ErrorRecorder 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/ExecutionContextAllocationStrategy/README.md: -------------------------------------------------------------------------------- 1 | # Execution Context Allocation Strategy 2 | 3 | + Steps to run 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/GPUAllocator/README.md: -------------------------------------------------------------------------------- 1 | # GPU Allocator 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/HardwareCompatibility/README.md: -------------------------------------------------------------------------------- 1 | # Hardware Compatibility 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Build a hardware-compatibility TensorRT engine which can run on different device. 10 | 11 | + GPU of both building or running must be Ampere or above. 12 | -------------------------------------------------------------------------------- /cookbook/04-Feature/LabeledDimension/README.md: -------------------------------------------------------------------------------- 1 | # Labeled Dimension 2 | 3 | + Steps to run 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/LeanAndDispatchRuntime/README.md: -------------------------------------------------------------------------------- 1 | # Lean and Dispatch Rutnime 2 | 3 | + Use Lean and Dispatch Rutnime to do inference. 4 | 5 | + We need packages of `tensorrt_lean` and `tensorrt_dispatch` respectively. 6 | 7 | + Steps to run. 8 | 9 | ```bash 10 | python3 main.py 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/04-Feature/Logger/README.md: -------------------------------------------------------------------------------- 1 | # Logger 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/Logger/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/04-Feature/OutputAllocator/README.md: -------------------------------------------------------------------------------- 1 | # Output Allocator 2 | 3 | + Refer to `02-API/Layer/NonzeroLayer`. 4 | -------------------------------------------------------------------------------- /cookbook/04-Feature/OutputAllocator/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/04-Feature/Profiler/README.md: -------------------------------------------------------------------------------- 1 | # Profiler 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/Profiler/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/04-Feature/ProfilingVerbosity/README.md: -------------------------------------------------------------------------------- 1 | # Profiling Verbosity 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/ProgressMonitor/README.md: -------------------------------------------------------------------------------- 1 | # Porgress Monitor 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/README.md: -------------------------------------------------------------------------------- 1 | # 04-Feature 2 | 3 | + Examples of the feature APIs, which are not necessary in a basic workflow. 4 | 5 | ## 6 | 7 | ## Steps to run 8 | 9 | ## Timing Cache 10 | 11 | + Steps to run. 12 | 13 | ## Version Compatibility 14 | 15 | + Steps to run. 16 | 17 | ## Weight Streaming 18 | 19 | + Steps to run. 20 | -------------------------------------------------------------------------------- /cookbook/04-Feature/Refit/README.md: -------------------------------------------------------------------------------- 1 | # Refit 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Several methods to refit a TensorRT engine. 10 | 11 | + In case `case_dummy_engine`, we build a dummy engine with untrained weights. 12 | + Here we also show the usage of stripping weights engine, i.e., building a TensorRT engine wwithout weights inside. 13 | + We can see the size of stripping weights engine file (453 KiB) is much smaller than normal one (13 MiB) with weights inside. 14 | 15 | + In case `case_set_weights`, we refit the dummay engine with new weights, it contains two equivalent implementations. 16 | 17 | + In case `case_set_weights_gpu`, we refit the dummay engine with new weights, which GPU is on GPU. 18 | 19 | + In case `case_refit_engine`, we refit the dummy engine with trained ONNX file directly, it contains three equivalent implementations. 20 | -------------------------------------------------------------------------------- /cookbook/04-Feature/Safety/README.md: -------------------------------------------------------------------------------- 1 | # Safety mode 2 | 3 | + Safety mode is only for Drive Platform (QNX). 4 | 5 | + https://github.com/NVIDIA/TensorRT/issues/2156 6 | -------------------------------------------------------------------------------- /cookbook/04-Feature/Safety/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/04-Feature/SerializationConfig/README.md: -------------------------------------------------------------------------------- 1 | # Serialization Config 2 | 3 | + Refer to `02-API/CudaEngine`. 4 | -------------------------------------------------------------------------------- /cookbook/04-Feature/SerializationConfig/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/04-Feature/Sparsity/README.md: -------------------------------------------------------------------------------- 1 | # 2 | 3 | ## Steps to run 4 | 5 | ```shell 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/Sparsity/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/04-Feature/TimingCache/README.md: -------------------------------------------------------------------------------- 1 | # Timing Cache 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/VersionCompatibility/README.md: -------------------------------------------------------------------------------- 1 | # Version Compatibility 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | 9 | + Build a hardware-compatibility TensorRT engine which can run on different TensorRT version. 10 | 11 | + GPU of both building or running must be Ampere or above. 12 | -------------------------------------------------------------------------------- /cookbook/04-Feature/WeightStreaming/README.md: -------------------------------------------------------------------------------- 1 | # Weight Streaming 2 | 3 | + Steps to run. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/04-Feature/build-README.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from tensorrt_cookbook import build_readme 17 | 18 | outline = \ 19 | """ 20 | + Examples of the feature APIs, which are not necessary in a basic workflow. 21 | """ 22 | 23 | build_readme(__file__, outline) 24 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/APIs-V2-deprecated/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) main.exe 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | main.exe: main.o cookbookHelper.o $(PLUGIN_O) 16 | $(COMMAND_EXE) 17 | 18 | main.o: main.cpp 19 | $(COMMAND_O) 20 | 21 | .PHONY: test 22 | test: 23 | make clean; make all -j; rm -rf *.trt 24 | ./main.exe > log-main.exe.log 25 | 26 | .PHONY: clean 27 | clean: 28 | rm -rf *.d *.o *.so *.exe *.trt 29 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/APIs-V2-deprecated/README.md: -------------------------------------------------------------------------------- 1 | # APIs-V2-deprecated 2 | 3 | + The same as `APIs`, but using `IPluginV2DynamicExt` (deprecated) class. 4 | 5 | + Use class `PluginV3`. 6 | 7 | + Steps to run. 8 | 9 | ```bash 10 | make test 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/APIs/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) main.exe 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | main.exe: main.o cookbookHelper.o $(PLUGIN_O) 16 | $(COMMAND_EXE) 17 | 18 | main.o: main.cpp 19 | $(COMMAND_O) 20 | 21 | .PHONY: test 22 | test: 23 | make clean; make all -j; rm -rf *.trt 24 | ./main.exe > log-main.exe.log 25 | 26 | .PHONY: clean 27 | clean: 28 | rm -rf *.d *.o *.so *.exe *.trt 29 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/APIs/README.md: -------------------------------------------------------------------------------- 1 | # APIs 2 | 3 | + Example of showing all APIs of plugin. 4 | 5 | + Use class `PluginV3`. 6 | 7 | + Steps to run. 8 | 9 | ```bash 10 | make test 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/BasicExample-V2-deprecated/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) main.exe 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | main.exe: main.o cookbookHelper.o $(PLUGIN_O) 16 | $(COMMAND_EXE) 17 | 18 | main.o: main.cpp 19 | $(COMMAND_O) 20 | 21 | .PHONY: test 22 | test: 23 | make clean; make all -j; rm -rf *.trt 24 | ./main.exe > log-main.exe.log 25 | 26 | .PHONY: clean 27 | clean: 28 | rm -rf *.d *.o *.so *.exe *.trt 29 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/BasicExample-V2-deprecated/README.md: -------------------------------------------------------------------------------- 1 | # Basic Example - V2DynamicExt (deprecated) 2 | 3 | + The same as Basic Example, but using `IPluginV2DynamicExt` (deprecated) class. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | make test 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/BasicExample/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) main.exe 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | main.exe: main.o cookbookHelper.o $(PLUGIN_O) 16 | $(COMMAND_EXE) 17 | 18 | main.o: main.cpp 19 | $(COMMAND_O) 20 | 21 | .PHONY: test 22 | test: 23 | make clean; make all -j; rm -rf *.trt 24 | ./main.exe > log-main.exe.log 25 | 26 | .PHONY: clean 27 | clean: 28 | rm -rf *.d *.o *.so *.exe *.trt 29 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/BasicExample/README.md: -------------------------------------------------------------------------------- 1 | # Basic Example 2 | 3 | + Basic example of using `PluginV3` to add a scalar onto the input tensor. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | make test 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/DataDependentShape/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | .PHONY: test 16 | test: 17 | make clean; make all -j; rm -rf *.trt 18 | 19 | .PHONY: clean 20 | clean: 21 | rm -rf *.d *.o *.so *.exe *.trt 22 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/DataDependentShape/README.md: -------------------------------------------------------------------------------- 1 | # Data Dependent Shape 2 | 3 | + Example of using a Data-Dependent-Shape plugin to move all non-zero elements to the left side. 4 | 5 | + There are two output tensors from the plugin: 6 | + 0: the compressed tensor. 7 | + 1: maximum of non-zeros elements among rows. 8 | 9 | + For example: 10 | 11 | $$ 12 | \text{input} = 13 | \left[\begin{matrix} 14 | 0 & 1 & 2 & 0 \\ 3 & 4 & 5 & 0 \\ 0 & 6 & 0 & 7 \\ 0 & 0 & 0 & 0 15 | \end{matrix}\right] 16 | , 17 | \text{output} = 18 | \left[\begin{matrix} 19 | 1 & 2 & 0 \\ 3 & 4 & 5 \\ 6 & 7 & 0 \\ 0 & 0 & 0 20 | \end{matrix}\right] 21 | $$ 22 | 23 | + Steps to run. 24 | 25 | ```bash 26 | make test 27 | ``` 28 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/IdentityPlugin/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | .PHONY: test 16 | test: 17 | make clean; make all -j; rm -rf *.trt 18 | 19 | .PHONY: clean 20 | clean: 21 | rm -rf *.d *.o *.so *.exe *.trt 22 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/IdentityPlugin/README.md: -------------------------------------------------------------------------------- 1 | # Identity plugin 2 | 3 | + Basic example of using `PluginV3` to add a scalar onto the input tensor. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | make test 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/InPlacePlugin/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | .PHONY: test 16 | test: 17 | make clean; make all -j; rm -rf *.trt 18 | 19 | .PHONY: clean 20 | clean: 21 | rm -rf *.d *.o *.so *.exe *.trt 22 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/InPlacePlugin/README.md: -------------------------------------------------------------------------------- 1 | # In-Place Plugin 2 | 3 | + The same as Basic Example, but use in-place plugin (input and output tensor share the same buffer). 4 | 5 | + Enable since TensorRT-10.3.28. 6 | 7 | + Steps to run. 8 | 9 | ```bash 10 | make test 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/MultiVersion/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | .PHONY: test 16 | test: 17 | make clean; make all -j; rm -rf *.trt 18 | 19 | .PHONY: clean 20 | clean: 21 | rm -rf *.d *.o *.so *.exe *.trt 22 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/MultiVersion/README.md: -------------------------------------------------------------------------------- 1 | # Multi-Version 2 | 3 | + The same as BasicExample, but multiple versions of the plugin are provided to be chose at runtime. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | make test 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/ONNXParserWithPlugin/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | .PHONY: test 16 | test: 17 | make clean; make all -j; rm -rf *.trt 18 | 19 | .PHONY: clean 20 | clean: 21 | rm -rf *.d *.o *.so *.exe *.trt 22 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/ONNXParserWithPlugin/README.md: -------------------------------------------------------------------------------- 1 | # ONNX Parser and Plugin 2 | 3 | + Example of combinating the usage of model from ONNX and plugin. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | make test 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/PassHostData/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | .PHONY: test 16 | test: 17 | make clean; make all -j; rm -rf *.trt 18 | 19 | .PHONY: clean 20 | clean: 21 | rm -rf *.d *.o *.so *.exe *.trt 22 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/PassHostData/README.md: -------------------------------------------------------------------------------- 1 | # Pass Host Data 2 | 3 | + Example of passing a host pointer (pointing to anything like array, structure or even nullptr) into plugin at runtime. 4 | 5 | + This is usually useful in C++ environment, which we can pass a complex structure or even nullptr into plugin without constrian of TensorRT's data type or nullptr check. 6 | 7 | + Steps to run. 8 | 9 | ```bash 10 | make test 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/PluginInsideEngine-C++/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) main.exe 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | main.exe: main.o cookbookHelper.o $(PLUGIN_O) 16 | $(COMMAND_EXE) 17 | 18 | main.o: main.cpp 19 | $(COMMAND_O) 20 | 21 | .PHONY: test 22 | test: 23 | make clean; make all -j; rm -rf *.trt 24 | ./main.exe > log-main.exe.log 25 | 26 | .PHONY: clean 27 | clean: 28 | rm -rf *.d *.o *.so *.exe *.trt 29 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/PluginInsideEngine-C++/README.md: -------------------------------------------------------------------------------- 1 | # Plugin Inside Engine - C++ 2 | 3 | + Example of serializing a plugin inside a TensorRT engine (no `.so` needed at runtime) using C++ APIs. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | make test 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/PluginInsideEngine-Python/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | .PHONY: test 16 | test: 17 | make clean; make all -j; rm -rf *.trt 18 | 19 | .PHONY: clean 20 | clean: 21 | rm -rf *.d *.o *.so *.exe *.trt 22 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/PluginInsideEngine-Python/README.md: -------------------------------------------------------------------------------- 1 | # Plugin Inside Engine - Python 2 | 3 | + Example of serializing a plugin inside a TensorRT engine (no `.so` needed at runtime) using C++ APIs. 4 | 5 | ```bash 6 | make test 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/ShapeInputTensor/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | .PHONY: test 16 | test: 17 | make clean; make all -j; rm -rf *.trt 18 | 19 | .PHONY: clean 20 | clean: 21 | rm -rf *.d *.o *.so *.exe *.trt 22 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/ShapeInputTensor/README.md: -------------------------------------------------------------------------------- 1 | # Shape Input Tensor 2 | 3 | + Example of sending a shape input tensor into plugin to reshape another execution tensor by the values of it. 4 | 5 | + ONNX parser is not able to work for this feature in TensoRT. 6 | 7 | + Steps to run. 8 | 9 | ```bash 10 | make test 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/Tactic+TimingCache/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | .PHONY: test 16 | test: 17 | make clean; make all -j; rm -rf *.trt 18 | 19 | .PHONY: clean 20 | clean: 21 | rm -rf *.d *.o *.so *.exe *.trt 22 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/Tactic+TimingCache/README.md: -------------------------------------------------------------------------------- 1 | # Tactic+TimingCache 2 | 3 | + The same as BasicExample, but we use our own tactics and timing-cache in the plugin. 4 | 5 | + To understand timing-cache generally, refer to `02-API/TimingCache`. 6 | 7 | + In this example we create a network with two same plugin layers, and build the engine two times. 8 | 9 | + For each plugin layer, we provide two custom tactics for trying. 10 | 11 | + So we can see TensorRT profiling the two custom tactics and reuse the result cross layers and buildings by timing cache. 12 | 13 | + We can use VERBOSE log and add `-DDEBUG` into compilation commands to see the process of TensorRT building. 14 | 15 | + Steps to run. 16 | 17 | ```bash 18 | make test 19 | ``` 20 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/UseCuBLAS-V2-deprecated/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | MORE_LIB = -lcublas 4 | PLUGIN_CU = $(shell find . -name '*.cu') 5 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 6 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 7 | 8 | all: $(PLUGIN_SO) 9 | 10 | %.so: %.o cookbookHelper.o 11 | $(COMMAND_SO) $(MORE_LIB) 12 | 13 | $(PLUGIN_O): $(PLUGIN_CU) 14 | $(COMMAND_O) 15 | 16 | .PHONY: test 17 | test: 18 | make clean; make all -j; rm -rf *.trt 19 | 20 | .PHONY: clean 21 | clean: 22 | rm -rf *.d *.o *.so *.exe *.trt 23 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/UseCuBLAS-V2-deprecated/README.md: -------------------------------------------------------------------------------- 1 | # Use cuBLAS 2 | 3 | + Example of using cuBLAS in plugin. 4 | 5 | ## Steps to run 6 | 7 | ```shell 8 | make test 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/UseCuBLAS/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | MORE_LIB = -lcublas 4 | PLUGIN_CU = $(shell find . -name '*.cu') 5 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 6 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 7 | 8 | all: $(PLUGIN_SO) useCuBLASAlone.exe 9 | 10 | %.so: %.o cookbookHelper.o 11 | $(COMMAND_SO) $(MORE_LIB) 12 | 13 | $(PLUGIN_O): $(PLUGIN_CU) 14 | $(COMMAND_O) 15 | 16 | useCuBLASAlone.exe: useCuBLASAlone.o 17 | $(COMMAND_EXE) $(MORE_LIB) 18 | 19 | useCuBLASAlone.o: useCuBLASAlone.cpp 20 | $(COMMAND_O) 21 | 22 | .PHONY: test 23 | test: 24 | make clean; make all -j; rm -rf *.trt 25 | ./useCuBLASAlone.exe > log-useCuBLASAlone.exe.log 26 | 27 | .PHONY: clean 28 | clean: 29 | rm -rf *.d *.o *.so *.exe *.trt 30 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/UseCuBLAS/README.md: -------------------------------------------------------------------------------- 1 | # Use cuBLAS 2 | 3 | + Example of using cuBLAS in plugin. 4 | 5 | ## Steps to run 6 | 7 | ```shell 8 | make test 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/UseFP16/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | .PHONY: test 16 | test: 17 | make clean; make all -j; rm -rf *.trt 18 | 19 | .PHONY: clean 20 | clean: 21 | rm -rf *.d *.o *.so *.exe *.trt 22 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/UseFP16/README.md: -------------------------------------------------------------------------------- 1 | # UseFP16 2 | 3 | + The same as BasicExample, but enabling FP16 mode. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | make test 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/UseINT8-PTQ-V2-deprecated/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | .PHONY: test 16 | test: 17 | make clean; make all -j; rm -rf *.trt 18 | 19 | .PHONY: clean 20 | clean: 21 | rm -rf *.d *.o *.so *.exe *.trt 22 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/UseINT8-PTQ-V2-deprecated/README.md: -------------------------------------------------------------------------------- 1 | # UseINT8-PTQ - V2DynamicExt (deprecated) 2 | 3 | + The same as BasicExample, but enabling INT8-PTQ mode with `IPluginV2DynamicExt` (deprecated). 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | make test 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/UseINT8-PTQ-V2-deprecated/model.int8cache: -------------------------------------------------------------------------------- 1 | TRT-100600-EntropyCalibration2 2 | inputT0: 3ebfcfb9 3 | (Unnamed Layer* 0) [PluginV2DynamicExt]_output: 3ec3d80a 4 | outputT0: 3ec3d80a 5 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/UseINT8-PTQ-error/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | PLUGIN_CU = $(shell find . -name '*.cu') 4 | PLUGIN_O = $(PLUGIN_CU:.cu=.o) 5 | PLUGIN_SO = $(PLUGIN_CU:.cu=.so) 6 | 7 | all: $(PLUGIN_SO) 8 | 9 | %.so: %.o cookbookHelper.o 10 | $(COMMAND_SO) 11 | 12 | $(PLUGIN_O): $(PLUGIN_CU) 13 | $(COMMAND_O) 14 | 15 | .PHONY: test 16 | test: 17 | make clean; make all -j; rm -rf *.trt 18 | 19 | .PHONY: clean 20 | clean: 21 | rm -rf *.d *.o *.so *.exe *.trt 22 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/UseINT8-PTQ-error/README.md: -------------------------------------------------------------------------------- 1 | # UseINT8-PTQ 2 | 3 | + The same as BasicExample, but enabling INT8-PTQ mode. 4 | 5 | + TODO 6 | + The output is incorrect. 7 | 8 | + Steps to run. 9 | 10 | ```bash 11 | make test 12 | ``` 13 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/UseINT8-PTQ-error/model.int8cache: -------------------------------------------------------------------------------- 1 | TRT-100001-EntropyCalibration2 2 | inputT0: 3ebfcfb9 3 | (Unnamed Layer* 0) [PluginV3_V3ONE]_output: 3ec3d80a 4 | outputT0: 3ec3d80a 5 | -------------------------------------------------------------------------------- /cookbook/05-Plugin/build-README.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from tensorrt_cookbook import build_readme 17 | 18 | outline = \ 19 | """ 20 | + Examples of using TensorRT plugins. 21 | """ 22 | 23 | build_readme(__file__, outline) 24 | -------------------------------------------------------------------------------- /cookbook/06-DLFrameworkTRT/README.md: -------------------------------------------------------------------------------- 1 | # 06-DLFrameworkTRT 2 | 3 | + TensorRT APIs in other Deep Learning Framework. 4 | 5 | ## Torch-TensorRT 6 | 7 | + Use Torch-TensorRT to convert a torch network into torch-script and run it. 8 | -------------------------------------------------------------------------------- /cookbook/06-DLFrameworkTRT/Torch-TensorRT/README.md: -------------------------------------------------------------------------------- 1 | # Torch-TensorRT 2 | 3 | + Use Torch-TensorRT to convert a torch network into torch-script and run it. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/06-DLFrameworkTRT/build-README.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from tensorrt_cookbook import build_readme 17 | 18 | outline = \ 19 | """ 20 | + TensorRT APIs in other Deep Learning Framework. 21 | """ 22 | 23 | build_readme(__file__, outline) 24 | -------------------------------------------------------------------------------- /cookbook/07-Tool/CheckTorchOperator/README.md: -------------------------------------------------------------------------------- 1 | # Check pyTorch Operator 2 | 3 | + A template to check whether a operator in pyTorch can be parsed into TensorRT. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/07-Tool/ContextPrinter/README.md: -------------------------------------------------------------------------------- 1 | # Context Printer 2 | 3 | + Print information of input / output shape set for the context. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/07-Tool/EnginePrinter/README.md: -------------------------------------------------------------------------------- 1 | # Engine Printer 2 | 3 | + Print information of layers and tensors in the TensorRT engine. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/07-Tool/FP16Tuning/.gitignore: -------------------------------------------------------------------------------- 1 | *.txt 2 | -------------------------------------------------------------------------------- /cookbook/07-Tool/FP16Tuning/README.md: -------------------------------------------------------------------------------- 1 | # FP16 Tuning 2 | 3 | + Thanks Xuewei Li for providing the solution 4 | + Steps to run 5 | 6 | ```bash 7 | python3 main.py 8 | ``` 9 | -------------------------------------------------------------------------------- /cookbook/07-Tool/ListAPIs/.gitignore: -------------------------------------------------------------------------------- 1 | !*.log 2 | -------------------------------------------------------------------------------- /cookbook/07-Tool/ListAPIs/README.md: -------------------------------------------------------------------------------- 1 | # List APIs 2 | 3 | + List all the APIs in TensorRT package. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/07-Tool/ListAPIs/log-main.py.log: -------------------------------------------------------------------------------- 1 | Finish 2 | -------------------------------------------------------------------------------- /cookbook/07-Tool/ListAPIs/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Netron/README.md: -------------------------------------------------------------------------------- 1 | # Netron 2 | 3 | + A visualization tool of neural network, which supports **ONNX** and many other frameworks. 4 | 5 | + [Repo](https://github.com/lutzroeder/Netron) on GitHub. 6 | 7 | + Original repository and download website: [Link](https://github.com/lutzroeder/Netron) 8 | 9 | + We need to run `00-Data/get_model.py` firstly, then we can open `00-Data/model*.onnx` by Netron. 10 | 11 | + We usually use this tool to check metadata of compute graph, structure of network, node information and tensor information of the model. 12 | 13 | + Other interesting tools: 14 | + **Netron-online** [Link](https://netron.app/), which is an online version of Netron. 15 | + **onnx-modifier** [Link](https://github.com/ZhangGe6/onnx-modifier), which can edit ONNX in UI. But it becomes much slower if ONNX file is large, and the type of graph edition are limited. 16 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Netron/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/07-Tool/NetworkPrinter/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | all: main.exe 4 | 5 | main.exe: main.o cookbookHelper.o 6 | $(COMMAND_EXE) 7 | 8 | main.o: main.cpp 9 | $(COMMAND_O) 10 | 11 | .PHONY: test 12 | test: 13 | make clean; make all -j; rm -rf *.trt 14 | ./main.exe > log-main.exe.log 15 | 16 | .PHONY: clean 17 | clean: 18 | rm -rf *.d *.o *.exe *.trt 19 | -------------------------------------------------------------------------------- /cookbook/07-Tool/NetworkPrinter/README.md: -------------------------------------------------------------------------------- 1 | # Network Printer 2 | 3 | + Print information of layers and tensors in the network. 4 | 5 | + This is NOT for printing information in the TensorRT engine, use Engine Inspector for this purpose. 6 | 7 | + In this example, we create a simple network and then print it. 8 | 9 | + For Python workflow. 10 | 11 | ```bash 12 | python3 main.py 13 | ``` 14 | 15 | + For C++ workflow. 16 | 17 | ```bash 18 | make 19 | ./main.exe 20 | ``` 21 | -------------------------------------------------------------------------------- /cookbook/07-Tool/NetworkPrinter/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | make test 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | make clean 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/07-Tool/NetworkSerialization/C++-TODO/Makefile: -------------------------------------------------------------------------------- 1 | include ./Makefile.inc 2 | 3 | LDFLAG += -lz 4 | SOURCE_CPP = $(shell find . -name '*.cpp' 2>/dev/null) 5 | OBJ = $(shell find . -name *.o 2>/dev/null) 6 | DEP = $(OBJ:.o=.d) 7 | TARGET_SO = main.exe 8 | 9 | -include $(DEP) 10 | 11 | all: $(TARGET_SO) 12 | 13 | main.exe: main.o cnpy.o 14 | $(NVCC) $(LDFLAG) $(LD_LZ) -o $@ $+ 15 | 16 | %.o: %.cpp 17 | $(NVCC) $(CUFLAG) $(INCLUDE) -M -MT $@ -o $(@:.o=.d) $< 18 | $(NVCC) $(CUFLAG) $(INCLUDE) -o $@ -c $< 19 | 20 | cnpy.o: cnpy.cpp 21 | $(NVCC) $(CCFLAG) $(INCLUDE) -M -MT $@ -o $(@:.o=.d) $< 22 | $(NVCC) $(CCFLAG) $(INCLUDE) -o $@ -c $< 23 | 24 | .PHONY: test 25 | test: 26 | make clean 27 | make -j2 28 | 29 | .PHONY: clean 30 | clean: 31 | rm -rf ./*.d ./*.o ./*.exe ./*.trt 32 | -------------------------------------------------------------------------------- /cookbook/07-Tool/NetworkSerialization/C++-TODO/Makefile.inc: -------------------------------------------------------------------------------- 1 | CUDA_PATH = /usr/local/cuda 2 | NVCC = $(CUDA_PATH)/bin/nvcc 3 | TRT_INC_PATH = /usr/include/x86_64-linux-gnu 4 | TRT_LIB_PATH = /usr/lib/x86_64-linux-gnu 5 | GENCODE = -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_86,code=sm_86 6 | DEBUG_MACRO = -UDEBUG 7 | WARNING_MACRO = -w 8 | CUFLAG = -std=c++14 -O3 $(DEBUG_MACRO) -Xcompiler -fPIC $(GENCODE) 9 | CCFLAG = -std=c++14 -O3 $(DEBUG_MACRO) -Xcompiler -fPIC -use_fast_math 10 | SOFLAG = -shared 11 | INCLUDE = -I. -I$(CUDA_PATH)/include -I$(TRT_INC_PATH) 12 | INCLUDE += -I../../include -I../../../include 13 | LDFLAG = -L$(CUDA_PATH)/lib64 -lcudart -L$(TRT_LIB_PATH) -lnvinfer 14 | -------------------------------------------------------------------------------- /cookbook/07-Tool/NsightSystems/Help-cancel.txt: -------------------------------------------------------------------------------- 1 | 2 | usage: nsys cancel [] 3 | 4 | -h, --help=[] 5 | 6 | Print the command's help menu. The switch can take one optional 7 | argument that will be used as a tag. If a tag is provided, only options 8 | relevant to the tag will be printed. 9 | The available help menu tags for this command are: 10 | 11 | interactive, and session. 12 | 13 | --session= 14 | 15 | Cancel the collection in the given session. The option argument must represent a 16 | valid session name or ID as reported by 'nsys sessions list'. 17 | Any '%q{ENV_VAR}' pattern in the option argument will be substituted with the value of the 18 | environment variable. 19 | Any '%h' pattern in the option argument will be substituted with the hostname of the system. 20 | Any '%%' pattern in the option argument will be substituted with '%'. 21 | -------------------------------------------------------------------------------- /cookbook/07-Tool/NsightSystems/Help-service.txt: -------------------------------------------------------------------------------- 1 | 2 | usage: nsys service [] 3 | 4 | -h, --help=[] 5 | 6 | Print the command's help menu. The switch can take one optional 7 | argument that will be used as a tag. If a tag is provided, only options 8 | relevant to the tag will be printed. 9 | -------------------------------------------------------------------------------- /cookbook/07-Tool/NsightSystems/Help-sessions-list.txt: -------------------------------------------------------------------------------- 1 | 2 | usage: nsys sessions list [] 3 | 4 | -h, --help=[] 5 | 6 | Print the command's help menu. The switch can take one optional 7 | argument that will be used as a tag. If a tag is provided, only options 8 | relevant to the tag will be printed. 9 | The available help menu tags for this command are: 10 | 11 | head, header, and list. 12 | 13 | -p, --show-header= [default: true] 14 | 15 | Controls whether a header should appear in the output. 16 | -------------------------------------------------------------------------------- /cookbook/07-Tool/NsightSystems/Help-stop.txt: -------------------------------------------------------------------------------- 1 | 2 | usage: nsys stop [] 3 | 4 | -h, --help=[] 5 | 6 | Print the command's help menu. The switch can take one optional 7 | argument that will be used as a tag. If a tag is provided, only options 8 | relevant to the tag will be printed. 9 | The available help menu tags for this command are: 10 | 11 | interactive, and session. 12 | 13 | --session= 14 | 15 | Stop the collection in the given session. The option argument must represent a 16 | valid session name or ID as reported by 'nsys sessions list'. 17 | Any '%q{ENV_VAR}' pattern in the option argument will be substituted with the value of the 18 | environment variable. 19 | Any '%h' pattern in the option argument will be substituted with the hostname of the system. 20 | Any '%%' pattern in the option argument will be substituted with '%'. 21 | -------------------------------------------------------------------------------- /cookbook/07-Tool/ONNX-TODO/README.md: -------------------------------------------------------------------------------- 1 | # Onnx 2 | 3 | + An open source format for AI models, both deep learning and traditional ML. 4 | 5 | + Repository [Link](https://github.com/onnx/onnx/tree/main) 6 | 7 | + Steps to run. 8 | 9 | ```bash 10 | ./command.sh 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/07-Tool/ONNX-TODO/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | if [ $TRT_COOKBOOK_CLEAN ]; then 23 | rm -rf *.log *.onnx *.weight 24 | fi 25 | 26 | echo "Finish `basename $(pwd)`" 27 | -------------------------------------------------------------------------------- /cookbook/07-Tool/OnnxGraphSurgeon/README.md: -------------------------------------------------------------------------------- 1 | # Onnx Graphsurgeon 2 | 3 | + A python library for ONNX compute graph edition, which different from the library *onnx*. 4 | 5 | + Installation: `pip install nvidia-pyindex onnx-graphsungeon` 6 | 7 | + Document [Link](https://docs.nvidia.com/deeplearning/tensorrt/onnx-graphsurgeon/docs/index.html) 8 | 9 | + The example code here refers to the NVIDIA official repository about TensorRT tools [Link](https://github.com/NVIDIA/TensorRT/tree/master/tools/onnx-graphsurgeon/examples). 10 | 11 | + Function: 12 | + Modify metadata/node / tensor / weight data of compute graph. 13 | + Modify subgraph: Add / Delete / Replace / Isolate 14 | + Optimize: constant folding / topological sorting / removing useless layers. 15 | 16 | + Steps to run. 17 | 18 | ```bash 19 | ./command.sh 20 | ``` 21 | -------------------------------------------------------------------------------- /cookbook/07-Tool/OnnxWeightProcess/README.md: -------------------------------------------------------------------------------- 1 | # Onnx Weight Separator 2 | 3 | + A tool to seperate weights from a ONNX file, usually for visualization of a remote large ONNX file. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Onnxruntime/README.md: -------------------------------------------------------------------------------- 1 | # Onnx Weight Separator 2 | 3 | + A tool to seperate weights from a ONNX file, usually for visualization of a remote large ONNX file. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/API/README.md: -------------------------------------------------------------------------------- 1 | # Polygraphy - API tool 2 | 3 | + Here are just some templates of using polygraphy APIs. 4 | 5 | + Steps to run (introduction is included in the script). 6 | 7 | ```bash 8 | ./main.sh 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/Check/README.md: -------------------------------------------------------------------------------- 1 | # Polygraphy - Check mode 2 | 3 | + Steps to run (introduction is included in the script). 4 | 5 | ```bash 6 | chmod +x ./main.sh 7 | main.sh 8 | ``` 9 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/Convert/README.md: -------------------------------------------------------------------------------- 1 | # Polygraphy - Convert mode 2 | 3 | + Steps to run (introduction is included in the script). 4 | 5 | ```bash 6 | ./main.sh 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/Data/README.md: -------------------------------------------------------------------------------- 1 | # Polygraphy - Data mode 2 | 3 | + Steps to run (introduction is included in the script). 4 | 5 | ```bash 6 | ./main.sh 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/Debug/README.md: -------------------------------------------------------------------------------- 1 | # Polygraphy - Debug mode 2 | 3 | + Steps to run (introduction is included in the script). 4 | 5 | ```bash 6 | ./main.sh 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/Inspect/.gitignore: -------------------------------------------------------------------------------- 1 | bad/ 2 | good/ 3 | polygraphy_capability_dumps/ 4 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/Inspect/README.md: -------------------------------------------------------------------------------- 1 | # Polygraphy - Inspect mode 2 | 3 | + Steps to run (introduction is included in the script). 4 | 5 | ```bash 6 | ./main.sh 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/Plugin-Unfini/README.md: -------------------------------------------------------------------------------- 1 | # Polygraphy - Plugin mode 2 | 3 | + Steps to run (introduction is included in the script). 4 | 5 | ```bash 6 | ./main.sh 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/README.md: -------------------------------------------------------------------------------- 1 | # Polygraphy - Client tool 2 | 3 | + CLI tool of polygraphy (deep learning model debugger). 4 | 5 | + Method of installation 6 | 7 | ```bash 8 | pip install polygraph\ 9 | ``` 10 | 11 | + Features: 12 | + Do inference computation using multiple backends, including TensorRT, onnxruntime, TensorFlow etc. 13 | + Compare results of computation layer by layer among different backends. 14 | + Generate TensorRT engine from model file and serialize it as .trt file. 15 | + Print the detailed information of model. 16 | + Modify ONNX model, such as extracting subgraph, simplifying computation graph. 17 | + Analyze the failure of parsing ONNX model into TensorRT, and save the subgrapha that can / cannot be converted to TensorRT. 18 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/Run/.gitignore: -------------------------------------------------------------------------------- 1 | polygraphy_run.py 2 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/Run/README.md: -------------------------------------------------------------------------------- 1 | # Polygraphy - Run mode 2 | 3 | + Steps to run (introduction is included in the script). 4 | 5 | ```bash 6 | ./main.sh 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/Surgeon/README.md: -------------------------------------------------------------------------------- 1 | # Polygraphy - Surgeon mode 2 | 3 | + Steps to run (introduction is included in the script). 4 | 5 | ```bash 6 | ./main.sh 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/Template/.gitignore: -------------------------------------------------------------------------------- 1 | modify_config.py 2 | modify_network.py 3 | modify_onnx.py 4 | -------------------------------------------------------------------------------- /cookbook/07-Tool/Polygraphy/Template/README.md: -------------------------------------------------------------------------------- 1 | # Polygraphy - Template mode 2 | 3 | + Steps to run (introduction is included in the script). 4 | 5 | ```bash 6 | ./main.sh 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/07-Tool/PolygraphyExtensionTrtexec/README.md: -------------------------------------------------------------------------------- 1 | # Polygraphy Extension Trtexec 2 | 3 | + [link](https://github.com/NVIDIA/TensorRT/tree/main/tools/polygraphy-extension-trtexec) 4 | -------------------------------------------------------------------------------- /cookbook/07-Tool/PolygraphyExtensionTrtexec/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/07-Tool/TRTEngineExplorer/README.md: -------------------------------------------------------------------------------- 1 | # TRT Engine Explorer 2 | 3 | + [link](https://github.com/NVIDIA/TensorRT/tree/main/tools/experimental/trt-engine-explorer) 4 | -------------------------------------------------------------------------------- /cookbook/07-Tool/TRTEngineExplorer/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/07-Tool/TriPy/README.md: -------------------------------------------------------------------------------- 1 | # TriPy 2 | 3 | + A Python programming model for TensorRT that aims to provide an excellent user experience without compromising performance. 4 | 5 | + GitHub [Link](https://github.com/NVIDIA/TensorRT-Incubator/tree/main/tripy) 6 | + [Document](https://nvidia.github.io/TensorRT-Incubator/pre0_user_guides/introduction-to-tripy.html) 7 | 8 | + Installation (according to GitHub) 9 | + Download packages from https://nvidia.github.io/TensorRT-Incubator/packages.html, we need `mlir_tensorrt_compiler`, `mlir_tensorrt_runtime` and `tripy`. 10 | + `pip install` all the packages. 11 | + According to my last test, tripy works well with these packages: 12 | + mlir_tensorrt_compiler-0.1.31+cuda12.trt102-cp310-cp310-linux_x86_64.whl 13 | + mlir_tensorrt_runtime-0.1.31+cuda12.trt102-cp310-cp310-linux_x86_64.whl 14 | + tripy-0.0.2-py3-none-any.whl 15 | -------------------------------------------------------------------------------- /cookbook/07-Tool/TriPy/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | python3 main.py > log-main.py.log 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | rm -rf *.log 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/07-Tool/build-README.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from tensorrt_cookbook import build_readme 17 | 18 | outline = \ 19 | """ 20 | + Tools of using TensorRT outside of the APIs. 21 | """ 22 | 23 | build_readme(__file__, outline) 24 | -------------------------------------------------------------------------------- /cookbook/07-Tool/nvtx/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | all: main.exe 4 | 5 | main.exe: main.o cookbookHelper.o 6 | $(COMMAND_EXE) 7 | 8 | main.o: main.cpp 9 | $(COMMAND_O) 10 | 11 | .PHONY: test 12 | test: 13 | make clean; make all -j; rm -rf *.trt 14 | ./main.exe > log-main.exe.log 15 | 16 | .PHONY: clean 17 | clean: 18 | rm -rf *.d *.o *.exe *.trt 19 | -------------------------------------------------------------------------------- /cookbook/07-Tool/nvtx/README.md: -------------------------------------------------------------------------------- 1 | # nvtx 2 | 3 | + Use NVIDIA®Tools Extension SDK to add mark in timeline of Nsight systems. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | nsys profile \ 9 | --force-overwrite=true \ 10 | -o py \ 11 | python3 main.py 12 | 13 | make 14 | nsys profile \ 15 | --force-overwrite=true \ 16 | -o cpp \ 17 | ./main.exe 18 | ``` 19 | -------------------------------------------------------------------------------- /cookbook/07-Tool/nvtx/main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/cookbook/07-Tool/nvtx/main.png -------------------------------------------------------------------------------- /cookbook/07-Tool/trtexec/README.md: -------------------------------------------------------------------------------- 1 | # trtexec 2 | 3 | + Command-line tool of TensorRT, attached with an end-to-end performance test tool. 4 | 5 | + Installation: attached with TensorRT, the executable program is /opt/bin/trtexec 6 | 7 | + Steps to run (introduction is included in the script). 8 | 9 | ```bash 10 | ./unit_test.sh 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/08-Advance/C++StaticCompilation/Makefile: -------------------------------------------------------------------------------- 1 | include ../../include/Makefile.inc 2 | 3 | MORE_LIB = -L$(CUDA_PATH)/lib64 -lcudart -lnvrtc -lnvptxcompiler_static -L$(TRT_LIB_PATH) -lnvinfer_static -lcudnn -L$(TRT_LIB_PATH)/stubs/ 4 | 5 | all: main.exe 6 | 7 | main.exe: main.o cookbookHelper.o 8 | $(COMMAND_EXE) $(MORE_LIB) 9 | 10 | main.o: main.cpp 11 | $(COMMAND_O) 12 | 13 | .PHONY: test 14 | test: 15 | make clean; make all -j; rm -rf *.trt 16 | ./main.exe > log-main.exe.log 17 | 18 | .PHONY: clean 19 | clean: 20 | rm -rf *.d *.exe *.o *.so *.exe *.trt 21 | -------------------------------------------------------------------------------- /cookbook/08-Advance/C++StaticCompilation/README.md: -------------------------------------------------------------------------------- 1 | # C++ Static Compilation 2 | 3 | + Static compilation the TensorRT engine into a executable file. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | make test 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/08-Advance/C++StaticCompilation/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | make test 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | make clean 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/08-Advance/CudaGraph/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | all: main.exe 4 | 5 | main.exe: main.o cookbookHelper.o 6 | $(COMMAND_EXE) 7 | 8 | main.o: main.cpp 9 | $(COMMAND_O) 10 | 11 | .PHONY: test 12 | test: 13 | make clean; make all -j; rm -rf *.trt 14 | ./main.exe > log-main.exe.log 15 | 16 | .PHONY: clean 17 | clean: 18 | rm -rf *.d *.o *.exe *.trt 19 | -------------------------------------------------------------------------------- /cookbook/08-Advance/CudaGraph/README.md: -------------------------------------------------------------------------------- 1 | # CUDA graph 2 | 3 | + Use CUDA graph to solve launch bound issue (usually appear in small TensorRT engines). 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | make test 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/08-Advance/CudaGraph/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | make test 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | make clean 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/08-Advance/EmptyTensor-Unfini/README.md: -------------------------------------------------------------------------------- 1 | # Empty Tensor 2 | 3 | + Use case of empty tensors. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/08-Advance/EmptyTensor-Unfini/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | echo "Finish `basename $(pwd)`" 23 | -------------------------------------------------------------------------------- /cookbook/08-Advance/MultiContext/README.md: -------------------------------------------------------------------------------- 1 | # Multi Context 2 | 3 | + Use multiple execution context to do inference. 4 | 5 | + Steps to run. 6 | 7 | ```shell 8 | python3 main.py 9 | ``` 10 | 11 | + This example has strong relationship with `../MultiOptimizationProfile`, those two features can be used at the same time. 12 | -------------------------------------------------------------------------------- /cookbook/08-Advance/MultiOptimizationProfile/README.md: -------------------------------------------------------------------------------- 1 | # Multi Optimization Profile 2 | 3 | + Use multiple Optimization-Profile to do inference. 4 | 5 | + Steps to run. 6 | 7 | ```shell 8 | python3 main.py 9 | ``` 10 | 11 | + This example has strong relationship with `../MultiContext`, those two features can be used at the same time. 12 | -------------------------------------------------------------------------------- /cookbook/08-Advance/MultiStream/README.md: -------------------------------------------------------------------------------- 1 | # Multi-Stream 2 | 3 | + Use one execution context with multiple CUDA stream. 4 | 5 | + Steps to run. 6 | 7 | ```bash 8 | python3 main.py 9 | ``` 10 | 11 | + For usage of Page-lock memory, refer to `08-Acvance/StreamAndAsync`. 12 | 13 | + Get timeline to observe 14 | 15 | ```bash 16 | nsys profile -o Multi-Stream -f true python3 main.py 17 | ``` 18 | -------------------------------------------------------------------------------- /cookbook/08-Advance/README.md: -------------------------------------------------------------------------------- 1 | # 08-Advance 2 | 3 | + Tool combinations of using TensorRT and other CUDA / pyTorch features. 4 | 5 | ## C++ Static Compilation 6 | 7 | + Static compilation the TensorRT engine into a executable file. 8 | 9 | ## CUDA graph 10 | 11 | + Use CUDA graph to solve launch bound issue (usually appear in small TensorRT engines). 12 | 13 | ## Use PluginV2DynamicExt 14 | 15 | + Data type and data format supported in TensorRT. 16 | 17 | ## Empty Tensor 18 | 19 | + Use case of empty tensors. 20 | 21 | ## Multi Context 22 | 23 | + Use multiple execution context to do inference. 24 | 25 | ## Multi Optimization Profile 26 | 27 | + Use multiple Optimization-Profile to do inference. 28 | 29 | ## Multi-Stream 30 | 31 | + Use one execution context with multiple CUDA stream. 32 | 33 | ## Stream and Async 34 | 35 | + Use customeized CUDA stream to do asynchronized inference. 36 | 37 | ## Subgraph 38 | 39 | + Use cases of parsing ONNX file with subgraph into TensorRT. 40 | -------------------------------------------------------------------------------- /cookbook/08-Advance/StreamAndAsync/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | all: main.exe 4 | 5 | main.exe: main.o cookbookHelper.o 6 | $(COMMAND_EXE) 7 | 8 | main.o: main.cpp 9 | $(COMMAND_O) 10 | 11 | .PHONY: test 12 | test: 13 | make clean; make all -j; rm -rf *.trt 14 | ./main.exe > log-main.exe.log 15 | 16 | .PHONY: clean 17 | clean: 18 | rm -rf *.d *.o *.exe *.trt 19 | -------------------------------------------------------------------------------- /cookbook/08-Advance/StreamAndAsync/README.md: -------------------------------------------------------------------------------- 1 | # Stream and Async 2 | 3 | + Use customeized CUDA stream to do asynchronized inference. 4 | 5 | + Steps to run. 6 | 7 | ```shell 8 | make test 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/08-Advance/StreamAndAsync/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | make test 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | make clean 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/08-Advance/Subgraph-Unfini/README.md: -------------------------------------------------------------------------------- 1 | # Subgraph 2 | 3 | + Use cases of parsing ONNX file with subgraph into TensorRT. 4 | 5 | + Steps to run. 6 | 7 | ```shell 8 | python3 main.py 9 | ``` 10 | -------------------------------------------------------------------------------- /cookbook/08-Advance/build-README.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from tensorrt_cookbook import build_readme 17 | 18 | outline = \ 19 | """ 20 | + Tool combinations of using TensorRT and other CUDA / pyTorch features. 21 | """ 22 | 23 | build_readme(__file__, outline) 24 | -------------------------------------------------------------------------------- /cookbook/09-TRTLLM/README.md: -------------------------------------------------------------------------------- 1 | # 09-TRTLLM 2 | 3 | + Tools related to TensorRT-LLM. 4 | -------------------------------------------------------------------------------- /cookbook/09-TRTLLM/build-README.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from tensorrt_cookbook import build_readme 17 | 18 | outline = \ 19 | """ 20 | + Tools related to TensorRT-LLM. 21 | """ 22 | 23 | build_readme(__file__, outline) 24 | -------------------------------------------------------------------------------- /cookbook/98-Uncategorized/DeviceInfo/README.md: -------------------------------------------------------------------------------- 1 | # get_device_info.py 2 | 3 | + Get all properties of the GPU devices. 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/98-Uncategorized/DeviceInfo/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | #clear 6 | 7 | python3 main.py > log-main.py.log 8 | 9 | if [ ! $TRT_COOKBOOK_CLEAN_AFTER_UNIT_TEST ]; then 10 | rm -rf *.log 11 | fi 12 | 13 | echo "Finish `basename $(pwd)`" 14 | -------------------------------------------------------------------------------- /cookbook/98-Uncategorized/LibraryInfo/README.md: -------------------------------------------------------------------------------- 1 | # get_device_info.py 2 | 3 | + Get software version of Driver, CUDA, cuDNN, cuBLAS, TensorRT, TensorRT(python), pyTorch, TensorFlow, 4 | 5 | ```bash 6 | python3 main.py 7 | ``` 8 | -------------------------------------------------------------------------------- /cookbook/98-Uncategorized/LibraryInfo/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | #clear 6 | 7 | python3 main.py > log-main.py.log 8 | 9 | if [ ! $TRT_COOKBOOK_CLEAN_AFTER_UNIT_TEST ]; then 10 | rm -rf *.log 11 | fi 12 | 13 | echo "Finish `basename $(pwd)`" 14 | -------------------------------------------------------------------------------- /cookbook/98-Uncategorized/NpyAndNpz/Makefile: -------------------------------------------------------------------------------- 1 | include $(TRT_COOKBOOK_PATH)/include/Makefile.inc 2 | 3 | all: main.exe 4 | 5 | main.exe: main.o cookbookHelper.o 6 | $(COMMAND_EXE) -lz 7 | 8 | main.o: main.cpp 9 | $(COMMAND_O) 10 | 11 | .PHONY: test 12 | test: 13 | python3 get_data.py 14 | make all -j 15 | ./main.exe > log-main.exe.log 16 | 17 | .PHONY: clean 18 | clean: 19 | rm -rf *.d *.o *.exe *.log *.npy *.npz 20 | -------------------------------------------------------------------------------- /cookbook/98-Uncategorized/NpyAndNpz/README.md: -------------------------------------------------------------------------------- 1 | # MpyAndNpz 2 | 3 | + Read and write Npy / Npz file in C++. 4 | 5 | + Source [link](https://github.com/rogersce/cnpy). 6 | 7 | + Steps to run 8 | 9 | ```shell 10 | make test 11 | ``` 12 | -------------------------------------------------------------------------------- /cookbook/98-Uncategorized/NpyAndNpz/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | set -x 20 | #clear 21 | 22 | make test 23 | 24 | if [ $TRT_COOKBOOK_CLEAN ]; then 25 | make clean 26 | fi 27 | 28 | echo "Finish `basename $(pwd)`" 29 | -------------------------------------------------------------------------------- /cookbook/98-Uncategorized/Number/README.md: -------------------------------------------------------------------------------- 1 | # Number 2 | 3 | + Get the range and layout information of floating data type. 4 | 5 | + Steps to run 6 | 7 | ```bash 8 | python3 build_number_md.py 9 | python3 build_number_picture.py 10 | ``` 11 | 12 | + Some output files of typical data types are listed in `output/`. 13 | -------------------------------------------------------------------------------- /cookbook/98-Uncategorized/Number/output/DataType.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA/trt-samples-for-hackathon-cn/d37952a30d083b02921a201b40045ca7ed00e221/cookbook/98-Uncategorized/Number/output/DataType.png -------------------------------------------------------------------------------- /cookbook/98-Uncategorized/Number/unit_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -x 5 | #clear 6 | 7 | python3 build_number_md.py > log-build_number_md.py.log 8 | python3 build_number_picture.py > log-build_number_picture.py.log 9 | 10 | if [ ! $TRT_COOKBOOK_CLEAN_AFTER_UNIT_TEST ]; then 11 | rm -rf *.log 12 | fi 13 | 14 | echo "Finish `basename $(pwd)`" 15 | -------------------------------------------------------------------------------- /cookbook/98-Uncategorized/README.md: -------------------------------------------------------------------------------- 1 | # 98-Uncategorized 2 | 3 | + Common tools, which are not limited to TensorRT. 4 | 5 | ## get_device_info.py 6 | 7 | + Get all properties of the GPU devices. 8 | 9 | ## get_device_info.py 10 | 11 | + Get software version of Driver, CUDA, cuDNN, cuBLAS, TensorRT, TensorRT(python), pyTorch, TensorFlow. 12 | 13 | ## MpyAndNpz 14 | 15 | + Read and write Npy / Npz file in C++. 16 | 17 | ## Number 18 | 19 | + Get the range and layout information of floating data type. 20 | -------------------------------------------------------------------------------- /cookbook/98-Uncategorized/build-README.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from tensorrt_cookbook import build_readme 17 | 18 | outline = \ 19 | """ 20 | + Common tools, which are not limited to TensorRT. 21 | """ 22 | 23 | build_readme(__file__, outline) 24 | -------------------------------------------------------------------------------- /cookbook/99-Todo/README.md: -------------------------------------------------------------------------------- 1 | # TODO 2 | 3 | + To-Do list for the cookbook. 4 | 5 | + 05-Plugin/INT8-QDQ-Plugin 6 | + EinsumLayer implicit mode 7 | + IGpuAsyncAllocator 8 | + Nonzero Layer + Gather Layer 9 | + Random seed appearance of Fill layer 10 | + TensorRT LWS? 11 | + TF2 API examples: TF-TRT, OnnxWorkflowWithPlugin 12 | + Unify 05-Plugin/PluginRepository/ 13 | + APILanguage in 05-Plugin/APIs (C++ / Python) 14 | + Comment of Resize layer 15 | + Algorithm Selector serialize / deserialize to get deterministic engine 16 | + Better Network printer 17 | 18 | + trt.weights (https://docs.nvidia.com/deeplearning/tensorrt/api/python_api/infer/FoundationalTypes/Weights.html) 19 | + trt.Dims (https://docs.nvidia.com/deeplearning/tensorrt/api/python_api/infer/FoundationalTypes/Dims.html) 20 | 21 | + set_calibration_profile / get_calibration_profile in BuilderConfig 22 | -------------------------------------------------------------------------------- /cookbook/requirement-dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | tf2onnx # tool of exporting model in TensorFlow (.pb) into ONNX file 3 | paddlepaddle # Paddlepaddle 4 | paddle2onnx # tool of exporting model in Paddlepaddle into ONNX file 5 | pytorch-quantization==2.1.2 --extra-index-url https://pypi.nvidia.com # problems for pytorch-quantization>2.1.2 + tensorrt>8.6 6 | 7 | # TREX related 8 | #dtale==2.2.0 9 | #graphviz 10 | #ipyfilechooser 11 | #ipywidgets 12 | #jupyter 13 | #jupyter-dash 14 | #jupyterlab 15 | #netron 16 | #openpyxl # for excel reporting 17 | #pandas==1.5 18 | #plotly 19 | #protobuf==3.20.3 20 | #qgrid 21 | #pytest 22 | #setuptools # for qgrid 23 | #wheel # for qgrid 24 | -------------------------------------------------------------------------------- /cookbook/tensorrt_cookbook/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import sys 17 | 18 | from .utils import * # isort:disable 19 | from .version import __version__ 20 | 21 | print(f"[TensorRT-Cookbook] Version: {__version__}") 22 | 23 | sys.stdout.flush() 24 | -------------------------------------------------------------------------------- /cookbook/tensorrt_cookbook/version.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # SPDX-License-Identifier: Apache-2.0 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | __version__ = "0.1.0" 17 | -------------------------------------------------------------------------------- /old/Makefile: -------------------------------------------------------------------------------- 1 | SHELL=/bin/bash 2 | 3 | all: 4 | cd build; make 5 | 6 | clean: 7 | cd build; make clean 8 | --------------------------------------------------------------------------------