├── .clang-format ├── .gitattributes ├── .github └── workflows │ ├── codeql.yml │ ├── pre-commit.yml │ └── python-package.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── VERSION ├── build_wheel.sh ├── docs ├── README.md ├── bls_quick_start.md ├── checkpoints.md ├── cli.md ├── config.md ├── config_search.md ├── ensemble_quick_start.md ├── install.md ├── kubernetes_deploy.md ├── launch_modes.md ├── metrics.md ├── mm_quick_start.md ├── model_types.md ├── quick_start.md └── report.md ├── examples ├── bls_input_data.json ├── bls_result_summary_table.jpg ├── bls_result_summary_top.jpg ├── ensemble_result_summary_table.jpg ├── ensemble_result_summary_top.jpg ├── mm_result_summary_constraint_table.jpg ├── mm_result_summary_constraint_top.jpg ├── mm_result_summary_table.jpg ├── mm_result_summary_top.jpg ├── offline_detailed_report.pdf ├── offline_summary.pdf ├── online_detailed_report.pdf ├── online_summary.pdf └── quick-start │ ├── add │ ├── 1 │ │ └── model.py │ └── config.pbtxt │ ├── add_sub │ ├── 1 │ │ └── model.pt │ ├── config.pbtxt │ └── output0_labels.txt │ ├── bls │ ├── 1 │ │ └── model.py │ ├── bls_input_data.json │ └── config.pbtxt │ ├── ensemble_add_sub │ └── config.pbtxt │ ├── resnet50_python │ ├── 1 │ │ └── model.py │ └── config.pbtxt │ └── sub │ ├── 1 │ └── model.py │ └── config.pbtxt ├── experiments ├── checkpoint_experiment_data.py ├── config_command_experiment.py ├── data │ ├── resnet50_libtorch │ │ ├── 0.ckpt │ │ └── config.pbtxt │ └── vgg19_libtorch │ │ ├── 0.ckpt │ │ └── config.pbtxt ├── evaluate_config_generator.py ├── experiment_config_command_creator.py ├── experiment_data.py ├── experiment_evaluator.py ├── experiment_file_writer.py ├── experiment_plot.py ├── generator_experiment_factory.py ├── main.py ├── scripts │ ├── minimize_latency.yml │ ├── sweep_checkpoints.py │ └── test_pa.py └── start_profiles.sh ├── format.py ├── helm-chart ├── .helmignore ├── Chart.yaml ├── templates │ ├── _helpers.tpl │ ├── config-map.yaml │ └── job.yaml └── values.yaml ├── model_analyzer ├── __init__.py ├── analyzer.py ├── cli │ ├── __init__.py │ └── cli.py ├── config │ ├── __init__.py │ ├── generate │ │ ├── __init__.py │ │ ├── automatic_model_config_generator.py │ │ ├── base_model_config_generator.py │ │ ├── brute_plus_binary_parameter_search_run_config_generator.py │ │ ├── brute_run_config_generator.py │ │ ├── concurrency_sweeper.py │ │ ├── config_generator_interface.py │ │ ├── coordinate.py │ │ ├── coordinate_data.py │ │ ├── generator_utils.py │ │ ├── manual_model_config_generator.py │ │ ├── model_config_generator_factory.py │ │ ├── model_profile_spec.py │ │ ├── model_run_config_generator.py │ │ ├── model_variant_name_manager.py │ │ ├── neighborhood.py │ │ ├── optuna_plus_concurrency_sweep_run_config_generator.py │ │ ├── optuna_run_config_generator.py │ │ ├── perf_analyzer_config_generator.py │ │ ├── quick_plus_concurrency_sweep_run_config_generator.py │ │ ├── quick_run_config_generator.py │ │ ├── run_config_generator_factory.py │ │ ├── search_config.py │ │ ├── search_dimension.py │ │ ├── search_dimensions.py │ │ ├── search_parameter.py │ │ └── search_parameters.py │ ├── input │ │ ├── __init__.py │ │ ├── config_command.py │ │ ├── config_command_profile.py │ │ ├── config_command_report.py │ │ ├── config_defaults.py │ │ ├── config_enum.py │ │ ├── config_field.py │ │ ├── config_list_generic.py │ │ ├── config_list_numeric.py │ │ ├── config_list_string.py │ │ ├── config_none.py │ │ ├── config_object.py │ │ ├── config_primitive.py │ │ ├── config_status.py │ │ ├── config_sweep.py │ │ ├── config_union.py │ │ ├── config_utils.py │ │ ├── config_value.py │ │ ├── objects │ │ │ ├── __init__.py │ │ │ ├── config_model_profile_spec.py │ │ │ ├── config_model_report_spec.py │ │ │ ├── config_plot.py │ │ │ └── config_protobuf_utils.py │ │ └── yaml_config_validator.py │ └── run │ │ ├── __init__.py │ │ ├── model_run_config.py │ │ └── run_config.py ├── constants.py ├── device │ ├── __init__.py │ ├── device.py │ ├── gpu_device.py │ └── gpu_device_factory.py ├── entrypoint.py ├── log_formatter.py ├── model_analyzer_exceptions.py ├── model_manager.py ├── monitor │ ├── __init__.py │ ├── cpu_monitor.py │ ├── dcgm │ │ ├── DcgmDiag.py │ │ ├── DcgmFieldGroup.py │ │ ├── DcgmGroup.py │ │ ├── DcgmHandle.py │ │ ├── DcgmJsonReader.py │ │ ├── DcgmReader.py │ │ ├── DcgmStatus.py │ │ ├── DcgmSystem.py │ │ ├── __init__.py │ │ ├── common │ │ │ ├── __init__.py │ │ │ ├── dcgm_client_cli_parser.py │ │ │ └── dcgm_client_main.py │ │ ├── dcgm_agent.py │ │ ├── dcgm_collectd_plugin.py │ │ ├── dcgm_errors.py │ │ ├── dcgm_field_helpers.py │ │ ├── dcgm_fields.py │ │ ├── dcgm_fields_collectd.py │ │ ├── dcgm_fields_internal.py │ │ ├── dcgm_fluentd.py │ │ ├── dcgm_monitor.py │ │ ├── dcgm_prometheus.py │ │ ├── dcgm_structs.py │ │ ├── dcgm_telegraf.py │ │ ├── dcgm_value.py │ │ ├── dcgmvalue.py │ │ ├── denylist_recommendations.py │ │ └── pydcgm.py │ ├── monitor.py │ └── remote_monitor.py ├── output │ ├── __init__.py │ ├── file_writer.py │ └── output_writer.py ├── perf_analyzer │ ├── __init__.py │ ├── genai_perf_config.py │ ├── perf_analyzer.py │ └── perf_config.py ├── plots │ ├── __init__.py │ ├── detailed_plot.py │ ├── plot_manager.py │ └── simple_plot.py ├── record │ ├── __init__.py │ ├── gpu_record.py │ ├── metrics_manager.py │ ├── record.py │ ├── record_aggregator.py │ └── types │ │ ├── __init__.py │ │ ├── cpu_available_ram.py │ │ ├── cpu_used_ram.py │ │ ├── gpu_free_memory.py │ │ ├── gpu_power_usage.py │ │ ├── gpu_total_memory.py │ │ ├── gpu_used_memory.py │ │ ├── gpu_utilization.py │ │ ├── inter_token_latency_avg.py │ │ ├── inter_token_latency_base.py │ │ ├── inter_token_latency_max.py │ │ ├── inter_token_latency_min.py │ │ ├── inter_token_latency_p25.py │ │ ├── inter_token_latency_p50.py │ │ ├── inter_token_latency_p75.py │ │ ├── inter_token_latency_p90.py │ │ ├── inter_token_latency_p95.py │ │ ├── inter_token_latency_p99.py │ │ ├── output_token_throughput.py │ │ ├── perf_client_response_wait.py │ │ ├── perf_client_send_recv.py │ │ ├── perf_latency.py │ │ ├── perf_latency_avg.py │ │ ├── perf_latency_base.py │ │ ├── perf_latency_p90.py │ │ ├── perf_latency_p95.py │ │ ├── perf_latency_p99.py │ │ ├── perf_server_compute_infer.py │ │ ├── perf_server_compute_input.py │ │ ├── perf_server_compute_output.py │ │ ├── perf_server_queue.py │ │ ├── perf_throughput.py │ │ ├── time_to_first_token_avg.py │ │ ├── time_to_first_token_base.py │ │ ├── time_to_first_token_max.py │ │ ├── time_to_first_token_min.py │ │ ├── time_to_first_token_p25.py │ │ ├── time_to_first_token_p50.py │ │ ├── time_to_first_token_p75.py │ │ ├── time_to_first_token_p90.py │ │ ├── time_to_first_token_p95.py │ │ └── time_to_first_token_p99.py ├── reports │ ├── __init__.py │ ├── html_report.py │ ├── pdf_report.py │ ├── report.py │ ├── report_factory.py │ ├── report_manager.py │ └── report_utils.py ├── result │ ├── __init__.py │ ├── constraint_manager.py │ ├── model_config_measurement.py │ ├── model_constraints.py │ ├── parameter_search.py │ ├── result_manager.py │ ├── result_statistics.py │ ├── result_table.py │ ├── result_table_manager.py │ ├── result_utils.py │ ├── results.py │ ├── run_config_measurement.py │ ├── run_config_result.py │ ├── run_config_result_comparator.py │ └── sorted_results.py ├── state │ ├── __init__.py │ ├── analyzer_state.py │ └── analyzer_state_manager.py └── triton │ ├── __init__.py │ ├── client │ ├── __init__.py │ ├── client.py │ ├── client_factory.py │ ├── grpc_client.py │ └── http_client.py │ ├── model │ ├── __init__.py │ ├── model_config.py │ └── model_config_variant.py │ └── server │ ├── __init__.py │ ├── server.py │ ├── server_config.py │ ├── server_docker.py │ ├── server_factory.py │ └── server_local.py ├── nvidia_entrypoint.sh ├── pyproject.toml ├── qa ├── L0_bls_model │ ├── check_results.py │ ├── input_data.json │ ├── test.sh │ └── test_config_generator.py ├── L0_config │ ├── config_generator.py │ └── test.sh ├── L0_config_range │ ├── config_generator.py │ └── test.sh ├── L0_config_search │ ├── test.sh │ └── test_config_generator.py ├── L0_cpu │ ├── test.sh │ └── test_config_generator.py ├── L0_custom_flags │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_custom_ops │ ├── test.sh │ └── test_config_generator.py ├── L0_doa │ └── test.sh ├── L0_doc_links │ ├── mkdocs.yml │ └── test.sh ├── L0_ensemble_model │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_metrics │ ├── test.sh │ └── test_config_generator.py ├── L0_multi_model_profile │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_no_config_file │ ├── test.sh │ └── test_config_generator.py ├── L0_optuna_bls_model │ ├── check_results.py │ ├── input_data.json │ ├── test.sh │ └── test_config_generator.py ├── L0_optuna_ensemble_model │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_optuna_multi_model │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_optuna_search │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_output_fields │ ├── config_generator.py │ └── test.sh ├── L0_perf_analyzer │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_profile │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_profile_quick │ └── test.sh ├── L0_profile_request_rate │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_quick_search │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_quick_search_multi_model │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_quick_start │ └── test.sh ├── L0_results │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_server_launch_modes │ ├── check_gpus.py │ ├── test.sh │ └── test_config_generator.py ├── L0_ssl_grpc │ ├── broken_config.yml │ ├── test.sh │ └── working_config.yml ├── L0_ssl_https │ ├── broken_config.yml │ ├── nginx.conf │ ├── test.sh │ └── working_config.yml ├── L0_stability_metrics │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_stability_perf │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_stability_result │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_stability_steps │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_state_management │ ├── check_results.py │ ├── test.sh │ └── test_config_generator.py ├── L0_unit_tests │ ├── count_tests.py │ └── test.sh └── common │ ├── check_analyzer_results.sh │ ├── check_copyright.py │ └── util.sh ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── common ├── bls-ckpt │ ├── 0.ckpt │ ├── README │ ├── golden-metrics-model-gpu.csv │ ├── golden-metrics-model-inference.csv │ └── golden-metrics-server-only.csv ├── ensemble-ckpt │ ├── 0.ckpt │ ├── README │ ├── golden-metrics-model-gpu.csv │ ├── golden-metrics-model-inference.csv │ └── golden-metrics-server-only.csv ├── multi-model-ckpt │ ├── 0.ckpt │ ├── README │ ├── golden-metrics-model-gpu.csv │ ├── golden-metrics-model-inference.csv │ ├── golden-metrics-server-only.csv │ └── plot_manager.json ├── request-rate-ckpt │ ├── 0.ckpt │ ├── README │ ├── golden-metrics-model-gpu.csv │ ├── golden-metrics-model-inference.csv │ └── golden-metrics-server-only.csv ├── single-model-ckpt │ ├── 0.ckpt │ ├── README │ ├── golden-metrics-model-gpu.csv │ ├── golden-metrics-model-inference.csv │ ├── golden-metrics-server-only.csv │ └── plot_manager.json ├── test_result_collector.py └── test_utils.py ├── mocks ├── __init__.py ├── mock_api_error.py ├── mock_base.py ├── mock_client.py ├── mock_config.py ├── mock_dcgm.py ├── mock_dcgm_agent.py ├── mock_dcgm_field_group_watcher.py ├── mock_glob.py ├── mock_gpu_device_factory.py ├── mock_io.py ├── mock_json.py ├── mock_matplotlib.py ├── mock_model_config.py ├── mock_numba.py ├── mock_os.py ├── mock_perf_analyzer.py ├── mock_psutil.py ├── mock_requests.py ├── mock_run_configs.py ├── mock_server.py ├── mock_server_docker.py └── mock_server_local.py ├── test_analyzer.py ├── test_bls_report_manager.py ├── test_cli.py ├── test_config.py ├── test_constraint_manager.py ├── test_coordinate.py ├── test_coordinate_data.py ├── test_cpu_monitor.py ├── test_dcgm_monitor.py ├── test_ensemble_report_manager.py ├── test_file_writer.py ├── test_html_report.py ├── test_logger.py ├── test_model_config.py ├── test_model_config_generator.py ├── test_model_config_measurement.py ├── test_model_constraints.py ├── test_model_manager.py ├── test_model_variant_name_manager.py ├── test_neighborhood.py ├── test_optuna_run_config_generator.py ├── test_parameter_search.py ├── test_pdf_report.py ├── test_perf_analyzer.py ├── test_perf_analyzer_config_generator.py ├── test_plot.py ├── test_plot_manager.py ├── test_quick_run_config_generator.py ├── test_record_aggregator.py ├── test_record_types.py ├── test_remote_monitor.py ├── test_report_manager.py ├── test_report_utils.py ├── test_result_manager.py ├── test_result_table.py ├── test_result_table_manager.py ├── test_results.py ├── test_run_config.py ├── test_run_config_generator.py ├── test_run_config_measurement.py ├── test_run_config_result.py ├── test_run_config_result_comparator.py ├── test_search_config.py ├── test_search_dimension.py ├── test_search_dimensions.py ├── test_search_parameters.py ├── test_sorted_results.py ├── test_state_manager.py ├── test_triton_client.py ├── test_triton_server.py ├── test_triton_server_factory.py └── test_yaml_options.py /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | BasedOnStyle: Google 3 | 4 | IndentWidth: 2 5 | ColumnLimit: 80 6 | ContinuationIndentWidth: 4 7 | UseTab: Never 8 | MaxEmptyLinesToKeep: 2 9 | 10 | SortIncludes: true 11 | CompactNamespaces: true 12 | ReflowComments: true 13 | 14 | DerivePointerAlignment: false 15 | PointerAlignment: Left 16 | 17 | AllowShortIfStatementsOnASingleLine: false 18 | AllowShortBlocksOnASingleLine: false 19 | AllowShortFunctionsOnASingleLine: Inline 20 | 21 | AlwaysBreakAfterReturnType: TopLevelDefinitions 22 | AlignAfterOpenBracket: AlwaysBreak 23 | BreakBeforeBraces: Custom 24 | BraceWrapping: 25 | AfterClass: false 26 | AfterControlStatement: false 27 | AfterEnum: false 28 | AfterFunction: true 29 | AfterNamespace: false 30 | AfterStruct: false 31 | AfterUnion: false 32 | BeforeCatch: true 33 | 34 | BinPackArguments: true 35 | BinPackParameters: true 36 | ConstructorInitializerAllOnOneLineOrOnePerLine: false 37 | 38 | IndentCaseLabels: true 39 | 40 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # * Redistributions of source code must retain the above copyright 7 | # notice, this list of conditions and the following disclaimer. 8 | # * Redistributions in binary form must reproduce the above copyright 9 | # notice, this list of conditions and the following disclaimer in the 10 | # documentation and/or other materials provided with the distribution. 11 | # * Neither the name of NVIDIA CORPORATION nor the names of its 12 | # contributors may be used to endorse or promote products derived 13 | # from this software without specific prior written permission. 14 | # 15 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY 16 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 18 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 19 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 20 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 22 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 23 | # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | name: pre-commit 28 | 29 | on: 30 | pull_request: 31 | 32 | jobs: 33 | pre-commit: 34 | runs-on: ubuntu-22.04 35 | steps: 36 | - uses: actions/checkout@v3 37 | - uses: actions/setup-python@v3 38 | - uses: pre-commit/action@v3.0.0 39 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # Contributing 18 | 19 | Triton Memory Analyzer follows similar contributing guidelines to Triton Inference Server. Please see [the Triton Inference Server contributing document]. 20 | 21 | [the Triton Inference Server contributing document]: https://github.com/triton-inference-server/server/blob/master/CONTRIBUTING.md 22 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 1.48.0dev 2 | -------------------------------------------------------------------------------- /examples/bls_input_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": [ 3 | { 4 | "MODEL_NAME": [ 5 | "add" 6 | ], 7 | "INPUT0": [ 8 | 0.74106514, 9 | 0.7371813, 10 | 0.5274665, 11 | 0.13930903 12 | ], 13 | "INPUT1": [ 14 | 0.7845891, 15 | 0.88089234, 16 | 0.8466405, 17 | 0.55024815 18 | ] 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /examples/bls_result_summary_table.jpg: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:ca9c4635772a8cbd1c83164689f48fd128bff0648b1e63b4f22369a9909587c2 3 | size 123725 4 | -------------------------------------------------------------------------------- /examples/bls_result_summary_top.jpg: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:98e305516a1ecc4983d313ebcfd409c13a934cb0aa39eac93b08b775e8748903 3 | size 128361 4 | -------------------------------------------------------------------------------- /examples/ensemble_result_summary_table.jpg: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5452c38e12c5cc13e5b7ab43e598ca24b2801afe742be9941eb0d0c3705b30c8 3 | size 167500 4 | -------------------------------------------------------------------------------- /examples/ensemble_result_summary_top.jpg: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:8dcc7335ec26639d644be6d831d8850d89d30bbb16e3e21cf5be57dcaf81efae 3 | size 137205 4 | -------------------------------------------------------------------------------- /examples/mm_result_summary_constraint_table.jpg: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:8cb1716475329b59c72123b54542ff9fed79430e56af44cd373bad7c581d766f 3 | size 85162 4 | -------------------------------------------------------------------------------- /examples/mm_result_summary_constraint_top.jpg: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:712324c98d4e6431fe598612e31ac0b5218760490552905d513bb3702953e3bc 3 | size 56469 4 | -------------------------------------------------------------------------------- /examples/mm_result_summary_table.jpg: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:063b6e7580c47e59c93b75271b4753651cb881603d91852ea4ed0703491c380a 3 | size 88070 4 | -------------------------------------------------------------------------------- /examples/mm_result_summary_top.jpg: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:8c695721b7decbc7b0679e05254d2e3f9c1fb13acc9e686154fc850423582b9d 3 | size 56439 4 | -------------------------------------------------------------------------------- /examples/offline_detailed_report.pdf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:f74f7b335601d6c4a62b35df8e235abb400db4caebf98743bbb1a7f92b06fcb7 3 | size 107902 4 | -------------------------------------------------------------------------------- /examples/offline_summary.pdf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:c7c6f0a4cca1a3b1905dd5a1aaf5e868197f4d4aab88df5a92699bb0714d77eb 3 | size 78536 4 | -------------------------------------------------------------------------------- /examples/online_detailed_report.pdf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:bda6b7436b1412c9099868e8be000c0f7f18018e89b21179add0f1736290cc2d 3 | size 303537 4 | -------------------------------------------------------------------------------- /examples/online_summary.pdf: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:3c4f8f4798940dddb1fc2f49c2be945dd005f061624a2730ffacb466b64c09d4 3 | size 159628 4 | -------------------------------------------------------------------------------- /examples/quick-start/add/config.pbtxt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | name: "add" 17 | backend: "python" 18 | 19 | input [ 20 | { 21 | name: "INPUT0" 22 | data_type: TYPE_FP32 23 | dims: [ 4 ] 24 | } 25 | ] 26 | input [ 27 | { 28 | name: "INPUT1" 29 | data_type: TYPE_FP32 30 | dims: [ 4 ] 31 | } 32 | ] 33 | output [ 34 | { 35 | name: "OUTPUT" 36 | data_type: TYPE_FP32 37 | dims: [ 4 ] 38 | } 39 | ] 40 | 41 | instance_group [{ kind: KIND_CPU }] 42 | -------------------------------------------------------------------------------- /examples/quick-start/add_sub/1/model.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/triton-inference-server/model_analyzer/5e3746f738b56118b31f28d9472db04f7361aaf8/examples/quick-start/add_sub/1/model.pt -------------------------------------------------------------------------------- /examples/quick-start/add_sub/config.pbtxt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: "add_sub" 16 | platform: "pytorch_libtorch" 17 | max_batch_size: 8 18 | 19 | input [ 20 | { 21 | name: "INPUT__0" 22 | data_type: TYPE_FP32 23 | dims: [ 16 ] 24 | }, 25 | { 26 | name: "INPUT__1" 27 | data_type: TYPE_FP32 28 | dims: [ 16 ] 29 | } 30 | ] 31 | output [ 32 | { 33 | name: "OUTPUT__0" 34 | data_type: TYPE_FP32 35 | dims: [ 16 ] 36 | label_filename: "output0_labels.txt" 37 | }, 38 | { 39 | name: "OUTPUT__1" 40 | data_type: TYPE_FP32 41 | dims: [ 16 ] 42 | } 43 | ] 44 | instance_group [ { kind: KIND_GPU }] 45 | -------------------------------------------------------------------------------- /examples/quick-start/add_sub/output0_labels.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | label0 16 | label1 17 | label2 18 | label3 19 | label4 20 | label5 21 | label6 22 | label7 23 | label8 24 | label9 25 | label10 26 | label11 27 | label12 28 | label13 29 | label14 30 | label15 31 | -------------------------------------------------------------------------------- /examples/quick-start/bls/bls_input_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": [ 3 | { 4 | "MODEL_NAME": [ 5 | "add" 6 | ], 7 | "INPUT0": [ 8 | 0.74106514, 9 | 0.7371813, 10 | 0.5274665, 11 | 0.13930903 12 | ], 13 | "INPUT1": [ 14 | 0.7845891, 15 | 0.88089234, 16 | 0.8466405, 17 | 0.55024815 18 | ] 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /examples/quick-start/bls/config.pbtxt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: "bls" 16 | backend: "python" 17 | 18 | input [ 19 | { 20 | name: "MODEL_NAME" 21 | data_type: TYPE_STRING 22 | dims: [ 1 ] 23 | }, 24 | { 25 | name: "INPUT0" 26 | data_type: TYPE_FP32 27 | dims: [ 4 ] 28 | }, 29 | { 30 | name: "INPUT1" 31 | data_type: TYPE_FP32 32 | dims: [ 4 ] 33 | } 34 | ] 35 | 36 | output [ 37 | { 38 | name: "OUTPUT" 39 | data_type: TYPE_FP32 40 | dims: [ 4 ] 41 | } 42 | ] 43 | 44 | instance_group [ { kind: KIND_CPU }] 45 | -------------------------------------------------------------------------------- /examples/quick-start/ensemble_add_sub/config.pbtxt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | name: "ensemble_add_sub" 17 | platform: "ensemble" 18 | 19 | input [ 20 | { 21 | name: "INPUT0" 22 | data_type: TYPE_FP32 23 | dims: [ 4 ] 24 | }, 25 | { 26 | name: "INPUT1" 27 | data_type: TYPE_FP32 28 | dims: [ 4 ] 29 | } 30 | ] 31 | 32 | output [ 33 | { 34 | name: "OUTPUT0" 35 | data_type: TYPE_FP32 36 | dims: [ 4 ] 37 | }, 38 | { 39 | name: "OUTPUT1" 40 | data_type: TYPE_FP32 41 | dims: [ 4 ] 42 | } 43 | ] 44 | 45 | ensemble_scheduling { 46 | step [ 47 | { 48 | model_name: "add" 49 | model_version: 1 50 | input_map { 51 | key: "INPUT0" 52 | value: "INPUT0" 53 | } 54 | input_map { 55 | key: "INPUT1" 56 | value: "INPUT1" 57 | } 58 | output_map { 59 | key: "OUTPUT" 60 | value: "OUTPUT0" 61 | } 62 | }, 63 | { 64 | model_name: "sub" 65 | model_version: 1 66 | input_map { 67 | key: "INPUT0" 68 | value: "INPUT0" 69 | } 70 | input_map { 71 | key: "INPUT1" 72 | value: "INPUT1" 73 | } 74 | output_map { 75 | key: "OUTPUT" 76 | value: "OUTPUT1" 77 | } 78 | } 79 | ] 80 | } 81 | -------------------------------------------------------------------------------- /examples/quick-start/resnet50_python/1/model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | import torch 18 | import triton_python_backend_utils as pb_utils 19 | from torch.utils.dlpack import to_dlpack 20 | 21 | 22 | class TritonPythonModel: 23 | def initialize(self, args): 24 | """ 25 | This function initializes pre-trained ResNet50 model. 26 | """ 27 | self.device = "cuda" if args["model_instance_kind"] == "GPU" else "cpu" 28 | self.model = ( 29 | torch.hub.load("pytorch/vision", "resnet50", weights="IMAGENET1K_V2") 30 | .to(self.device) 31 | .eval() 32 | ) 33 | 34 | def execute(self, requests): 35 | """ 36 | This function receives a list of requests (`pb_utils.InferenceRequest`), 37 | performs inference on every request and appends it to responses. 38 | """ 39 | responses = [] 40 | for request in requests: 41 | input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") 42 | result = self.model( 43 | torch.as_tensor(input_tensor.as_numpy(), device=self.device) 44 | ) 45 | out_tensor = pb_utils.Tensor.from_dlpack("OUTPUT0", to_dlpack(result)) 46 | responses.append(pb_utils.InferenceResponse([out_tensor])) 47 | return responses 48 | -------------------------------------------------------------------------------- /examples/quick-start/resnet50_python/config.pbtxt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | name: "resnet50_python" 16 | backend: "python" 17 | max_batch_size: 128 18 | input { 19 | name: "INPUT0" 20 | data_type: TYPE_FP32 21 | format: FORMAT_NCHW 22 | dims: [ 3, 224, 224 ] 23 | } 24 | output { 25 | name: "OUTPUT0" 26 | data_type: TYPE_FP32 27 | dims: [ 1000 ] 28 | } 29 | -------------------------------------------------------------------------------- /examples/quick-start/sub/config.pbtxt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | name: "sub" 17 | backend: "python" 18 | 19 | input [ 20 | { 21 | name: "INPUT0" 22 | data_type: TYPE_FP32 23 | dims: [ 4 ] 24 | } 25 | ] 26 | input [ 27 | { 28 | name: "INPUT1" 29 | data_type: TYPE_FP32 30 | dims: [ 4 ] 31 | } 32 | ] 33 | output [ 34 | { 35 | name: "OUTPUT" 36 | data_type: TYPE_FP32 37 | dims: [ 4 ] 38 | } 39 | ] 40 | 41 | instance_group [{ kind: KIND_CPU }] 42 | -------------------------------------------------------------------------------- /experiments/data/resnet50_libtorch/config.pbtxt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: "resnet50_libtorch" 16 | platform: "pytorch_libtorch" 17 | max_batch_size: 128 18 | input { 19 | name: "INPUT__0" 20 | data_type: TYPE_FP32 21 | format: FORMAT_NCHW 22 | dims: [ 3, 224, 224 ] 23 | } 24 | output { 25 | name: "OUTPUT__0" 26 | data_type: TYPE_FP32 27 | dims: [ 1000 ] 28 | label_filename: "resnet50_labels.txt" 29 | } 30 | -------------------------------------------------------------------------------- /experiments/data/vgg19_libtorch/config.pbtxt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: "vgg19_libtorch" 16 | platform: "pytorch_libtorch" 17 | max_batch_size: 128 18 | input { 19 | name: "INPUT__0" 20 | data_type: TYPE_FP32 21 | format: FORMAT_NCHW 22 | dims: [ 3, 224, 224 ] 23 | } 24 | output { 25 | name: "OUTPUT__0" 26 | data_type: TYPE_FP32 27 | dims: [ 1000 ] 28 | label_filename: "vgg19_labels.txt" 29 | } 30 | -------------------------------------------------------------------------------- /experiments/scripts/minimize_latency.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | objectives: 16 | - perf_latency_p99 17 | -------------------------------------------------------------------------------- /experiments/start_profiles.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | CHECKPOINT_DIR="/mnt/nvdl/datasets/inferenceserver/model_analyzer_profile_results" 17 | 18 | for model in inception_v1_graphdef resnet50_libtorch vgg19_libtorch; do 19 | for radius in {2..8}; do 20 | for min_initialized in {2..8}; do 21 | echo "Profiling $model (radius = $radius, min-initialized = $min_initialized)" 22 | python3 main.py --save \ 23 | --model-name $model \ 24 | --run-config-search-mode quick \ 25 | --data-path $CHECKPOINT_DIR \ 26 | --output-path output \ 27 | --radius $radius \ 28 | --min-initialized $min_initialized 29 | done 30 | done 31 | done 32 | -------------------------------------------------------------------------------- /helm-chart/.helmignore: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Patterns to ignore when building packages. 16 | # This supports shell glob matching, relative path matching, and 17 | # negation (prefixed with !). Only one pattern per line. 18 | .DS_Store 19 | # Common VCS dirs 20 | .git/ 21 | .gitignore 22 | .bzr/ 23 | .bzrignore 24 | .hg/ 25 | .hgignore 26 | .svn/ 27 | # Common backup files 28 | *.swp 29 | *.bak 30 | *.tmp 31 | *~ 32 | # Various IDEs 33 | .project 34 | .idea/ 35 | *.tmproj 36 | .vscode/ 37 | -------------------------------------------------------------------------------- /helm-chart/Chart.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | appVersion: "1.0" 17 | description: A Helm chart for Kubernetes 18 | name: model-analyzer 19 | version: 0.1.0 20 | -------------------------------------------------------------------------------- /helm-chart/templates/config-map.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: ConfigMap 17 | metadata: 18 | name: analyzer-config 19 | namespace: default 20 | data: 21 | config.yaml: | 22 | ###################### 23 | # Config for profile # 24 | ###################### 25 | 26 | override_output_model_repository: True 27 | run_config_search_disable: True 28 | triton_http_endpoint: localhost:8000 29 | triton_grpc_endpoint: localhost:8001 30 | triton_metrics_url: http://localhost:8002/metrics 31 | 32 | concurrency: 1,2 33 | batch_sizes: 1 34 | 35 | profile_models: 36 | add_sub: 37 | model_config_parameters: 38 | instance_group: 39 | - 40 | kind: KIND_GPU 41 | count: [1] 42 | dynamic_batching: 43 | 44 | ###################### 45 | # Config for report # 46 | ###################### 47 | 48 | report_model_configs: 49 | - add_sub_config_default 50 | -------------------------------------------------------------------------------- /helm-chart/values.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Values for Triton Model Analyzer 16 | # This is a YAML-formatted file. 17 | # Declare variables to be passed into your templates. 18 | 19 | # Job timeout value specified in seconds 20 | jobTimeout: 900 21 | 22 | ## Configurations for mounting volumes 23 | 24 | # Local path to model directory 25 | modelPath: /home/models 26 | 27 | # Local path export model config variants 28 | outputModelPath: /home/output_models 29 | 30 | # Local path to export data 31 | resultsPath: /home/results 32 | 33 | # Local path to store checkpoints 34 | checkpointPath: /home/checkpoints 35 | 36 | ## Images 37 | images: 38 | 39 | analyzer: 40 | image: model-analyzer 41 | 42 | triton: 43 | image: nvcr.io/nvidia/tritonserver 44 | tag: 24.12-py3 45 | -------------------------------------------------------------------------------- /model_analyzer/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/cli/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/config/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/config/generate/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/config/generate/config_generator_interface.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import abc 18 | from typing import Any, Generator, List, Optional 19 | 20 | from model_analyzer.result.run_config_measurement import RunConfigMeasurement 21 | 22 | 23 | class ConfigGeneratorInterface(abc.ABC): 24 | """ 25 | An interface class for config generators 26 | """ 27 | 28 | @classmethod 29 | def __subclasshook__(cls, subclass: Any) -> bool: 30 | return ( 31 | hasattr(subclass, "__init__") 32 | and callable(subclass.__init__) 33 | and hasattr(subclass, "get_configs") 34 | and callable(subclass.get_configs) 35 | and hasattr(subclass, "set_last_results") 36 | and callable(subclass.set_last_results) 37 | or NotImplemented 38 | ) 39 | 40 | @abc.abstractmethod 41 | def __init__(self) -> None: 42 | raise NotImplementedError 43 | 44 | @abc.abstractmethod 45 | def get_configs(self) -> Generator[Any, None, None]: 46 | raise NotImplementedError 47 | 48 | @abc.abstractmethod 49 | def set_last_results( 50 | self, measurements: List[Optional[RunConfigMeasurement]] 51 | ) -> None: 52 | raise NotImplementedError 53 | -------------------------------------------------------------------------------- /model_analyzer/config/generate/search_parameter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from dataclasses import dataclass 18 | from enum import Enum, auto 19 | from typing import Any, List, Optional 20 | 21 | 22 | class ParameterUsage(Enum): 23 | MODEL = auto() 24 | RUNTIME = auto() 25 | BUILD = auto() 26 | 27 | 28 | class ParameterCategory(Enum): 29 | INTEGER = auto() 30 | EXPONENTIAL = auto() 31 | STR_LIST = auto() 32 | INT_LIST = auto() 33 | 34 | 35 | @dataclass 36 | class SearchParameter: 37 | """ 38 | A dataclass that holds information about a configuration's search parameter 39 | """ 40 | 41 | usage: ParameterUsage 42 | category: ParameterCategory 43 | 44 | # This is only applicable to the LIST categories 45 | enumerated_list: Optional[List[Any]] = None 46 | 47 | # These are only applicable to INTEGER and EXPONENTIAL categories 48 | min_range: Optional[int] = None 49 | max_range: Optional[int] = None 50 | -------------------------------------------------------------------------------- /model_analyzer/config/input/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/config/input/config_status.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | class ConfigStatus: 19 | def __init__(self, status, message=None, config_object=None): 20 | """ 21 | Create a new ConfigStatus 22 | 23 | Parameters 24 | ---------- 25 | status : int 26 | Status of the config parsing. Accepted 27 | values are CONFIG_PARSER_SUCCESS and 28 | CONFIG_PARSER_FAILURE. 29 | 30 | message : str 31 | A string message containing the description. 32 | 33 | config_object : ConfigValue 34 | ConfigObject that is creating this status. 35 | """ 36 | 37 | self._status = status 38 | self._message = message 39 | self._config_object = config_object 40 | 41 | def status(self): 42 | """ 43 | Get the config status. 44 | 45 | Returns 46 | ------- 47 | int 48 | Config status 49 | """ 50 | 51 | return self._status 52 | 53 | def message(self): 54 | """ 55 | Get the message. 56 | 57 | Returns 58 | ------- 59 | str 60 | The message for the status. 61 | """ 62 | 63 | return self._message 64 | 65 | def config_object(self): 66 | """ 67 | Get the config object for this status. 68 | 69 | Returns 70 | ------- 71 | ConfigValue or None 72 | Config object that created this status. 73 | """ 74 | 75 | return self._config_object 76 | -------------------------------------------------------------------------------- /model_analyzer/config/input/objects/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/config/run/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/constants.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from typing import Any, Dict 18 | 19 | # Config constants 20 | CONFIG_PARSER_SUCCESS = 1 21 | CONFIG_PARSER_FAILURE = 0 22 | 23 | # Result Table constants 24 | RESULT_TABLE_COLUMN_PADDING = 2 25 | 26 | # Result Comparator Constants 27 | COMPARISON_SCORE_THRESHOLD = 0 28 | 29 | # Dict of parameters to apply on top of the default 30 | # config to result in the default config (empty dict) 31 | DEFAULT_CONFIG_PARAMS: Dict[str, Any] = {} 32 | 33 | # Run Search 34 | THROUGHPUT_MINIMUM_GAIN = 0.05 35 | THROUGHPUT_MINIMUM_CONSECUTIVE_PARAMETER_TRIES = 4 36 | THROUGHPUT_MINIMUM_CONSECUTIVE_BATCH_SIZE_TRIES = 4 37 | 38 | # Quick search algorithm constants 39 | RADIUS = 3 40 | MIN_INITIALIZED = 3 41 | 42 | # Reports 43 | TOP_MODELS_REPORT_KEY = "Best Configs Across All Models" 44 | 45 | # State Management 46 | MAX_NUMBER_OF_INTERRUPTS = 3 47 | 48 | # Perf Analyzer 49 | MEASUREMENT_WINDOW_STEP = 1000 50 | MEASUREMENT_REQUEST_COUNT_STEP = 50 51 | INTERVAL_SLEEP_TIME = 1 52 | PERF_ANALYZER_MEASUREMENT_WINDOW = 5000 53 | PERF_ANALYZER_MINIMUM_REQUEST_COUNT = 50 54 | SECONDS_TO_MILLISECONDS_MULTIPLIER = 1000 55 | 56 | # Triton Server 57 | SERVER_OUTPUT_TIMEOUT_SECS = 5 58 | 59 | # Logging 60 | LOGGER_NAME = "model_analyzer_logger" 61 | 62 | # PA Error Log Filename 63 | PA_ERROR_LOG_FILENAME = "perf_analyzer_error.log" 64 | 65 | # Constraints 66 | GLOBAL_CONSTRAINTS_KEY = "__default__" 67 | 68 | # Measurement constants 69 | INVALID_MEASUREMENT_THRESHOLD = 2 70 | 71 | # Model analyzer package name 72 | PACKAGE_NAME = "triton-model-analyzer" 73 | 74 | # GENAI-PERF 75 | GENAI_PERF_CSV = "profile_export_genai_perf.csv" 76 | GENAI_PERF_COLLATERAL = ["llm_inputs.json", "profile_export.json"] 77 | -------------------------------------------------------------------------------- /model_analyzer/device/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/device/device.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | class Device: 19 | """ 20 | Generic device class representing devices being monitored 21 | """ 22 | 23 | def __init__(self): 24 | pass 25 | -------------------------------------------------------------------------------- /model_analyzer/log_formatter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | import sys 19 | 20 | from model_analyzer.constants import LOGGER_NAME 21 | 22 | 23 | class LogFormatter(logging.Formatter): 24 | """Class to handle formatting of the logger outputs""" 25 | 26 | def __init__(self): 27 | logger = logging.getLogger(LOGGER_NAME) 28 | self._log_level = logger.getEffectiveLevel() 29 | super().__init__(datefmt="%H:%M:%S") 30 | 31 | def format(self, record): 32 | front = "%(asctime)s " if self._log_level is logging.DEBUG else "" 33 | if record.levelno == logging.INFO: 34 | self._style._fmt = f"{front}[Model Analyzer] %(message)s" 35 | else: 36 | self._style._fmt = f"{front}[Model Analyzer] %(levelname)s: %(message)s" 37 | return super().format(record) 38 | 39 | 40 | def setup_logging(quiet, verbose): 41 | """ 42 | Setup logger format 43 | 44 | Parameters 45 | ---------- 46 | quiet : bool 47 | If true, don't print anything other than errors 48 | verbose : bool 49 | If true and quiet is not true, print debug information 50 | """ 51 | 52 | if quiet: 53 | log_level = logging.ERROR 54 | elif verbose: 55 | log_level = logging.DEBUG 56 | else: 57 | log_level = logging.INFO 58 | 59 | logger = logging.getLogger(LOGGER_NAME) 60 | logger.setLevel(level=log_level) 61 | 62 | handler = logging.StreamHandler(sys.stdout) 63 | handler.setFormatter(LogFormatter()) 64 | logger.addHandler(handler) 65 | logger.propagate = False 66 | -------------------------------------------------------------------------------- /model_analyzer/model_analyzer_exceptions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | class TritonModelAnalyzerException(Exception): 19 | """ 20 | A custom exception specific to the Triton Model 21 | Analyzer 22 | """ 23 | 24 | pass 25 | -------------------------------------------------------------------------------- /model_analyzer/monitor/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/monitor/dcgm/DcgmStatus.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import model_analyzer.monitor.dcgm.pydcgm as pydcgm 16 | import model_analyzer.monitor.dcgm.dcgm_agent as dcgm_agent 17 | import model_analyzer.monitor.dcgm.dcgm_structs as dcgm_structs 18 | 19 | 20 | class DcgmStatus: 21 | 22 | def __init__(self): 23 | self.handle = dcgm_agent.dcgmStatusCreate() 24 | self.errors = [] 25 | 26 | def __del__(self): 27 | dcgm_agent.dcgmStatusDestroy(self.handle) 28 | 29 | ''' 30 | Take any errors stored in our handle and update self.errors with them 31 | ''' 32 | 33 | def UpdateErrors(self): 34 | errorCount = dcgm_agent.dcgmStatusGetCount(self.handle) 35 | if errorCount < 1: 36 | return 37 | 38 | for i in range(errorCount): 39 | self.errors.append(dcgm_agent.dcgmStatusPopError(self.handle)) 40 | 41 | ''' 42 | Throw an exception if any errors are stored in our status handle 43 | 44 | The exception text will contain all of the errors 45 | ''' 46 | 47 | def ThrowExceptionOnErrors(self): 48 | #Make sure we've captured all errors before looking at them 49 | self.UpdateErrors() 50 | 51 | if len(self.errors) < 1: 52 | return 53 | 54 | errorString = "Errors: " 55 | for value in self.errors: 56 | errorString += "\"%s\"" % value 57 | raise dcgm_structs.DCGMError(value.status) 58 | -------------------------------------------------------------------------------- /model_analyzer/monitor/dcgm/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/monitor/dcgm/common/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /model_analyzer/monitor/dcgm/dcgm_fields_internal.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | ## 15 | # Python bindings for the internal API of DCGM library (dcgm_fields_internal.hpp) 16 | ## 17 | 18 | from ctypes import * 19 | from ctypes.util import find_library 20 | import model_analyzer.monitor.dcgm.dcgm_structs as dcgm_structs 21 | 22 | # Provides access to functions 23 | dcgmFP = dcgm_structs._dcgmGetFunctionPointer 24 | 25 | #internal-only fields 26 | DCGM_FI_DEV_MEM_COPY_UTIL_SAMPLES = 210 #Memory utilization samples 27 | DCGM_FI_DEV_GPU_UTIL_SAMPLES = 211 #SM utilization samples 28 | DCGM_FI_DEV_GRAPHICS_PIDS = 220 #Graphics processes running on the GPU. 29 | DCGM_FI_DEV_COMPUTE_PIDS = 221 #Compute processes running on the GPU. 30 | -------------------------------------------------------------------------------- /model_analyzer/monitor/dcgm/dcgm_fluentd.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from model_analyzer.monitor.dcgm.common.dcgm_client_main import main 15 | from model_analyzer.monitor.dcgm.DcgmJsonReader import DcgmJsonReader 16 | from socket import socket, AF_INET, SOCK_DGRAM 17 | 18 | # Displayed to the user 19 | FLUENTD_NAME = 'Fluentd' 20 | DEFAULT_FLUENTD_PORT = 24225 21 | 22 | # Fluentd Configuration 23 | # ===================== 24 | # In order to use this client, Fluentd needs to accept json over udp. 25 | # The default port is 24225 26 | 27 | 28 | class DcgmFluentd(DcgmJsonReader): 29 | ########################################################################### 30 | def __init__(self, publish_hostname, publish_port, **kwargs): 31 | self.m_sock = socket(AF_INET, SOCK_DGRAM) 32 | self.m_dest = (publish_hostname, publish_port) 33 | super(DcgmFluentd, self).__init__(**kwargs) 34 | 35 | ########################################################################### 36 | def SendToFluentd(self, payload): 37 | self.m_sock.sendto(payload, self.m_dest) 38 | 39 | ########################################################################### 40 | def CustomJsonHandler(self, outJson): 41 | self.SendToFluentd(outJson) 42 | 43 | 44 | if __name__ == '__main__': # pragma: no cover 45 | main(DcgmFluentd, FLUENTD_NAME, DEFAULT_FLUENTD_PORT, add_target_host=True) 46 | -------------------------------------------------------------------------------- /model_analyzer/monitor/dcgm/pydcgm.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | def _python_version_check(): 17 | import sys 18 | python_version = sys.version.split(None, 1)[0] 19 | if python_version < '3': 20 | print( 21 | '[ERROR] Detected Python version {}. These bindings are for Python 3.5+. Please load the Python 2 bindings found at /usr/local/dcgm/bindings' 22 | .format(python_version)) 23 | sys.exit(1) 24 | 25 | 26 | _python_version_check() 27 | 28 | #Bring classes into this namespace 29 | from model_analyzer.monitor.dcgm.DcgmHandle import * 30 | from model_analyzer.monitor.dcgm.DcgmGroup import * 31 | from model_analyzer.monitor.dcgm.DcgmStatus import * 32 | from model_analyzer.monitor.dcgm.DcgmSystem import * 33 | from model_analyzer.monitor.dcgm.DcgmFieldGroup import * 34 | 35 | import os 36 | if '__DCGM_TESTING_FRAMEWORK_ACTIVE' in os.environ and os.environ[ 37 | '__DCGM_TESTING_FRAMEWORK_ACTIVE'] == '1': 38 | import utils 39 | import model_analyzer.monitor.dcgm.dcgm_structs as dcgm_structs 40 | dcgm_structs._dcgmInit(utils.get_testing_framework_library_path()) 41 | ''' 42 | Define a unique exception type we will return so that callers can distinguish our exceptions from python standard ones 43 | ''' 44 | 45 | 46 | class DcgmException(Exception): 47 | pass 48 | -------------------------------------------------------------------------------- /model_analyzer/output/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/output/file_writer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from model_analyzer.model_analyzer_exceptions import TritonModelAnalyzerException 18 | 19 | from .output_writer import OutputWriter 20 | 21 | 22 | class FileWriter(OutputWriter): 23 | """ 24 | Writes table to a file or stdout 25 | """ 26 | 27 | def __init__(self, filename=None): 28 | """ 29 | Parameters 30 | ---------- 31 | filename : File 32 | The full path to the file or stream to write the output to. 33 | Writes to stdout if filename is None 34 | """ 35 | 36 | self._filename = filename 37 | 38 | def write(self, out, append=False): 39 | """ 40 | Writes the output to a file or stdout 41 | 42 | Parameters 43 | ---------- 44 | out : str 45 | The string to be written to the 46 | file or stdout 47 | 48 | Raises 49 | ------ 50 | TritonModelAnalyzerException 51 | If there is an error or exception while writing 52 | the output. 53 | """ 54 | 55 | write_mode = "a+" if append else "w+" 56 | if self._filename: 57 | try: 58 | with open(self._filename, write_mode) as f: 59 | f.write(out) 60 | except OSError as e: 61 | raise TritonModelAnalyzerException(e) 62 | else: 63 | print(out, end="") 64 | -------------------------------------------------------------------------------- /model_analyzer/output/output_writer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from abc import ABC, abstractmethod 18 | 19 | 20 | class OutputWriter(ABC): 21 | """ 22 | Interface that receives a table 23 | and writes the table to a file or stream. 24 | """ 25 | 26 | @abstractmethod 27 | def write(self, out): 28 | """ 29 | Writes the output to a file 30 | (stdout, .txt, .csv etc.) 31 | 32 | Parameters 33 | ---------- 34 | out : str 35 | The string to be written out 36 | 37 | Raises 38 | ------ 39 | TritonModelAnalyzerException 40 | If there is an error or exception while writing 41 | the output. 42 | """ 43 | -------------------------------------------------------------------------------- /model_analyzer/perf_analyzer/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/plots/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/record/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/record/gpu_record.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .record import Record 18 | 19 | 20 | class GPURecord(Record): 21 | """ 22 | This is a base class for any 23 | GPU based record 24 | """ 25 | 26 | def __init__(self, value, device_uuid=None, timestamp=0): 27 | """ 28 | Parameters 29 | ---------- 30 | value : float 31 | The value of the GPU metrtic 32 | device_uuid : str 33 | The GPU device uuid this metric is associated 34 | with. 35 | timestamp : int 36 | The timestamp for the record in nanoseconds 37 | """ 38 | 39 | super().__init__(value, timestamp) 40 | self._device_uuid = device_uuid 41 | 42 | def device_uuid(self): 43 | """ 44 | Returns 45 | ------- 46 | str 47 | uuid for the GPU that this metric was sampled on 48 | """ 49 | 50 | return self._device_uuid 51 | 52 | @classmethod 53 | def from_dict(cls, record_dict): 54 | record = cls(0) 55 | for key in ["_value", "_timestamp", "_device"]: 56 | if key in record_dict: 57 | setattr(record, key, record_dict[key]) 58 | return record 59 | 60 | 61 | class IncreasingGPURecord(GPURecord): 62 | def _positive_is_better(self) -> bool: 63 | return True 64 | 65 | 66 | class DecreasingGPURecord(GPURecord): 67 | def _positive_is_better(self) -> bool: 68 | return False 69 | -------------------------------------------------------------------------------- /model_analyzer/record/types/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/record/types/inter_token_latency_avg.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.inter_token_latency_base import InterTokenLatencyBase 20 | 21 | 22 | @total_ordering 23 | class InterTokenLatencyAvg(InterTokenLatencyBase): 24 | """ 25 | A record for perf_analyzer Inter token latency metric 26 | """ 27 | 28 | tag = "inter_token_latency_avg" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "Avg Inter Token Latency (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/inter_token_latency_max.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.inter_token_latency_base import InterTokenLatencyBase 20 | 21 | 22 | @total_ordering 23 | class InterTokenLatencyMax(InterTokenLatencyBase): 24 | """ 25 | A record for perf_analyzer Inter token latency metric 26 | """ 27 | 28 | tag = "inter_token_latency_max" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "Max Inter Token Latency (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/inter_token_latency_min.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.inter_token_latency_base import InterTokenLatencyBase 20 | 21 | 22 | @total_ordering 23 | class InterTokenLatencyMin(InterTokenLatencyBase): 24 | """ 25 | A record for perf_analyzer Inter token latency metric 26 | """ 27 | 28 | tag = "inter_token_latency_min" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "Min Inter Token Latency (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/inter_token_latency_p25.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.inter_token_latency_base import InterTokenLatencyBase 20 | 21 | 22 | @total_ordering 23 | class InterTokenLatencyP25(InterTokenLatencyBase): 24 | """ 25 | A record for perf_analyzer Inter token latency metric 26 | """ 27 | 28 | tag = "inter_token_latency_p25" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p25 Inter Token Latency (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/inter_token_latency_p50.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.inter_token_latency_base import InterTokenLatencyBase 20 | 21 | 22 | @total_ordering 23 | class InterTokenLatencyP50(InterTokenLatencyBase): 24 | """ 25 | A record for perf_analyzer Inter token latency metric 26 | """ 27 | 28 | tag = "inter_token_latency_p50" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p50 Inter Token Latency (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/inter_token_latency_p75.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.inter_token_latency_base import InterTokenLatencyBase 20 | 21 | 22 | @total_ordering 23 | class InterTokenLatencyP75(InterTokenLatencyBase): 24 | """ 25 | A record for perf_analyzer Inter token latency metric 26 | """ 27 | 28 | tag = "inter_token_latency_p75" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p75 Inter Token Latency (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/inter_token_latency_p90.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.inter_token_latency_base import InterTokenLatencyBase 20 | 21 | 22 | @total_ordering 23 | class InterTokenLatencyP90(InterTokenLatencyBase): 24 | """ 25 | A record for perf_analyzer Inter token latency metric 26 | """ 27 | 28 | tag = "inter_token_latency_p90" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p90 Inter Token Latency (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/inter_token_latency_p95.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.inter_token_latency_base import InterTokenLatencyBase 20 | 21 | 22 | @total_ordering 23 | class InterTokenLatencyP95(InterTokenLatencyBase): 24 | """ 25 | A record for perf_analyzer Inter token latency metric 26 | """ 27 | 28 | tag = "inter_token_latency_p95" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p95 Inter Token Latency (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/inter_token_latency_p99.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.inter_token_latency_base import InterTokenLatencyBase 20 | 21 | 22 | @total_ordering 23 | class InterTokenLatencyP99(InterTokenLatencyBase): 24 | """ 25 | A record for perf_analyzer Inter token latency metric 26 | """ 27 | 28 | tag = "inter_token_latency_p99" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p99 Inter Token Latency (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/perf_latency_avg.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.perf_latency_base import PerfLatencyBase 20 | 21 | 22 | @total_ordering 23 | class PerfLatencyAvg(PerfLatencyBase): 24 | """ 25 | A record for perf_analyzer latency metric 26 | """ 27 | 28 | tag = "perf_latency_avg" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "Avg Latency (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/perf_latency_base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.record import DecreasingRecord 20 | 21 | 22 | @total_ordering 23 | class PerfLatencyBase(DecreasingRecord): 24 | """ 25 | A base class for perf_analyzer latency metric 26 | """ 27 | 28 | def __init__(self, value, timestamp=0): 29 | """ 30 | Parameters 31 | ---------- 32 | value : float 33 | the latency extracted from the perf analyzer output 34 | timestamp : float 35 | Elapsed time from start of program 36 | """ 37 | 38 | super().__init__(value, timestamp) 39 | 40 | def __eq__(self, other): 41 | """ 42 | Allows checking for 43 | equality between two records 44 | """ 45 | 46 | return self.value() == other.value() 47 | 48 | def __lt__(self, other): 49 | """ 50 | Allows checking if 51 | this record is less than 52 | the other 53 | """ 54 | 55 | return self.value() > other.value() 56 | 57 | def __add__(self, other): 58 | """ 59 | Allows adding two records together 60 | to produce a brand new record. 61 | """ 62 | 63 | return self.__class__(value=(self.value() + other.value())) 64 | 65 | def __sub__(self, other): 66 | """ 67 | Allows subbing two records together 68 | to produce a brand new record. 69 | 70 | ** Note this does reverse subtraction because 71 | of the inverted nature of latency (lower is better) 72 | """ 73 | 74 | return self.__class__(value=(other.value() - self.value())) 75 | -------------------------------------------------------------------------------- /model_analyzer/record/types/perf_latency_p90.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.perf_latency_base import PerfLatencyBase 20 | 21 | 22 | @total_ordering 23 | class PerfLatencyP90(PerfLatencyBase): 24 | """ 25 | A record for perf_analyzer latency metric 26 | """ 27 | 28 | tag = "perf_latency_p90" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p90 Latency (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/perf_latency_p95.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.perf_latency_base import PerfLatencyBase 20 | 21 | 22 | @total_ordering 23 | class PerfLatencyP95(PerfLatencyBase): 24 | """ 25 | A record for perf_analyzer latency metric 26 | """ 27 | 28 | tag = "perf_latency_p95" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p95 Latency (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/perf_latency_p99.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.perf_latency_base import PerfLatencyBase 20 | 21 | 22 | @total_ordering 23 | class PerfLatencyP99(PerfLatencyBase): 24 | """ 25 | A record for perf_analyzer latency metric 26 | """ 27 | 28 | tag = "perf_latency_p99" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p99 Latency (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/time_to_first_token_avg.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.time_to_first_token_base import TimeToFirstTokenBase 20 | 21 | 22 | @total_ordering 23 | class TimeToFirstTokenAvg(TimeToFirstTokenBase): 24 | """ 25 | A record for perf_analyzer Time to first token metric 26 | """ 27 | 28 | tag = "time_to_first_token_avg" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "Avg Time To First Token (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/time_to_first_token_max.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.time_to_first_token_base import TimeToFirstTokenBase 20 | 21 | 22 | @total_ordering 23 | class TimeToFirstTokenMax(TimeToFirstTokenBase): 24 | """ 25 | A record for perf_analyzer Time to first token metric 26 | """ 27 | 28 | tag = "time_to_first_token_max" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "Max Time To First Token (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/time_to_first_token_min.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.time_to_first_token_base import TimeToFirstTokenBase 20 | 21 | 22 | @total_ordering 23 | class TimeToFirstTokenMin(TimeToFirstTokenBase): 24 | """ 25 | A record for perf_analyzer Time to first token metric 26 | """ 27 | 28 | tag = "time_to_first_token_min" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "Min Time To First Token (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/time_to_first_token_p25.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.time_to_first_token_base import TimeToFirstTokenBase 20 | 21 | 22 | @total_ordering 23 | class TimeToFirstTokenP25(TimeToFirstTokenBase): 24 | """ 25 | A record for perf_analyzer Time to first token metric 26 | """ 27 | 28 | tag = "time_to_first_token_p25" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p25 Time To First Token (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/time_to_first_token_p50.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.time_to_first_token_base import TimeToFirstTokenBase 20 | 21 | 22 | @total_ordering 23 | class TimeToFirstTokenP50(TimeToFirstTokenBase): 24 | """ 25 | A record for perf_analyzer Time to first token metric 26 | """ 27 | 28 | tag = "time_to_first_token_p50" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p50 Time To First Token (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/time_to_first_token_p75.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.time_to_first_token_base import TimeToFirstTokenBase 20 | 21 | 22 | @total_ordering 23 | class TimeToFirstTokenP75(TimeToFirstTokenBase): 24 | """ 25 | A record for perf_analyzer Time to first token metric 26 | """ 27 | 28 | tag = "time_to_first_token_p75" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p75 Time To First Token (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/time_to_first_token_p90.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.time_to_first_token_base import TimeToFirstTokenBase 20 | 21 | 22 | @total_ordering 23 | class TimeToFirstTokenP90(TimeToFirstTokenBase): 24 | """ 25 | A record for perf_analyzer Time to first token metric 26 | """ 27 | 28 | tag = "time_to_first_token_p90" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p90 Time To First Token (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/time_to_first_token_p95.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.time_to_first_token_base import TimeToFirstTokenBase 20 | 21 | 22 | @total_ordering 23 | class TimeToFirstTokenP95(TimeToFirstTokenBase): 24 | """ 25 | A record for perf_analyzer Time to first token metric 26 | """ 27 | 28 | tag = "time_to_first_token_p95" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p95 Time To First Token (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/record/types/time_to_first_token_p99.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from functools import total_ordering 18 | 19 | from model_analyzer.record.types.time_to_first_token_base import TimeToFirstTokenBase 20 | 21 | 22 | @total_ordering 23 | class TimeToFirstTokenP99(TimeToFirstTokenBase): 24 | """ 25 | A record for perf_analyzer Time to first token metric 26 | """ 27 | 28 | tag = "time_to_first_token_p99" 29 | 30 | def __init__(self, value, timestamp=0): 31 | """ 32 | Parameters 33 | ---------- 34 | value : float 35 | the latency extracted from the perf analyzer output 36 | timestamp : float 37 | Elapsed time from start of program 38 | """ 39 | 40 | super().__init__(value, timestamp) 41 | 42 | @classmethod 43 | def header(cls, aggregation_tag=False): 44 | """ 45 | Parameters 46 | ---------- 47 | aggregation_tag: bool 48 | An optional tag that may be displayed 49 | as part of the header indicating that 50 | this record has been aggregated using 51 | max, min or average etc. 52 | 53 | Returns 54 | ------- 55 | str 56 | The full name of the 57 | metric. 58 | """ 59 | 60 | return "p99 Time To First Token (ms)" 61 | -------------------------------------------------------------------------------- /model_analyzer/reports/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/reports/pdf_report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import pdfkit 18 | 19 | from .html_report import HTMLReport 20 | 21 | 22 | class PDFReport(HTMLReport): 23 | """ 24 | A report that takes 25 | an html report and converts 26 | it to PDF 27 | """ 28 | 29 | def __init__(self, html_report=None): 30 | super().__init__(html_report) 31 | 32 | def write_report(self, filename): 33 | """ 34 | Write the report to disk with 35 | filename 36 | 37 | Parameters 38 | ---------- 39 | filename : str 40 | The name of the report 41 | """ 42 | 43 | pdfkit.from_string(self.document(), f"{filename}", options={"quiet": ""}) 44 | 45 | def get_file_extension(self): 46 | """ 47 | Return the file extension for 48 | the type of report 49 | """ 50 | return "pdf" 51 | -------------------------------------------------------------------------------- /model_analyzer/reports/report_factory.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | import shutil 19 | 20 | from model_analyzer.constants import LOGGER_NAME 21 | 22 | from .html_report import HTMLReport 23 | from .pdf_report import PDFReport 24 | 25 | logger = logging.getLogger(LOGGER_NAME) 26 | 27 | 28 | class ReportFactory: 29 | """ 30 | Factory that returns the correct report object 31 | """ 32 | 33 | PDF_PACKAGE = "wkhtmltopdf" 34 | WARNING_PRINTED = False 35 | 36 | @staticmethod 37 | def create_report(): 38 | if ReportFactory._is_package_installed(f"{ReportFactory.PDF_PACKAGE}"): 39 | return ReportFactory._create_pdf_report() 40 | else: 41 | if not ReportFactory.WARNING_PRINTED: 42 | ReportFactory.WARNING_PRINTED = True 43 | logging.warning( 44 | f"Warning: html reports are being generated instead of pdf because " 45 | f"{ReportFactory.PDF_PACKAGE} is not installed. If you want pdf " 46 | f"reports, run the following command and then rerun Model Analyzer: " 47 | f'"sudo apt-get update && sudo apt-get install wkhtmltopdf"' 48 | ) 49 | return ReportFactory._create_html_report() 50 | 51 | @staticmethod 52 | def _create_pdf_report(): 53 | return PDFReport() 54 | 55 | @staticmethod 56 | def _create_html_report(): 57 | return HTMLReport() 58 | 59 | @staticmethod 60 | def _is_package_installed(package_name): 61 | package_found = shutil.which(f"{package_name}") 62 | return package_found is not None 63 | -------------------------------------------------------------------------------- /model_analyzer/reports/report_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | def truncate_model_config_name(model_config_name): 19 | """ 20 | Truncates the model configuration name if its length exceeds the threshold length. 21 | ex: long_model_name_config_4 --> long_mod..._config_4 22 | Parameters 23 | ---------- 24 | model_config_name: string 25 | Returns 26 | ------- 27 | string 28 | The truncated model configuration name, 29 | or the original name if it is shorter than the threshold length. 30 | """ 31 | max_model_config_name_len = 35 32 | 33 | if len(model_config_name) > max_model_config_name_len: 34 | config_name = model_config_name[model_config_name.rfind("config_") :] 35 | 36 | return ( 37 | model_config_name[: (max_model_config_name_len - len(config_name) - 3)] 38 | + "..." 39 | + config_name 40 | ) 41 | 42 | return model_config_name 43 | -------------------------------------------------------------------------------- /model_analyzer/result/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/result/result_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """ 17 | A module for various functions 18 | needed across the result module 19 | """ 20 | 21 | 22 | def format_for_csv(obj, interior=False): 23 | """ 24 | Takes an object, which could be a string, int, or list of either 25 | and formats it so it will be parsable in a csv 26 | """ 27 | if type(obj) == list: 28 | if len(obj) > 1: 29 | if interior: 30 | return f" [{','.join([str(o) for o in obj])}]" 31 | else: 32 | return ( 33 | '"' 34 | + ",".join([format_for_csv(o, interior=True) for o in obj]) 35 | + '"' 36 | ) 37 | else: 38 | return str(obj[0]) 39 | elif type(obj) == str and "," in obj: 40 | return '"' + obj + '"' 41 | else: 42 | return str(obj) 43 | -------------------------------------------------------------------------------- /model_analyzer/state/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/triton/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/triton/client/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/triton/client/client_factory.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from .grpc_client import TritonGRPCClient 18 | from .http_client import TritonHTTPClient 19 | 20 | 21 | class TritonClientFactory: 22 | """ 23 | Base client creator class that declares 24 | a factory method 25 | """ 26 | 27 | @staticmethod 28 | def create_grpc_client(server_url, ssl_options={}): 29 | """ 30 | Parameters 31 | ---------- 32 | server_url : str 33 | The url for Triton server's GRPC endpoint 34 | ssl_options : dict 35 | Dictionary of SSL options for gRPC python client 36 | 37 | Returns 38 | ------- 39 | TritonGRPCClient 40 | """ 41 | return TritonGRPCClient(server_url=server_url, ssl_options=ssl_options) 42 | 43 | @staticmethod 44 | def create_http_client(server_url, ssl_options={}): 45 | """ 46 | Parameters 47 | ---------- 48 | server_url : str 49 | The url for Triton server's HTTP endpoint 50 | ssl_options : dict 51 | Dictionary of SSL options for HTTP python client 52 | 53 | Returns 54 | ------- 55 | TritonHTTPClient 56 | """ 57 | return TritonHTTPClient(server_url=server_url, ssl_options=ssl_options) 58 | -------------------------------------------------------------------------------- /model_analyzer/triton/model/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/triton/model/model_config_variant.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from dataclasses import dataclass 16 | 17 | from model_analyzer.triton.model.model_config import ModelConfig 18 | 19 | 20 | @dataclass 21 | class ModelConfigVariant: 22 | """ 23 | A dataclass that holds the ModelConfig as well as the variant name 24 | and cpu_only flag for the model 25 | """ 26 | 27 | model_config: ModelConfig 28 | variant_name: str 29 | cpu_only: bool = False 30 | -------------------------------------------------------------------------------- /model_analyzer/triton/server/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /model_analyzer/triton/server/server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from abc import ABC, abstractmethod 18 | from io import TextIOWrapper 19 | 20 | 21 | class TritonServer(ABC): 22 | """ 23 | Defines the interface for the objects created by 24 | TritonServerFactory 25 | """ 26 | 27 | @abstractmethod 28 | def start(self, env=None): 29 | """ 30 | Starts the tritonserver 31 | 32 | Parameters 33 | ---------- 34 | env: dict 35 | The environment to set for this tritonserver launch 36 | """ 37 | 38 | @abstractmethod 39 | def stop(self): 40 | """ 41 | Stops and cleans up after the server 42 | """ 43 | 44 | @abstractmethod 45 | def log_file(self) -> TextIOWrapper: 46 | """ 47 | Returns the server's log file 48 | """ 49 | 50 | @abstractmethod 51 | def cpu_stats(self): 52 | """ 53 | Returns the CPU memory usage and CPU available memory in MB 54 | """ 55 | 56 | def update_config(self, params): 57 | """ 58 | Update the server's arguments 59 | 60 | Parameters 61 | ---------- 62 | params: dict 63 | keys are argument names and values are their values. 64 | """ 65 | 66 | self._server_config.update_config(params) 67 | 68 | def config(self): 69 | """ 70 | Returns 71 | ------- 72 | TritonServerConfig 73 | This server's config 74 | """ 75 | 76 | return self._server_config 77 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [tool.codespell] 16 | # note: pre-commit passes explicit lists of files here, which this skip file list doesn't override - 17 | # this is only to allow you to run codespell interactively 18 | skip = "./.git,./.github" 19 | # ignore short words, and typename parameters like OffsetT 20 | ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b" 21 | # ignore allowed words 22 | ignore-words-list = "parms,passin" 23 | # use the 'clear' dictionary for unambiguous spelling mistakes 24 | builtin = "clear" 25 | # disable warnings about binary files and wrong encoding 26 | quiet-level = 3 27 | 28 | [tool.isort] 29 | profile = "black" 30 | use_parentheses = true 31 | multi_line_output = 3 32 | include_trailing_comma = true 33 | force_grid_wrap = 0 34 | ensure_newline_before_comments = true 35 | line_length = 88 36 | balanced_wrapping = true 37 | indent = " " 38 | skip = ["build"] 39 | 40 | -------------------------------------------------------------------------------- /qa/L0_bls_model/input_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": [ 3 | { 4 | "MODEL_NAME": [ 5 | "add" 6 | ], 7 | "INPUT0": [ 8 | 0.74106514, 9 | 0.7371813, 10 | 0.5274665, 11 | 0.13930903 12 | ], 13 | "INPUT1": [ 14 | 0.7845891, 15 | 0.88089234, 16 | 0.8466405, 17 | 0.55024815 18 | ] 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /qa/L0_cpu/test_config_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import argparse 18 | 19 | import yaml 20 | 21 | 22 | class TestConfigGenerator: 23 | """ 24 | This class contains functions that 25 | create configs for various test scenarios. 26 | 27 | TO ADD A TEST: Simply add a member function whose name starts 28 | with 'generate'. 29 | """ 30 | 31 | def __init__(self): 32 | test_functions = [ 33 | self.__getattribute__(name) 34 | for name in dir(self) 35 | if name.startswith("generate") 36 | ] 37 | 38 | for test_function in test_functions: 39 | self.setUp() 40 | test_function() 41 | 42 | def setUp(self): 43 | parser = argparse.ArgumentParser() 44 | parser.add_argument( 45 | "-m", 46 | "--model-names", 47 | type=str, 48 | required=True, 49 | help="The config file for this test", 50 | ) 51 | 52 | args = parser.parse_args() 53 | self.model_names = args.model_names.split(",") 54 | 55 | def generate_profile_config(self): 56 | self.config = {} 57 | self.config["run_config_search_max_concurrency"] = 4 58 | self.config["run_config_search_max_instance_count"] = 2 59 | self.config["run_config_search_max_model_batch_size"] = 1 60 | self.config["profile_models"] = { 61 | model_name: {"cpu_only": True} for model_name in self.model_names 62 | } 63 | with open("config-profile.yml", "w+") as f: 64 | yaml.dump(self.config, f) 65 | 66 | 67 | if __name__ == "__main__": 68 | TestConfigGenerator() 69 | -------------------------------------------------------------------------------- /qa/L0_doa/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | source ../common/util.sh 17 | create_logs_dir "L0_doa" 18 | create_result_paths -export-path false -checkpoints false 19 | 20 | MODEL_ANALYZER=`which model-analyzer` 21 | 22 | RET=0 23 | 24 | set +e 25 | run_analyzer 26 | if [ $? != 0 ]; then 27 | echo -e "\n***\n*** Failed to run model-analyzer. \n***" 28 | cat $ANALYZER_LOG 29 | RET=1 30 | fi 31 | set -e 32 | 33 | if [ $RET -eq 0 ]; then 34 | echo -e "\n***\n*** Test PASSED\n***" 35 | else 36 | echo -e "\n***\n*** Test FAILED\n***" 37 | fi 38 | exit $RET 39 | -------------------------------------------------------------------------------- /qa/L0_doc_links/mkdocs.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | site_name: CI Test 16 | use_directory_urls: False 17 | docs_dir: "../../docs" 18 | plugins: 19 | - htmlproofer 20 | - search 21 | -------------------------------------------------------------------------------- /qa/L0_doc_links/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | source ../common/util.sh 17 | create_logs_dir "L0_doc_links" 18 | 19 | mkdir -p $LOGS_DIR/logs 20 | LOG=$LOGS_DIR/logs/test.log 21 | CONFIG="`pwd`/mkdocs.yml" 22 | RET=0 23 | 24 | exec mkdocs serve -f $CONFIG > $LOG & 25 | PID=$! 26 | sleep 20 27 | 28 | until [[ (-z `pgrep mkdocs`) ]]; do 29 | kill -2 $PID 30 | sleep 2 31 | done 32 | 33 | if [[ ! -z `grep "invalid url" $LOG` ]]; then 34 | cat $LOG 35 | RET=1 36 | fi 37 | 38 | 39 | if [ $RET -eq 0 ]; then 40 | echo -e "\n***\n*** Test PASSED\n***" 41 | else 42 | echo -e "\n***\n*** Test FAILED\n***" 43 | fi 44 | exit $RET 45 | -------------------------------------------------------------------------------- /qa/L0_ensemble_model/test_config_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import argparse 18 | 19 | import yaml 20 | 21 | 22 | class TestConfigGenerator: 23 | """ 24 | This class contains functions that 25 | create configs for various test scenarios. 26 | 27 | The `setup` function does the work common to all tests 28 | 29 | TO ADD A TEST: Simply add a member function whose name starts 30 | with 'generate'. 31 | """ 32 | 33 | def __init__(self): 34 | test_functions = [ 35 | self.__getattribute__(name) 36 | for name in dir(self) 37 | if name.startswith("generate") 38 | ] 39 | 40 | for test_function in test_functions: 41 | self.setup() 42 | test_function() 43 | 44 | def setup(self): 45 | parser = argparse.ArgumentParser() 46 | parser.add_argument( 47 | "-m", 48 | "--profile-models", 49 | type=str, 50 | required=True, 51 | help="Comma separated list of models to be profiled", 52 | ) 53 | 54 | args = parser.parse_args() 55 | self.config = {} 56 | self.config["profile_models"] = sorted(args.profile_models.split(",")) 57 | 58 | def generate_config(self): 59 | with open("config.yml", "w+") as f: 60 | yaml.dump(self.config, f) 61 | 62 | 63 | if __name__ == "__main__": 64 | TestConfigGenerator() 65 | -------------------------------------------------------------------------------- /qa/L0_multi_model_profile/test_config_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import argparse 18 | 19 | import yaml 20 | 21 | 22 | class TestConfigGenerator: 23 | """ 24 | This class contains functions that 25 | create configs for various test scenarios. 26 | 27 | The `setup` function does the work common to all tests 28 | 29 | TO ADD A TEST: Simply add a member function whose name starts 30 | with 'generate'. 31 | """ 32 | 33 | def __init__(self): 34 | test_functions = [ 35 | self.__getattribute__(name) 36 | for name in dir(self) 37 | if name.startswith("generate") 38 | ] 39 | 40 | for test_function in test_functions: 41 | self.setup() 42 | test_function() 43 | 44 | def setup(self): 45 | parser = argparse.ArgumentParser() 46 | parser.add_argument( 47 | "-m", 48 | "--profile-models", 49 | type=str, 50 | required=True, 51 | help="The config file for this test", 52 | ) 53 | 54 | args = parser.parse_args() 55 | self.config = { 56 | "batch_sizes": [1, 2], 57 | "concurrency": [1, 2], 58 | } 59 | self.config["profile_models"] = sorted(args.profile_models.split(",")) 60 | self.config["run_config_search_disable"] = True 61 | self.config["run_config_profile_models_concurrently_enable"] = True 62 | 63 | def generate_config(self): 64 | with open("config.yml", "w+") as f: 65 | yaml.dump(self.config, f) 66 | 67 | 68 | if __name__ == "__main__": 69 | TestConfigGenerator() 70 | -------------------------------------------------------------------------------- /qa/L0_optuna_bls_model/input_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": [ 3 | { 4 | "MODEL_NAME": [ 5 | "add" 6 | ], 7 | "INPUT0": [ 8 | 0.74106514, 9 | 0.7371813, 10 | 0.5274665, 11 | 0.13930903 12 | ], 13 | "INPUT1": [ 14 | 0.7845891, 15 | 0.88089234, 16 | 0.8466405, 17 | 0.55024815 18 | ] 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /qa/L0_optuna_ensemble_model/test_config_generator.py: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import argparse 17 | 18 | import yaml 19 | 20 | 21 | class TestConfigGenerator: 22 | """ 23 | This class contains functions that 24 | create configs for various test scenarios. 25 | 26 | The `setup` function does the work common to all tests 27 | 28 | TO ADD A TEST: Simply add a member function whose name starts 29 | with 'generate'. 30 | """ 31 | 32 | def __init__(self): 33 | test_functions = [ 34 | self.__getattribute__(name) 35 | for name in dir(self) 36 | if name.startswith("generate") 37 | ] 38 | 39 | for test_function in test_functions: 40 | self.setup() 41 | test_function() 42 | 43 | def setup(self): 44 | parser = argparse.ArgumentParser() 45 | parser.add_argument( 46 | "-m", 47 | "--profile-models", 48 | type=str, 49 | required=True, 50 | help="Comma separated list of models to be profiled", 51 | ) 52 | 53 | args = parser.parse_args() 54 | self.config = {} 55 | self.config["profile_models"] = sorted(args.profile_models.split(",")) 56 | 57 | def generate_config(self): 58 | with open("config.yml", "w+") as f: 59 | yaml.dump(self.config, f) 60 | 61 | 62 | if __name__ == "__main__": 63 | TestConfigGenerator() 64 | -------------------------------------------------------------------------------- /qa/L0_optuna_multi_model/test_config_generator.py: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import argparse 17 | 18 | import yaml 19 | 20 | 21 | class TestConfigGenerator: 22 | """ 23 | This class contains functions that 24 | create configs for various test scenarios. 25 | 26 | The `setup` function does the work common to all tests 27 | 28 | TO ADD A TEST: Simply add a member function whose name starts 29 | with 'generate'. 30 | """ 31 | 32 | def __init__(self): 33 | test_functions = [ 34 | self.__getattribute__(name) 35 | for name in dir(self) 36 | if name.startswith("generate") 37 | ] 38 | 39 | for test_function in test_functions: 40 | self.setup() 41 | test_function() 42 | 43 | def setup(self): 44 | parser = argparse.ArgumentParser() 45 | parser.add_argument( 46 | "-m", 47 | "--profile-models", 48 | type=str, 49 | required=True, 50 | help="The config file for this test", 51 | ) 52 | 53 | args = parser.parse_args() 54 | self.config = {} 55 | self.config["profile_models"] = sorted(args.profile_models.split(",")) 56 | 57 | def generate_config(self): 58 | with open("config.yml", "w+") as f: 59 | yaml.dump(self.config, f) 60 | 61 | 62 | if __name__ == "__main__": 63 | TestConfigGenerator() 64 | -------------------------------------------------------------------------------- /qa/L0_optuna_search/test_config_generator.py: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import argparse 17 | 18 | import yaml 19 | 20 | 21 | class TestConfigGenerator: 22 | """ 23 | This class contains functions that 24 | create configs for various test scenarios. 25 | 26 | The `setup` function does the work common to all tests 27 | 28 | TO ADD A TEST: Simply add a member function whose name starts 29 | with 'generate'. 30 | """ 31 | 32 | def __init__(self): 33 | test_functions = [ 34 | self.__getattribute__(name) 35 | for name in dir(self) 36 | if name.startswith("generate") 37 | ] 38 | 39 | for test_function in test_functions: 40 | self.setup() 41 | test_function() 42 | 43 | def setup(self): 44 | parser = argparse.ArgumentParser() 45 | parser.add_argument( 46 | "-m", 47 | "--profile-models", 48 | type=str, 49 | required=True, 50 | help="The config file for this test", 51 | ) 52 | 53 | args = parser.parse_args() 54 | self.config = {} 55 | self.config["profile_models"] = sorted(args.profile_models.split(",")) 56 | 57 | def generate_config(self): 58 | with open("config.yml", "w+") as f: 59 | yaml.dump(self.config, f) 60 | 61 | 62 | if __name__ == "__main__": 63 | TestConfigGenerator() 64 | -------------------------------------------------------------------------------- /qa/L0_quick_search/test_config_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import argparse 18 | 19 | import yaml 20 | 21 | 22 | class TestConfigGenerator: 23 | """ 24 | This class contains functions that 25 | create configs for various test scenarios. 26 | 27 | The `setup` function does the work common to all tests 28 | 29 | TO ADD A TEST: Simply add a member function whose name starts 30 | with 'generate'. 31 | """ 32 | 33 | def __init__(self): 34 | test_functions = [ 35 | self.__getattribute__(name) 36 | for name in dir(self) 37 | if name.startswith("generate") 38 | ] 39 | 40 | for test_function in test_functions: 41 | self.setup() 42 | test_function() 43 | 44 | def setup(self): 45 | parser = argparse.ArgumentParser() 46 | parser.add_argument( 47 | "-m", 48 | "--profile-models", 49 | type=str, 50 | required=True, 51 | help="The config file for this test", 52 | ) 53 | 54 | args = parser.parse_args() 55 | self.config = {} 56 | self.config["profile_models"] = sorted(args.profile_models.split(",")) 57 | 58 | def generate_config(self): 59 | with open("config.yml", "w+") as f: 60 | yaml.dump(self.config, f) 61 | 62 | 63 | if __name__ == "__main__": 64 | TestConfigGenerator() 65 | -------------------------------------------------------------------------------- /qa/L0_quick_search_multi_model/test_config_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import argparse 18 | 19 | import yaml 20 | 21 | 22 | class TestConfigGenerator: 23 | """ 24 | This class contains functions that 25 | create configs for various test scenarios. 26 | 27 | The `setup` function does the work common to all tests 28 | 29 | TO ADD A TEST: Simply add a member function whose name starts 30 | with 'generate'. 31 | """ 32 | 33 | def __init__(self): 34 | test_functions = [ 35 | self.__getattribute__(name) 36 | for name in dir(self) 37 | if name.startswith("generate") 38 | ] 39 | 40 | for test_function in test_functions: 41 | self.setup() 42 | test_function() 43 | 44 | def setup(self): 45 | parser = argparse.ArgumentParser() 46 | parser.add_argument( 47 | "-m", 48 | "--profile-models", 49 | type=str, 50 | required=True, 51 | help="The config file for this test", 52 | ) 53 | 54 | args = parser.parse_args() 55 | self.config = {} 56 | self.config["profile_models"] = sorted(args.profile_models.split(",")) 57 | 58 | def generate_config(self): 59 | with open("config.yml", "w+") as f: 60 | yaml.dump(self.config, f) 61 | 62 | 63 | if __name__ == "__main__": 64 | TestConfigGenerator() 65 | -------------------------------------------------------------------------------- /qa/L0_quick_start/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | exit_early_if_nonzero () { 17 | if [[ $1 -ne 0 ]] ; then \ 18 | echo -e "\n***\n*** Test FAILED\n***" ; \ 19 | exit 1 ; \ 20 | fi 21 | } 22 | 23 | model-analyzer profile \ 24 | -m /opt/triton-model-analyzer/examples/quick-start \ 25 | --profile-models add_sub \ 26 | --run-config-search-max-concurrency 2 \ 27 | --run-config-search-max-instance-count 2 ; \ 28 | exit_early_if_nonzero $? 29 | 30 | mkdir profile_results ; \ 31 | exit_early_if_nonzero $? 32 | 33 | model-analyzer report \ 34 | --report-model-configs add_sub_config_0,add_sub_config_1 \ 35 | -e profile_results ; \ 36 | exit_early_if_nonzero $? 37 | 38 | echo -e "\n***\n*** Test PASSED\n***" 39 | exit 0 40 | -------------------------------------------------------------------------------- /qa/L0_ssl_grpc/broken_config.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Invalid config file for Model Analyzer. It uses the corrupted ssl keys 16 | # generated from the following lines: 17 | # cp client.key client2.key && sed -i "s/\b\(.\)/\u\1/g" client2.key 18 | # cp client.crt client2.crt && sed -i "s/\b\(.\)/\u\1/g" client2.crt 19 | 20 | batch_sizes: 21 | - 1 22 | - 2 23 | concurrency: 24 | - 1 25 | - 2 26 | perf_analyzer_flags: 27 | ssl-grpc-certificate-chain-file: ./client2.crt 28 | ssl-grpc-private-key-file: ./client2.key 29 | ssl-grpc-root-certifications-file: ./ca.crt 30 | ssl-grpc-use-ssl: true 31 | profile_models: 32 | - resnet50_libtorch 33 | run_config_search_disable: true 34 | triton_server_flags: 35 | grpc-root-cert: ./ca.crt 36 | grpc-server-cert: ./server.crt 37 | grpc-server-key: ./server.key 38 | grpc-use-ssl: '1' 39 | grpc-use-ssl-mutual: '1' 40 | -------------------------------------------------------------------------------- /qa/L0_ssl_grpc/working_config.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Valid config file for Model Analyzer. It uses the correct ssl keys. 16 | 17 | batch_sizes: 18 | - 1 19 | - 2 20 | concurrency: 21 | - 1 22 | - 2 23 | perf_analyzer_flags: 24 | ssl-grpc-certificate-chain-file: ./client.crt 25 | ssl-grpc-private-key-file: ./client.key 26 | ssl-grpc-root-certifications-file: ./ca.crt 27 | ssl-grpc-use-ssl: true 28 | profile_models: 29 | - resnet50_libtorch 30 | run_config_search_disable: true 31 | triton_server_flags: 32 | grpc-root-cert: ./ca.crt 33 | grpc-server-cert: ./server.crt 34 | grpc-server-key: ./server.key 35 | grpc-use-ssl: '1' 36 | grpc-use-ssl-mutual: '1' 37 | -------------------------------------------------------------------------------- /qa/L0_ssl_https/broken_config.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Invalid config file for Model Analyzer. It uses the corrupted ssl keys 16 | # generated from the following lines: 17 | # cp client.key client2.key && sed -i "s/\b\(.\)/\u\1/g" client2.key 18 | # cp client.crt client2.crt && sed -i "s/\b\(.\)/\u\1/g" client2.crt 19 | 20 | batch_sizes: 21 | - 1 22 | - 2 23 | concurrency: 24 | - 1 25 | - 2 26 | perf_analyzer_flags: 27 | ssl-https-ca-certificates-file: ./ca.crt 28 | ssl-https-client-certificate-file: ./client2.crt 29 | ssl-https-client-certificate-type: PEM 30 | ssl-https-private-key-file: ./client2.key 31 | ssl-https-private-key-type: PEM 32 | ssl-https-verify-host: 2 33 | ssl-https-verify-peer: 1 34 | profile_models: 35 | - resnet50_libtorch 36 | run_config_search_disable: true 37 | -------------------------------------------------------------------------------- /qa/L0_ssl_https/nginx.conf: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | server { 16 | listen 443 ssl; 17 | server_name localhost; 18 | 19 | ssl_certificate /etc/nginx/cert.crt; 20 | ssl_certificate_key /etc/nginx/cert.key; 21 | 22 | location / { 23 | proxy_pass http://localhost:8000; 24 | proxy_http_version 1.1; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /qa/L0_ssl_https/working_config.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Valid config file for Model Analyzer. It uses the correct ssl keys. 16 | 17 | batch_sizes: 18 | - 1 19 | - 2 20 | concurrency: 21 | - 1 22 | - 2 23 | perf_analyzer_flags: 24 | ssl-https-ca-certificates-file: ./ca.crt 25 | ssl-https-client-certificate-file: ./client.crt 26 | ssl-https-client-certificate-type: PEM 27 | ssl-https-private-key-file: ./client.key 28 | ssl-https-private-key-type: PEM 29 | ssl-https-verify-host: 2 30 | ssl-https-verify-peer: 1 31 | profile_models: 32 | - resnet50_libtorch 33 | run_config_search_disable: true 34 | -------------------------------------------------------------------------------- /qa/L0_stability_result/test_config_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import argparse 18 | 19 | import yaml 20 | 21 | 22 | class TestConfigGenerator: 23 | """ 24 | This class contains functions that 25 | create configs for various test scenarios. 26 | 27 | The `setup` function does the work common to all tests 28 | 29 | TO ADD A TEST: Simply add a member function whose name starts 30 | with 'generate'. 31 | """ 32 | 33 | def __init__(self): 34 | test_functions = [ 35 | self.__getattribute__(name) 36 | for name in dir(self) 37 | if name.startswith("generate") 38 | ] 39 | 40 | for test_function in test_functions: 41 | self.setup() 42 | test_function() 43 | 44 | def setup(self): 45 | parser = argparse.ArgumentParser() 46 | parser.add_argument( 47 | "-m", 48 | "--profile-models", 49 | type=str, 50 | required=True, 51 | help="The models used for this test", 52 | ) 53 | 54 | self.args = parser.parse_args() 55 | self.config = {} 56 | self.config["profile_models"] = {} 57 | for model in sorted(self.args.profile_models.split(",")): 58 | self.config["profile_models"][model] = { 59 | "objectives": {"perf_throughput": 10} 60 | } 61 | 62 | def generate_configs(self): 63 | with open("config.yaml", "w+") as f: 64 | yaml.dump(self.config, f) 65 | 66 | 67 | if __name__ == "__main__": 68 | TestConfigGenerator() 69 | -------------------------------------------------------------------------------- /qa/L0_stability_steps/test_config_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import argparse 18 | 19 | import yaml 20 | 21 | 22 | class TestConfigGenerator: 23 | """ 24 | This class contains functions that 25 | create configs for various test scenarios. 26 | 27 | The `setup` function does the work common to all tests 28 | 29 | TO ADD A TEST: Simply add a member function whose name starts 30 | with 'generate'. 31 | """ 32 | 33 | def __init__(self): 34 | test_functions = [ 35 | self.__getattribute__(name) 36 | for name in dir(self) 37 | if name.startswith("generate") 38 | ] 39 | 40 | for test_function in test_functions: 41 | self.setup() 42 | test_function() 43 | 44 | def setup(self): 45 | parser = argparse.ArgumentParser() 46 | parser.add_argument( 47 | "-m", 48 | "--profile-models", 49 | type=str, 50 | required=True, 51 | help="The models used for this test", 52 | ) 53 | 54 | self.args = parser.parse_args() 55 | self.profile_models = sorted(self.args.profile_models.split(",")) 56 | 57 | self.config = {} 58 | self.config["run_config_search_disable"] = False 59 | self.config["profile_models"] = self.profile_models 60 | 61 | def generate_configs(self): 62 | with open("config.yaml", "w+") as f: 63 | yaml.dump(self.config, f) 64 | 65 | 66 | if __name__ == "__main__": 67 | TestConfigGenerator() 68 | -------------------------------------------------------------------------------- /qa/L0_unit_tests/count_tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import argparse 18 | import importlib 19 | import inspect 20 | import os 21 | import sys 22 | 23 | sys.path.insert(0, "../../") 24 | 25 | 26 | def args(): 27 | parser = argparse.ArgumentParser("test_counter") 28 | parser.add_argument("--path", help="Path to use for counting the tests", type=str) 29 | opt = parser.parse_args() 30 | return opt 31 | 32 | 33 | if __name__ == "__main__": 34 | number_of_tests = 0 35 | opt = args() 36 | path = opt.path 37 | 38 | for file_path in os.listdir(path): 39 | # All the test files start with "Test" 40 | if file_path.startswith("test_"): 41 | module_name = "tests." + file_path.split(".")[0] 42 | module = importlib.import_module(module_name) 43 | classes = inspect.getmembers(module, inspect.isclass) 44 | for class_tuple in classes: 45 | class_name = class_tuple[0] 46 | class_object = class_tuple[1] 47 | 48 | # All the test classes start with "Test" 49 | if class_name.startswith("Test"): 50 | methods = inspect.getmembers(class_object, inspect.isroutine) 51 | for method_tuple in methods: 52 | method_name = method_tuple[0] 53 | if method_name.startswith("test_"): 54 | number_of_tests += 1 55 | 56 | # Print the number of tests 57 | print(number_of_tests) 58 | -------------------------------------------------------------------------------- /qa/L0_unit_tests/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | EXPECTED_NUM_TESTS=`python3 count_tests.py --path ../../tests/` 17 | source ../common/check_analyzer_results.sh 18 | source ../common/util.sh 19 | 20 | create_logs_dir "L0_unit_tests" 21 | create_result_paths -export-path false -checkpoints false 22 | 23 | RET=0 24 | 25 | set +e 26 | coverage run --branch --source=../../model_analyzer -m unittest discover -v -s ../../tests -t ../../ > $ANALYZER_LOG 2>&1 27 | if [ $? -ne 0 ]; then 28 | RET=1 29 | else 30 | check_unit_test_results $ANALYZER_LOG $EXPECTED_NUM_TESTS 31 | if [ $? -ne 0 ]; then 32 | cat $ANALYZER_LOG 33 | echo -e "\n***\n*** Test Result Verification Failed\n***" 34 | RET=1 35 | fi 36 | fi 37 | set -e 38 | 39 | if [ $RET -eq 0 ]; then 40 | echo -e "\n***\n*** Test PASSED\n***" 41 | else 42 | cat $ANALYZER_LOG 43 | echo -e "\n***\n*** Test FAILED\n***" 44 | fi 45 | 46 | # Generate html files 47 | echo `pwd` 48 | coverage html --directory $LOGS_DIR/html/ 49 | 50 | exit $RET 51 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | cryptography>=3.3.2 16 | distro>=1.5.0 17 | docker>=4.3.1 18 | gevent>=22.08.0 19 | grpcio>=1.41.0,<1.68 20 | httplib2>=0.19.0 21 | importlib_metadata>=7.1.0 22 | matplotlib>=3.3.4 23 | numba>=0.51.2 24 | optuna==3.6.1 25 | pdfkit>=0.6.1 26 | prometheus_client>=0.9.0 27 | protobuf 28 | psutil>=5.8.0 29 | pyyaml>=5.3.1 30 | requests>=2.24.0 31 | tritonclient[all]>=2.4.0 32 | urllib3>=2.0.7 33 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /tests/common/bls-ckpt/README: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | This checkpoint was created by running the model from the BLS Quick Start. 16 | -------------------------------------------------------------------------------- /tests/common/bls-ckpt/golden-metrics-server-only.csv: -------------------------------------------------------------------------------- 1 | Model,GPU UUID,GPU Memory Usage (MB),GPU Utilization (%),GPU Power Usage (W) 2 | triton-server,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,872.0,0.0,56.0 3 | 4 | -------------------------------------------------------------------------------- /tests/common/ensemble-ckpt/README: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | This checkpoint was created by running the Ensemble Quick Start. 16 | -------------------------------------------------------------------------------- /tests/common/ensemble-ckpt/golden-metrics-server-only.csv: -------------------------------------------------------------------------------- 1 | Model,GPU UUID,GPU Memory Usage (MB),GPU Utilization (%),GPU Power Usage (W) 2 | triton-server,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,870.0,0.0,56.1 3 | 4 | -------------------------------------------------------------------------------- /tests/common/multi-model-ckpt/README: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | Settings used to create this checkpoint: 16 | 17 | --profile-models resnet50_libtorch,vgg19_libtorch 18 | --run-config-profile-models-concurrently-enable 19 | --run-config-search-mode quick -------------------------------------------------------------------------------- /tests/common/multi-model-ckpt/golden-metrics-model-gpu.csv: -------------------------------------------------------------------------------- 1 | Model,GPU UUID,Batch,Concurrency,Model Config Path,Instance Group,Satisfies Constraints,GPU Memory Usage (MB),GPU Utilization (%),GPU Power Usage (W) 2 | "resnet50_libtorch,vgg19_libtorch",GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,"1,1","4,4","resnet50_libtorch_config_5,vgg19_libtorch_config_4","2:GPU,1:GPU",Yes,2602.6,100.0,276.1 3 | "resnet50_libtorch,vgg19_libtorch",GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,"1,1","1,1","resnet50_libtorch_config_5,vgg19_libtorch_config_4","2:GPU,1:GPU",Yes,2575.3,98.5,276.8 4 | "resnet50_libtorch,vgg19_libtorch",GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,"1,1","4,8","resnet50_libtorch_config_5,vgg19_libtorch_config_7","2:GPU,1:GPU",Yes,2680.2,100.0,274.2 5 | "resnet50_libtorch,vgg19_libtorch",GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,"1,1","1,1","resnet50_libtorch_config_5,vgg19_libtorch_config_7","2:GPU,1:GPU",Yes,2575.3,93.7,277.0 6 | "resnet50_libtorch,vgg19_libtorch",GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,"1,1","1,1","resnet50_libtorch_config_default,vgg19_libtorch_config_default","1:GPU,1:GPU",Yes,2296.4,99.7,273.8 7 | "resnet50_libtorch,vgg19_libtorch",GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,"1,1","4,4","resnet50_libtorch_config_default,vgg19_libtorch_config_default","1:GPU,1:GPU",Yes,2296.4,50.5,178.0 8 | "resnet50_libtorch,vgg19_libtorch",GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,"1,1","2,2","resnet50_libtorch_config_default,vgg19_libtorch_config_default","1:GPU,1:GPU",Yes,2296.4,99.5,277.9 9 | "resnet50_libtorch,vgg19_libtorch",GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,"1,1","8,8","resnet50_libtorch_config_default,vgg19_libtorch_config_default","1:GPU,1:GPU",Yes,2296.4,60.3,216.1 10 | "resnet50_libtorch,vgg19_libtorch",GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,"1,1","2,4","resnet50_libtorch_config_0,vgg19_libtorch_config_4","1:GPU,1:GPU",Yes,2323.6,99.3,275.4 11 | "resnet50_libtorch,vgg19_libtorch",GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,"1,1","2,2","resnet50_libtorch_config_0,vgg19_libtorch_config_0","1:GPU,1:GPU",Yes,2296.4,97.7,269.3 12 | "resnet50_libtorch,vgg19_libtorch",GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,"1,1","4,4","resnet50_libtorch_config_7,vgg19_libtorch_config_1","1:GPU,2:GPU",Yes,3089.1,79.7,280.6 13 | "resnet50_libtorch,vgg19_libtorch",GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,"1,1","2,8","resnet50_libtorch_config_0,vgg19_libtorch_config_7","1:GPU,1:GPU",Yes,2401.2,78.2,280.0 14 | 15 | -------------------------------------------------------------------------------- /tests/common/multi-model-ckpt/golden-metrics-model-inference.csv: -------------------------------------------------------------------------------- 1 | Model,Batch,Concurrency,Model Config Path,Instance Group,Max Batch Size,Satisfies Constraints,Throughput (infer/sec),p99 Latency (ms) 2 | "resnet50_libtorch,vgg19_libtorch","1,1","4,4","resnet50_libtorch_config_5,vgg19_libtorch_config_4","2:GPU,1:GPU","1,2",Yes,"421.0, [184.5,236.5]","20.7, [23.9,17.4]" 3 | "resnet50_libtorch,vgg19_libtorch","1,1","1,1","resnet50_libtorch_config_5,vgg19_libtorch_config_4","2:GPU,1:GPU","1,2",Yes,"340.1, [168.9,171.2]","6.5, [7.0,6.0]" 4 | "resnet50_libtorch,vgg19_libtorch","1,1","4,8","resnet50_libtorch_config_5,vgg19_libtorch_config_7","2:GPU,1:GPU","1,4",Yes,"419.6, [139.9,279.7]","30.1, [31.2,29.0]" 5 | "resnet50_libtorch,vgg19_libtorch","1,1","1,1","resnet50_libtorch_config_5,vgg19_libtorch_config_7","2:GPU,1:GPU","1,4",Yes,"299.7, [123.6,176.2]","8.0, [10.0,6.0]" 6 | "resnet50_libtorch,vgg19_libtorch","1,1","1,1","resnet50_libtorch_config_default,vgg19_libtorch_config_default","1:GPU,1:GPU","128,128",Yes,"344.1, [167.9,176.2]","6.8, [7.7,5.9]" 7 | "resnet50_libtorch,vgg19_libtorch","1,1","4,4","resnet50_libtorch_config_default,vgg19_libtorch_config_default","1:GPU,1:GPU","128,128",Yes,"338.8, [139.2,199.5]","25.6, [30.4,20.7]" 8 | "resnet50_libtorch,vgg19_libtorch","1,1","2,2","resnet50_libtorch_config_default,vgg19_libtorch_config_default","1:GPU,1:GPU","128,128",Yes,"339.1, [138.6,200.5]","13.0, [15.6,10.3]" 9 | "resnet50_libtorch,vgg19_libtorch","1,1","8,8","resnet50_libtorch_config_default,vgg19_libtorch_config_default","1:GPU,1:GPU","128,128",Yes,"338.1, [138.5,199.5]","50.7, [60.7,40.7]" 10 | "resnet50_libtorch,vgg19_libtorch","1,1","2,4","resnet50_libtorch_config_0,vgg19_libtorch_config_4","1:GPU,1:GPU","1,2",Yes,"368.4, [108.6,259.8]","17.9, [19.9,15.8]" 11 | "resnet50_libtorch,vgg19_libtorch","1,1","2,2","resnet50_libtorch_config_0,vgg19_libtorch_config_0","1:GPU,1:GPU","1,1",Yes,"341.4, [139.6,201.8]","12.9, [15.5,10.3]" 12 | "resnet50_libtorch,vgg19_libtorch","1,1","4,4","resnet50_libtorch_config_7,vgg19_libtorch_config_1","1:GPU,2:GPU","2,1",Yes,"345.4, [108.6,236.8]","28.4, [39.2,17.5]" 13 | "resnet50_libtorch,vgg19_libtorch","1,1","2,8","resnet50_libtorch_config_0,vgg19_libtorch_config_7","1:GPU,1:GPU","1,4",Yes,"374.7, [74.9,299.8]","27.1, [27.1,27.1]" 14 | 15 | -------------------------------------------------------------------------------- /tests/common/multi-model-ckpt/golden-metrics-server-only.csv: -------------------------------------------------------------------------------- 1 | Model,GPU UUID,GPU Memory Usage (MB),GPU Utilization (%),GPU Power Usage (W) 2 | triton-server,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,870.0,0.0,55.8 3 | 4 | -------------------------------------------------------------------------------- /tests/common/multi-model-ckpt/plot_manager.json: -------------------------------------------------------------------------------- 1 | {"_simple_plots": {"resnet50_libtorch,vgg19_libtorch": {"throughput_v_latency": {"_name": "throughput_v_latency", "_title": "Throughput vs. Latency", "_x_axis": "perf_latency_p99", "_y_axis": "perf_throughput", "_monotonic": true, "_data": {"resnet50_libtorch_config_5,vgg19_libtorch_config_4": {"x_data": [20.661, 6.497], "y_data": [420.97299999999996, 340.14300000000003]}, "resnet50_libtorch_config_5,vgg19_libtorch_config_7": {"x_data": [30.093, 8.0365], "y_data": [419.61899999999997, 299.746]}, "resnet50_libtorch_config_default,vgg19_libtorch_config_default": {"x_data": [6.7745, 25.5655, 12.9815, 50.6635], "y_data": [344.119, 338.762, 339.052, 338.05600000000004]}}}, "gpu_mem_v_latency": {"_name": "gpu_mem_v_latency", "_title": "GPU Memory vs. Latency", "_x_axis": "perf_latency_p99", "_y_axis": "gpu_used_memory", "_monotonic": false, "_data": {"resnet50_libtorch_config_5,vgg19_libtorch_config_4": {"x_data": [20.661, 6.497], "y_data": [2602.565632, 2575.302656]}, "resnet50_libtorch_config_5,vgg19_libtorch_config_7": {"x_data": [30.093, 8.0365], "y_data": [2680.160256, 2575.302656]}, "resnet50_libtorch_config_default,vgg19_libtorch_config_default": {"x_data": [6.7745, 25.5655, 12.9815, 50.6635], "y_data": [2296.38144, 2296.38144, 2296.38144, 2296.38144]}}}}}} -------------------------------------------------------------------------------- /tests/common/request-rate-ckpt/README: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | Settings used to create this checkpoint: 16 | 17 | --profile-models add_sub 18 | --request-rate 16,32 19 | --run-config-search-max-instance-count 2 20 | --run-config-search-max-model-batch-size 2 21 | --request-rate-search-enable 22 | -------------------------------------------------------------------------------- /tests/common/request-rate-ckpt/golden-metrics-model-inference.csv: -------------------------------------------------------------------------------- 1 | Model,Batch,Request Rate,Model Config Path,Instance Group,Max Batch Size,Satisfies Constraints,Throughput (infer/sec),p99 Latency (ms) 2 | add_sub,2,32,add_sub_config_5,3:GPU,2,Yes,64.0,1.4 3 | add_sub,2,16,add_sub_config_5,3:GPU,2,Yes,32.2,1.5 4 | add_sub,1,32,add_sub_config_5,3:GPU,2,Yes,32.0,1.5 5 | add_sub,1,16,add_sub_config_5,3:GPU,2,Yes,16.1,1.4 6 | add_sub,2,32,add_sub_config_7,4:GPU,2,Yes,64.0,1.4 7 | add_sub,2,16,add_sub_config_7,4:GPU,2,Yes,32.2,1.5 8 | add_sub,1,32,add_sub_config_7,4:GPU,2,Yes,32.0,1.5 9 | add_sub,1,16,add_sub_config_7,4:GPU,2,Yes,16.1,1.5 10 | add_sub,2,32,add_sub_config_3,2:GPU,2,Yes,64.0,1.5 11 | add_sub,2,16,add_sub_config_3,2:GPU,2,Yes,32.2,1.4 12 | add_sub,1,32,add_sub_config_3,2:GPU,2,Yes,32.0,1.4 13 | add_sub,1,16,add_sub_config_3,2:GPU,2,Yes,16.1,1.4 14 | add_sub,2,32,add_sub_config_default,1:GPU,8,Yes,64.0,1.4 15 | add_sub,2,16,add_sub_config_default,1:GPU,8,Yes,32.2,1.5 16 | add_sub,1,32,add_sub_config_default,1:GPU,8,Yes,32.0,1.4 17 | add_sub,1,16,add_sub_config_default,1:GPU,8,Yes,16.1,1.4 18 | add_sub,2,32,add_sub_config_9,5:GPU,2,Yes,64.0,1.5 19 | add_sub,2,16,add_sub_config_9,5:GPU,2,Yes,32.2,1.5 20 | add_sub,1,32,add_sub_config_9,5:GPU,2,Yes,32.0,1.5 21 | add_sub,1,16,add_sub_config_9,5:GPU,2,Yes,16.1,1.5 22 | add_sub,2,32,add_sub_config_1,1:GPU,2,Yes,64.0,1.5 23 | add_sub,2,16,add_sub_config_1,1:GPU,2,Yes,32.2,1.6 24 | add_sub,1,32,add_sub_config_1,1:GPU,2,Yes,32.0,1.5 25 | add_sub,1,16,add_sub_config_1,1:GPU,2,Yes,16.1,1.5 26 | add_sub,1,32,add_sub_config_4,3:GPU,1,Yes,32.0,1.5 27 | add_sub,1,16,add_sub_config_4,3:GPU,1,Yes,16.1,1.5 28 | add_sub,1,32,add_sub_config_6,4:GPU,1,Yes,32.0,1.5 29 | add_sub,1,16,add_sub_config_6,4:GPU,1,Yes,16.1,1.5 30 | add_sub,1,32,add_sub_config_0,1:GPU,1,Yes,32.0,1.5 31 | add_sub,1,16,add_sub_config_0,1:GPU,1,Yes,16.1,1.5 32 | add_sub,1,32,add_sub_config_2,2:GPU,1,Yes,32.0,1.5 33 | add_sub,1,16,add_sub_config_2,2:GPU,1,Yes,16.1,1.5 34 | add_sub,1,32,add_sub_config_8,5:GPU,1,Yes,32.0,1.4 35 | add_sub,1,16,add_sub_config_8,5:GPU,1,Yes,16.1,1.5 36 | 37 | -------------------------------------------------------------------------------- /tests/common/request-rate-ckpt/golden-metrics-server-only.csv: -------------------------------------------------------------------------------- 1 | Model,GPU UUID,GPU Memory Usage (MB),GPU Utilization (%),GPU Power Usage (W) 2 | triton-server,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,870.0,0.0,56.1 3 | 4 | -------------------------------------------------------------------------------- /tests/common/single-model-ckpt/README: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | Settings used to create this checkpoint: 16 | 17 | --profile-models add_sub 18 | -b 1,2 19 | -c 1,2 20 | --run-config-search-max-concurrency 2 21 | --run-config-search-max-instance-count 2 22 | --run-config-search-max-model-batch-size 2 -------------------------------------------------------------------------------- /tests/common/single-model-ckpt/golden-metrics-model-gpu.csv: -------------------------------------------------------------------------------- 1 | Model,GPU UUID,Batch,Concurrency,Model Config Path,Instance Group,Satisfies Constraints,GPU Memory Usage (MB),GPU Utilization (%),GPU Power Usage (W) 2 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,2,2,add_sub_config_default,1:GPU,Yes,872.4,5.5,56.4 3 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,1,2,add_sub_config_default,1:GPU,Yes,872.4,4.2,56.3 4 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,2,1,add_sub_config_default,1:GPU,Yes,872.4,3.0,56.2 5 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,1,1,add_sub_config_default,1:GPU,Yes,872.4,2.0,56.2 6 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,2,2,add_sub_config_1,1:GPU,Yes,872.4,6.0,56.6 7 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,1,2,add_sub_config_1,1:GPU,Yes,872.4,4.8,56.6 8 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,2,1,add_sub_config_1,1:GPU,Yes,872.4,3.5,56.5 9 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,1,1,add_sub_config_1,1:GPU,Yes,872.4,1.8,56.4 10 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,2,2,add_sub_config_3,2:GPU,Yes,874.5,2.8,56.8 11 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,2,1,add_sub_config_3,2:GPU,Yes,874.5,3.0,56.8 12 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,1,2,add_sub_config_3,2:GPU,Yes,874.5,3.0,56.6 13 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,1,1,add_sub_config_3,2:GPU,Yes,874.5,1.7,56.7 14 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,1,2,add_sub_config_0,1:GPU,Yes,872.4,4.0,56.5 15 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,1,1,add_sub_config_0,1:GPU,Yes,872.4,2.0,56.4 16 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,1,2,add_sub_config_2,2:GPU,Yes,874.5,2.0,56.7 17 | add_sub,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,1,1,add_sub_config_2,2:GPU,Yes,874.5,3.0,56.6 18 | 19 | -------------------------------------------------------------------------------- /tests/common/single-model-ckpt/golden-metrics-model-inference.csv: -------------------------------------------------------------------------------- 1 | Model,Batch,Concurrency,Model Config Path,Instance Group,Max Batch Size,Satisfies Constraints,Throughput (infer/sec),p99 Latency (ms) 2 | add_sub,2,2,add_sub_config_default,1:GPU,8,Yes,13205.0,0.5 3 | add_sub,1,2,add_sub_config_default,1:GPU,8,Yes,6176.0,0.6 4 | add_sub,2,1,add_sub_config_default,1:GPU,8,Yes,4876.1,0.7 5 | add_sub,1,1,add_sub_config_default,1:GPU,8,Yes,2508.0,0.5 6 | add_sub,2,2,add_sub_config_1,1:GPU,2,Yes,12528.9,0.6 7 | add_sub,1,2,add_sub_config_1,1:GPU,2,Yes,6109.9,0.6 8 | add_sub,2,1,add_sub_config_1,1:GPU,2,Yes,4804.5,0.6 9 | add_sub,1,1,add_sub_config_1,1:GPU,2,Yes,2186.0,0.6 10 | add_sub,2,2,add_sub_config_3,2:GPU,2,Yes,6967.9,0.8 11 | add_sub,2,1,add_sub_config_3,2:GPU,2,Yes,3988.7,0.8 12 | add_sub,1,2,add_sub_config_3,2:GPU,2,Yes,3270.7,0.8 13 | add_sub,1,1,add_sub_config_3,2:GPU,2,Yes,1752.1,0.8 14 | add_sub,1,2,add_sub_config_0,1:GPU,1,Yes,6114.9,0.6 15 | add_sub,1,1,add_sub_config_0,1:GPU,1,Yes,2334.7,0.6 16 | add_sub,1,2,add_sub_config_2,2:GPU,1,Yes,3363.1,0.9 17 | add_sub,1,1,add_sub_config_2,2:GPU,1,Yes,1915.6,0.8 18 | 19 | -------------------------------------------------------------------------------- /tests/common/single-model-ckpt/golden-metrics-server-only.csv: -------------------------------------------------------------------------------- 1 | Model,GPU UUID,GPU Memory Usage (MB),GPU Utilization (%),GPU Power Usage (W) 2 | triton-server,GPU-8557549f-9c89-4384-8bd6-1fd823c342e0,870.0,0.0,56.0 3 | 4 | -------------------------------------------------------------------------------- /tests/common/single-model-ckpt/plot_manager.json: -------------------------------------------------------------------------------- 1 | {"_simple_plots": {"add_sub": {"throughput_v_latency": {"_name": "throughput_v_latency", "_title": "Throughput vs. Latency", "_x_axis": "perf_latency_p99", "_y_axis": "perf_throughput", "_monotonic": true, "_data": {"add_sub_config_default": {"x_data": [0.524, 0.581, 0.658, 0.539], "y_data": [13205.0, 6176.0, 4876.12, 2508.04]}, "add_sub_config_1": {"x_data": [0.596, 0.607, 0.587, 0.599], "y_data": [12528.9, 6109.95, 4804.46, 2185.97]}, "add_sub_config_3": {"x_data": [0.81, 0.803, 0.838, 0.839], "y_data": [6967.92, 3988.65, 3270.69, 1752.08]}}}, "gpu_mem_v_latency": {"_name": "gpu_mem_v_latency", "_title": "GPU Memory vs. Latency", "_x_axis": "perf_latency_p99", "_y_axis": "gpu_used_memory", "_monotonic": false, "_data": {"add_sub_config_default": {"x_data": [0.524, 0.581, 0.658, 0.539], "y_data": [872.415232, 872.415232, 872.415232, 872.415232]}, "add_sub_config_1": {"x_data": [0.596, 0.607, 0.587, 0.599], "y_data": [872.415232, 872.415232, 872.415232, 872.415232]}, "add_sub_config_3": {"x_data": [0.81, 0.803, 0.838, 0.839], "y_data": [874.512384, 874.512384, 874.512384, 874.512384]}}}}}} -------------------------------------------------------------------------------- /tests/common/test_result_collector.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import json 18 | import unittest 19 | 20 | 21 | class TestResultCollector(unittest.TestCase): 22 | """ 23 | TestResultCollector stores test result and prints it to stdout. In order 24 | to use this class, unit tests must inherit this class. Use 25 | `check_test_results` bash function from `common/util.sh` to verify the 26 | expected number of tests produced by this class 27 | """ 28 | 29 | @classmethod 30 | def setResult(cls, total, errors, failures): 31 | cls.total, cls.errors, cls.failures = total, errors, failures 32 | 33 | @classmethod 34 | def tearDownClass(cls): 35 | # this method is called when all the unit tests in a class are 36 | # finished. 37 | json_res = {"total": cls.total, "errors": cls.errors, "failures": cls.failures} 38 | print(json.dumps(json_res)) 39 | 40 | def run(self, result=None): 41 | # result argument stores the accumulative test results 42 | test_result = super().run(result) 43 | if test_result: 44 | total = test_result.testsRun 45 | errors = len(test_result.errors) 46 | failures = len(test_result.failures) 47 | self.setResult(total, errors, failures) 48 | -------------------------------------------------------------------------------- /tests/mocks/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | -------------------------------------------------------------------------------- /tests/mocks/mock_api_error.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | class MockAPIError(Exception): 19 | """ 20 | Mock for Docker API errors 21 | """ 22 | 23 | def __init__(self): 24 | self.explanation = "port is already allocated" 25 | -------------------------------------------------------------------------------- /tests/mocks/mock_base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from abc import ABC, abstractmethod 18 | 19 | 20 | class MockBase(ABC): 21 | """ 22 | Base abstract class for all the mocks 23 | """ 24 | 25 | def __init__(self): 26 | self._patchers = [] 27 | 28 | @abstractmethod 29 | def _fill_patchers(self): 30 | pass 31 | 32 | def start(self): 33 | for patch in self._patchers: 34 | patch.start() 35 | 36 | def stop(self): 37 | for patch in self._patchers: 38 | patch.stop() 39 | -------------------------------------------------------------------------------- /tests/mocks/mock_config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from unittest.mock import MagicMock, mock_open, patch 18 | 19 | from .mock_base import MockBase 20 | 21 | 22 | class MockConfig(MockBase): 23 | def __init__(self, args, yaml_file_content): 24 | self.args = args 25 | self.yaml_file_content = yaml_file_content 26 | super().__init__() 27 | self._fill_patchers() 28 | 29 | def _fill_patchers(self): 30 | patchers = self._patchers 31 | 32 | patchers.append( 33 | patch("builtins.open", mock_open(read_data=self.yaml_file_content)) 34 | ) 35 | patchers.append(patch("sys.argv", self.args)) 36 | patchers.append(patch("numba.cuda.is_available", MagicMock(True))) 37 | -------------------------------------------------------------------------------- /tests/mocks/mock_dcgm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from unittest.mock import MagicMock, patch 18 | 19 | from .mock_base import MockBase 20 | from .mock_dcgm_agent import MockDCGMAgent 21 | from .mock_dcgm_field_group_watcher import MockDCGMFieldGroupWatcherHelper 22 | 23 | 24 | class MockDCGM(MockBase): 25 | """ 26 | Mocks dcgm_agent methods. 27 | """ 28 | 29 | def __init__(self): 30 | super().__init__() 31 | self._fill_patchers() 32 | 33 | def _fill_patchers(self): 34 | patchers = self._patchers 35 | 36 | structs_imports_path = [ 37 | "model_analyzer.monitor.dcgm.dcgm_monitor", 38 | "model_analyzer.device.gpu_device_factory", 39 | ] 40 | for import_path in structs_imports_path: 41 | patchers.append(patch(f"{import_path}.structs._dcgmInit", MagicMock())) 42 | 43 | dcgm_agent_imports_path = [ 44 | "model_analyzer.monitor.dcgm.dcgm_monitor", 45 | "model_analyzer.device.gpu_device_factory", 46 | ] 47 | for import_path in dcgm_agent_imports_path: 48 | patchers.append(patch(f"{import_path}.dcgm_agent", MockDCGMAgent)) 49 | 50 | patchers.append( 51 | patch( 52 | "model_analyzer.monitor.dcgm.dcgm_monitor.dcgm_field_helpers.DcgmFieldGroupWatcher", 53 | MockDCGMFieldGroupWatcherHelper, 54 | ) 55 | ) 56 | -------------------------------------------------------------------------------- /tests/mocks/mock_glob.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from unittest.mock import MagicMock, Mock, patch 18 | 19 | from .mock_base import MockBase 20 | 21 | 22 | class MockGlobMethods(MockBase): 23 | """ 24 | Mocks the methods for the os module 25 | """ 26 | 27 | def __init__(self): 28 | glob_attrs = {"glob": MagicMock(return_value=[])} 29 | self.patcher_glob = patch( 30 | "model_analyzer.state.analyzer_state_manager.glob", Mock(**glob_attrs) 31 | ) 32 | super().__init__() 33 | self._fill_patchers() 34 | 35 | def start(self): 36 | """ 37 | start the patchers 38 | """ 39 | 40 | self.glob_mock = self.patcher_glob.start() 41 | 42 | def _fill_patchers(self): 43 | """ 44 | Fills the patcher list for destruction 45 | """ 46 | 47 | self._patchers.append(self.patcher_glob) 48 | 49 | def set_glob_return_value(self, value): 50 | """ 51 | Sets the return value for the glob.glob call 52 | """ 53 | 54 | self.glob_mock.glob.return_value = value 55 | -------------------------------------------------------------------------------- /tests/mocks/mock_gpu_device_factory.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from unittest.mock import MagicMock, patch 18 | 19 | from .mock_base import MockBase 20 | 21 | 22 | class MockGPUDeviceFactory(MockBase): 23 | def __init__(self, gpus): 24 | self._cuda_visible_gpus = gpus 25 | super().__init__() 26 | self._fill_patchers() 27 | 28 | def _fill_patchers(self): 29 | self._patchers.append( 30 | patch( 31 | "model_analyzer.device.gpu_device_factory.GPUDeviceFactory.get_cuda_visible_gpus", 32 | MagicMock(return_value=dict.fromkeys(self._cuda_visible_gpus)), 33 | ) 34 | ) 35 | -------------------------------------------------------------------------------- /tests/mocks/mock_json.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from unittest.mock import MagicMock, Mock, patch 18 | 19 | from .mock_base import MockBase 20 | 21 | 22 | class MockJSONMethods(MockBase): 23 | """ 24 | Mocks the methods for the os module 25 | """ 26 | 27 | def __init__(self): 28 | json_attrs = {"load": MagicMock(), "dump": MagicMock()} 29 | self.patcher_json = patch( 30 | "model_analyzer.state.analyzer_state_manager.json", Mock(**json_attrs) 31 | ) 32 | super().__init__() 33 | self._fill_patchers() 34 | 35 | def start(self): 36 | """ 37 | start the patchers 38 | """ 39 | 40 | self.json_mock = self.patcher_json.start() 41 | 42 | def _fill_patchers(self): 43 | """ 44 | Fills the patcher list for destruction 45 | """ 46 | 47 | self._patchers.append(self.patcher_json) 48 | 49 | def set_json_load_return_value(self, value): 50 | """ 51 | Sets the return value for json load 52 | """ 53 | 54 | self.json_mock.load.return_value = value 55 | 56 | def set_json_load_side_effect(self, effect): 57 | """ 58 | Sets a side effet 59 | """ 60 | 61 | self.json_mock.load.side_effect = effect 62 | -------------------------------------------------------------------------------- /tests/mocks/mock_matplotlib.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from unittest.mock import MagicMock, Mock, patch 18 | 19 | from .mock_base import MockBase 20 | 21 | 22 | class MockMatplotlibMethods(MockBase): 23 | """ 24 | Mock class that mocks 25 | matplotlib 26 | """ 27 | 28 | def __init__(self): 29 | plt_attrs = {"subplots": Mock(return_value=(MagicMock(), MagicMock()))} 30 | self.patcher_pyplot = patch( 31 | "model_analyzer.plots.simple_plot.plt", Mock(**plt_attrs) 32 | ) 33 | super().__init__() 34 | self._fill_patchers() 35 | 36 | def start(self): 37 | """ 38 | Start mock 39 | """ 40 | 41 | self.pyplot_mock = self.patcher_pyplot.start() 42 | 43 | def _fill_patchers(self): 44 | """ 45 | Add patchers to list 46 | """ 47 | 48 | self._patchers.append(self.patcher_pyplot) 49 | 50 | def assert_called_subplots(self): 51 | """ 52 | Checks for a call to subplots 53 | """ 54 | 55 | self.pyplot_mock.subplots.assert_called() 56 | 57 | def assert_called_plot_with_args(self, x_data, y_data, marker, label): 58 | """ 59 | Checks for call to axes.plot 60 | """ 61 | 62 | self.pyplot_mock.subplots.return_value[1].plot.assert_called_with( 63 | x_data, y_data, marker=marker, label=label 64 | ) 65 | 66 | def assert_called_save_with_args(self, filepath): 67 | """ 68 | Checks for call to figure.savefig 69 | """ 70 | 71 | self.pyplot_mock.subplots.return_value[0].savefig.assert_called_with(filepath) 72 | -------------------------------------------------------------------------------- /tests/mocks/mock_model_config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from unittest.mock import MagicMock, mock_open, patch 18 | 19 | from .mock_base import MockBase 20 | 21 | 22 | class MockModelConfig(MockBase): 23 | def __init__(self, model_file_content=None): 24 | self._model_file_content = model_file_content 25 | super().__init__() 26 | self._fill_patchers() 27 | 28 | def _fill_patchers(self): 29 | patchers = self._patchers 30 | 31 | patchers.append( 32 | patch("builtins.open", mock_open(read_data=self._model_file_content)) 33 | ) 34 | patchers.append( 35 | patch( 36 | "model_analyzer.triton.model.model_config.os.path.exists", 37 | MagicMock(return_value=True), 38 | ) 39 | ) 40 | 41 | def isfile(file_name): 42 | if file_name.endswith(".pbtxt"): 43 | return True 44 | else: 45 | return False 46 | 47 | patchers.append( 48 | patch("model_analyzer.triton.model.model_config.os.path.isfile", isfile) 49 | ) 50 | -------------------------------------------------------------------------------- /tests/mocks/mock_psutil.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from unittest.mock import MagicMock, patch 18 | 19 | from .mock_base import MockBase 20 | 21 | 22 | class MockPSUtil(MockBase): 23 | def __init__(self): 24 | super().__init__() 25 | self._fill_patchers() 26 | 27 | def _fill_patchers(self): 28 | mock_process = MagicMock() 29 | mock_process().cpu_percent.return_value = 5 30 | self._patchers.append( 31 | patch( 32 | "model_analyzer.perf_analyzer.perf_analyzer.psutil.Process", 33 | mock_process, 34 | ) 35 | ) 36 | -------------------------------------------------------------------------------- /tests/mocks/mock_requests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from unittest.mock import Mock, patch 18 | 19 | from .mock_base import MockBase 20 | 21 | 22 | class MockRequests(MockBase): 23 | def __init__(self, mock_paths): 24 | self._mock_paths = mock_paths 25 | response_attrs = {"content": Mock()} 26 | request_attrs = {"get": Mock(**response_attrs)} 27 | self._patchers_requests = {} 28 | self._request_mocks = {} 29 | for path in mock_paths: 30 | self._patchers_requests[path] = patch( 31 | f"{path}.requests", Mock(**request_attrs) 32 | ) 33 | super().__init__() 34 | self._fill_patchers() 35 | 36 | def _fill_patchers(self): 37 | """ 38 | Fills the patcher list for destruction 39 | """ 40 | 41 | for patcher in self._patchers_requests.values(): 42 | self._patchers.append(patcher) 43 | 44 | def start(self): 45 | """ 46 | start the patchers 47 | """ 48 | 49 | for path in self._mock_paths: 50 | self._request_mocks[path] = self._patchers_requests[path].start() 51 | 52 | def set_get_request_response(self, response): 53 | for mock in self._request_mocks.values(): 54 | mock.get.return_value.content = response 55 | -------------------------------------------------------------------------------- /tests/mocks/mock_server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from abc import abstractmethod 18 | 19 | from .mock_base import MockBase 20 | 21 | 22 | class MockServerMethods(MockBase): 23 | """ 24 | Interface for a mock server declaring 25 | the methods it must provide. 26 | """ 27 | 28 | @abstractmethod 29 | def assert_server_process_start_called_with(self, *args, **kwargs): 30 | """ 31 | Asserts that the tritonserver process was started with 32 | the supplied arguments 33 | """ 34 | 35 | @abstractmethod 36 | def assert_server_process_terminate_called(self): 37 | """ 38 | Assert that the server process was stopped 39 | """ 40 | 41 | @abstractmethod 42 | def assert_cpu_stats_called(self): 43 | """ 44 | Assert that correct cpu memory stats 45 | api functions wer called for this implementation 46 | """ 47 | -------------------------------------------------------------------------------- /tests/test_pdf_report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import unittest 18 | from unittest.mock import MagicMock, patch 19 | 20 | from model_analyzer.reports.pdf_report import PDFReport 21 | 22 | from .common import test_result_collector as trc 23 | 24 | 25 | class TestPDFReportMethods(trc.TestResultCollector): 26 | """ 27 | Tests the methods of the PDFReport class 28 | """ 29 | 30 | def setUp(self): 31 | self.maxDiff = None 32 | self.report = PDFReport() 33 | 34 | def tearDown(self): 35 | patch.stopall() 36 | 37 | def test_write_report(self): 38 | with patch( 39 | "model_analyzer.reports.pdf_report.pdfkit", MagicMock() 40 | ) as pdfkit_mock: 41 | self.report.add_title("Test PDF Report") 42 | self.report.add_subheading("Throughput vs. Latency") 43 | test_paragraph = ( 44 | "This is a test paragraph with a lot to say." 45 | " There is more than one line in this paragraph." 46 | ) 47 | self.report.add_paragraph(test_paragraph, font_size=14) 48 | self.report.write_report("test_report_filename") 49 | 50 | expected_report_body = ( 51 | "

Test PDF Report

Throughput vs. Latency

" 52 | f'

{test_paragraph}

' 53 | "" 54 | ) 55 | pdfkit_mock.from_string.assert_called_with( 56 | expected_report_body, "test_report_filename", options={"quiet": ""} 57 | ) 58 | 59 | 60 | if __name__ == "__main__": 61 | unittest.main() 62 | -------------------------------------------------------------------------------- /tests/test_search_dimension.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from model_analyzer.config.generate.search_dimension import SearchDimension 18 | 19 | from .common import test_result_collector as trc 20 | 21 | 22 | class TestSearchDimension(trc.TestResultCollector): 23 | def test_linear(self): 24 | sd = SearchDimension("foo", SearchDimension.DIMENSION_TYPE_LINEAR, 0, 10) 25 | 26 | self.assertEqual("foo", sd.get_name()) 27 | self.assertEqual(0, sd.get_min_idx()) 28 | self.assertEqual(10, sd.get_max_idx()) 29 | self.assertEqual(1, sd.get_value_at_idx(0)) 30 | self.assertEqual(2, sd.get_value_at_idx(1)) 31 | self.assertEqual(3, sd.get_value_at_idx(2)) 32 | 33 | def test_exponential(self): 34 | sd = SearchDimension("foo", SearchDimension.DIMENSION_TYPE_EXPONENTIAL) 35 | self.assertEqual(1, sd.get_value_at_idx(0)) 36 | self.assertEqual(2, sd.get_value_at_idx(1)) 37 | self.assertEqual(4, sd.get_value_at_idx(2)) 38 | self.assertEqual(8, sd.get_value_at_idx(3)) 39 | 40 | def test_out_of_bounds(self): 41 | sd = SearchDimension("foo", SearchDimension.DIMENSION_TYPE_LINEAR, 2, 10) 42 | 43 | with self.assertRaises(IndexError): 44 | sd.get_value_at_idx(1) 45 | 46 | with self.assertRaises(IndexError): 47 | sd.get_value_at_idx(11) 48 | 49 | def test_no_max(self): 50 | sd = SearchDimension("foo", SearchDimension.DIMENSION_TYPE_LINEAR) 51 | 52 | # Confirm no assert 53 | sd.get_value_at_idx(100000000) 54 | --------------------------------------------------------------------------------