├── .gitignore ├── LICENSE ├── README.md ├── figures └── quantized_values.png ├── neural_compressor ├── __init__.py ├── adaptor │ ├── __init__.py │ ├── adaptor.py │ ├── keras.py │ ├── keras.yaml │ ├── keras_utils │ │ ├── __init__.py │ │ ├── conv2d.py │ │ ├── dense.py │ │ ├── depthwise_conv2d.py │ │ ├── pool2d.py │ │ ├── quantizer.py │ │ └── separable_conv2d.py │ ├── mxnet.py │ ├── mxnet.yaml │ ├── mxnet_utils │ │ ├── __init__.py │ │ └── util.py │ ├── onnxrt.py │ ├── onnxrt.yaml │ ├── onnxrt_cuda.yaml │ ├── onnxrt_dml.yaml │ ├── onnxrt_dnnl.yaml │ ├── onnxrt_trt.yaml │ ├── ox_utils │ │ ├── __init__.py │ │ ├── calibration.py │ │ ├── calibrator.py │ │ ├── operators │ │ │ ├── __init__.py │ │ │ ├── activation.py │ │ │ ├── argmax.py │ │ │ ├── attention.py │ │ │ ├── binary_op.py │ │ │ ├── concat.py │ │ │ ├── conv.py │ │ │ ├── direct_q8.py │ │ │ ├── embed_layernorm.py │ │ │ ├── gather.py │ │ │ ├── gavgpool.py │ │ │ ├── gemm.py │ │ │ ├── lstm.py │ │ │ ├── matmul.py │ │ │ ├── maxpool.py │ │ │ ├── norm.py │ │ │ ├── ops.py │ │ │ ├── pad.py │ │ │ ├── pooling.py │ │ │ ├── reduce.py │ │ │ ├── resize.py │ │ │ ├── split.py │ │ │ └── unary_op.py │ │ ├── quantizer.py │ │ ├── smooth_quant.py │ │ ├── util.py │ │ └── weight_only.py │ ├── pytorch.py │ ├── pytorch_cpu.yaml │ ├── pytorch_gpu.yaml │ ├── pytorch_ipex.yaml │ ├── query.py │ ├── tensorflow.py │ ├── tensorflow.yaml │ ├── tensorflow_itex.yaml │ ├── tf_utils │ │ ├── __init__.py │ │ ├── graph_converter.py │ │ ├── graph_converter_without_calib.py │ │ ├── graph_rewriter │ │ │ ├── __init__.py │ │ │ ├── bf16 │ │ │ │ ├── __init__.py │ │ │ │ ├── bf16_convert.py │ │ │ │ └── dequantize_cast_optimizer.py │ │ │ ├── generic │ │ │ │ ├── __init__.py │ │ │ │ ├── convert_add_to_biasadd.py │ │ │ │ ├── convert_layout.py │ │ │ │ ├── convert_leakyrelu.py │ │ │ │ ├── convert_nan_to_random.py │ │ │ │ ├── convert_placeholder_to_const.py │ │ │ │ ├── dilated_contraction.py │ │ │ │ ├── dummy_biasadd.py │ │ │ │ ├── expanddims_optimizer.py │ │ │ │ ├── fetch_weight_from_reshape.py │ │ │ │ ├── fold_batch_norm.py │ │ │ │ ├── fold_constant.py │ │ │ │ ├── fuse_biasadd_add.py │ │ │ │ ├── fuse_column_wise_mul.py │ │ │ │ ├── fuse_conv_with_math.py │ │ │ │ ├── fuse_decomposed_bn.py │ │ │ │ ├── fuse_decomposed_in.py │ │ │ │ ├── fuse_gelu.py │ │ │ │ ├── fuse_layer_norm.py │ │ │ │ ├── fuse_pad_with_conv.py │ │ │ │ ├── fuse_pad_with_fp32_conv.py │ │ │ │ ├── fuse_reshape_transpose.py │ │ │ │ ├── graph_cse_optimizer.py │ │ │ │ ├── grappler_pass.py │ │ │ │ ├── insert_print_node.py │ │ │ │ ├── move_squeeze_after_relu.py │ │ │ │ ├── pre_optimize.py │ │ │ │ ├── remove_training_nodes.py │ │ │ │ ├── rename_batch_norm.py │ │ │ │ ├── split_shared_input.py │ │ │ │ ├── strip_equivalent_nodes.py │ │ │ │ ├── strip_unused_nodes.py │ │ │ │ └── switch_optimizer.py │ │ │ ├── graph_base.py │ │ │ ├── int8 │ │ │ │ ├── __init__.py │ │ │ │ ├── freeze_fake_quant.py │ │ │ │ ├── freeze_value.py │ │ │ │ ├── freeze_value_without_calib.py │ │ │ │ ├── fuse_conv_redundant_dequantize.py │ │ │ │ ├── fuse_conv_requantize.py │ │ │ │ ├── fuse_matmul_redundant_dequantize.py │ │ │ │ ├── fuse_matmul_requantize.py │ │ │ │ ├── meta_op_optimizer.py │ │ │ │ ├── post_hostconst_converter.py │ │ │ │ ├── post_quantized_op_cse.py │ │ │ │ ├── rnn_convert.py │ │ │ │ └── scale_propagation.py │ │ │ ├── onnx │ │ │ │ ├── __init__.py │ │ │ │ ├── onnx_graph.py │ │ │ │ ├── onnx_node.py │ │ │ │ ├── onnx_schema.py │ │ │ │ └── tf2onnx_utils.py │ │ │ └── qdq │ │ │ │ ├── __init__.py │ │ │ │ ├── insert_qdq_pattern.py │ │ │ │ ├── merge_duplicated_qdq.py │ │ │ │ └── share_qdq_y_pattern.py │ │ ├── graph_util.py │ │ ├── quantize_graph │ │ │ ├── __init__.py │ │ │ ├── qat │ │ │ │ ├── __init__.py │ │ │ │ ├── fake_quantize.py │ │ │ │ ├── quantize_config.py │ │ │ │ ├── quantize_helper.py │ │ │ │ ├── quantize_layers │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── optimize_layer.py │ │ │ │ │ ├── quantize_layer_add.py │ │ │ │ │ ├── quantize_layer_base.py │ │ │ │ │ └── quantize_layer_bn.py │ │ │ │ └── quantize_wrapper.py │ │ │ ├── qdq │ │ │ │ ├── __init__.py │ │ │ │ ├── fuse_qdq_bn.py │ │ │ │ ├── fuse_qdq_concatv2.py │ │ │ │ ├── fuse_qdq_conv.py │ │ │ │ ├── fuse_qdq_deconv.py │ │ │ │ ├── fuse_qdq_in.py │ │ │ │ ├── fuse_qdq_matmul.py │ │ │ │ ├── fuse_qdq_pooling.py │ │ │ │ └── optimize_qdq.py │ │ │ ├── quantize_graph_base.py │ │ │ ├── quantize_graph_bn.py │ │ │ ├── quantize_graph_concatv2.py │ │ │ ├── quantize_graph_conv.py │ │ │ ├── quantize_graph_for_intel_cpu.py │ │ │ ├── quantize_graph_matmul.py │ │ │ └── quantize_graph_pooling.py │ │ ├── quantize_graph_common.py │ │ ├── smooth_quant_calibration.py │ │ ├── smooth_quant_scaler.py │ │ ├── tf2onnx_converter.py │ │ ├── transform_graph │ │ │ ├── __init__.py │ │ │ ├── bias_correction.py │ │ │ ├── graph_transform_base.py │ │ │ ├── insert_logging.py │ │ │ └── rerange_quantized_concat.py │ │ └── util.py │ └── torch_utils │ │ ├── __init__.py │ │ ├── autoround │ │ ├── __init__.py │ │ ├── autoround.py │ │ └── sign_sgd.py │ │ ├── awq.py │ │ ├── bf16_convert.py │ │ ├── gptq.py │ │ ├── hawq_metric.py │ │ ├── layer_wise_quant │ │ ├── __init__.py │ │ ├── modified_pickle.py │ │ ├── quantize.py │ │ ├── torch_load.py │ │ └── utils.py │ │ ├── mixed_precision.py │ │ ├── model_wrapper.py │ │ ├── pattern_detector.py │ │ ├── residual_utils.py │ │ ├── smooth_quant.py │ │ ├── symbolic_trace.py │ │ ├── teq.py │ │ ├── util.py │ │ └── weight_only.py ├── algorithm │ ├── __init__.py │ ├── algorithm.py │ ├── fast_bias_correction.py │ ├── smooth_quant.py │ └── weight_correction.py ├── benchmark.py ├── common │ ├── __init__.py │ ├── base_config.py │ ├── base_tune.py │ ├── logger.py │ └── utility.py ├── compression │ ├── __init__.py │ ├── callbacks.py │ ├── distillation │ │ ├── __init__.py │ │ ├── criterions.py │ │ ├── optimizers.py │ │ └── utility.py │ ├── hpo │ │ ├── __init__.py │ │ ├── sa_optimizer.py │ │ ├── search_algorithms.py │ │ └── search_space.py │ └── pruner │ │ ├── README.md │ │ ├── __init__.py │ │ ├── criteria.py │ │ ├── model_slim │ │ ├── README.md │ │ ├── __init__.py │ │ ├── auto_slim.py │ │ ├── imgs │ │ │ └── auto_slim_feed_forward_network.png │ │ ├── pattern_analyzer.py │ │ └── weight_slim.py │ │ ├── patterns │ │ ├── __init__.py │ │ ├── base.py │ │ ├── mha.py │ │ ├── ninm.py │ │ └── nxm.py │ │ ├── pruners │ │ ├── __init__.py │ │ ├── base.py │ │ ├── basic.py │ │ ├── block_mask.py │ │ ├── mha.py │ │ ├── pattern_lock.py │ │ ├── progressive.py │ │ ├── retrain_free.py │ │ └── sparse_gpt.py │ │ ├── pruning.py │ │ ├── regs.py │ │ ├── schedulers.py │ │ ├── tf_criteria.py │ │ └── utils.py ├── conf │ ├── README.md │ ├── __init__.py │ ├── config.py │ ├── dotdict.py │ └── pythonic_config.py ├── config.py ├── contrib │ ├── __init__.py │ └── strategy │ │ ├── __init__.py │ │ ├── sigopt.py │ │ └── tpe.py ├── data │ ├── __init__.py │ ├── dataloaders │ │ ├── __init__.py │ │ ├── base_dataloader.py │ │ ├── dataloader.py │ │ ├── default_dataloader.py │ │ ├── fetcher.py │ │ ├── mxnet_dataloader.py │ │ ├── onnxrt_dataloader.py │ │ ├── pytorch_dataloader.py │ │ ├── sampler.py │ │ └── tensorflow_dataloader.py │ ├── datasets │ │ ├── __init__.py │ │ ├── bert_dataset.py │ │ ├── coco_dataset.py │ │ ├── dataset.py │ │ ├── dummy_dataset.py │ │ ├── dummy_dataset_v2.py │ │ ├── imagenet_dataset.py │ │ └── style_transfer_dataset.py │ ├── filters │ │ ├── __init__.py │ │ ├── coco_filter.py │ │ └── filter.py │ └── transforms │ │ ├── __init__.py │ │ ├── coco_transform.py │ │ ├── imagenet_transform.py │ │ ├── postprocess.py │ │ ├── tokenization.py │ │ └── transform.py ├── experimental │ ├── __init__.py │ ├── benchmark.py │ ├── common │ │ ├── __init__.py │ │ ├── criterion.py │ │ ├── dataloader.py │ │ ├── metric.py │ │ ├── model.py │ │ ├── optimizer.py │ │ ├── postprocess.py │ │ └── torch_utils.py │ ├── component.py │ ├── compression │ │ └── __init__.py │ ├── contrib │ │ ├── __init__.py │ │ └── strategy │ │ │ ├── __init__.py │ │ │ ├── sigopt.py │ │ │ └── tpe.py │ ├── data │ │ ├── __init__.py │ │ ├── dataloaders │ │ │ ├── __init__.py │ │ │ ├── base_dataloader.py │ │ │ ├── dataloader.py │ │ │ ├── default_dataloader.py │ │ │ ├── fetcher.py │ │ │ ├── mxnet_dataloader.py │ │ │ ├── onnxrt_dataloader.py │ │ │ ├── pytorch_dataloader.py │ │ │ ├── sampler.py │ │ │ └── tensorflow_dataloader.py │ │ ├── datasets │ │ │ ├── __init__.py │ │ │ ├── bert_dataset.py │ │ │ ├── coco_dataset.py │ │ │ ├── dataset.py │ │ │ ├── dummy_dataset.py │ │ │ ├── dummy_dataset_v2.py │ │ │ ├── imagenet_dataset.py │ │ │ └── style_transfer_dataset.py │ │ ├── filters │ │ │ ├── __init__.py │ │ │ ├── coco_filter.py │ │ │ └── filter.py │ │ └── transforms │ │ │ ├── __init__.py │ │ │ ├── imagenet_transform.py │ │ │ ├── tokenization.py │ │ │ └── transform.py │ ├── distillation.py │ ├── export │ │ ├── __init__.py │ │ ├── qlinear2qdq.py │ │ ├── tf2onnx.py │ │ └── torch2onnx.py │ ├── graph_optimization.py │ ├── metric │ │ ├── __init__.py │ │ ├── bleu.py │ │ ├── bleu_util.py │ │ ├── coco_label_map.py │ │ ├── coco_tools.py │ │ ├── evaluate_squad.py │ │ ├── f1.py │ │ └── metric.py │ ├── mixed_precision.py │ ├── model_conversion.py │ ├── nas │ │ ├── __init__.py │ │ ├── basic_nas.py │ │ ├── dynas.py │ │ ├── nas.py │ │ ├── nas_utils.py │ │ └── search_algorithms.py │ ├── pruner_legacy │ │ ├── __init__.py │ │ ├── gradient_sensitivity.py │ │ ├── group_lasso.py │ │ ├── magnitude.py │ │ ├── pattern_lock.py │ │ ├── pruner.py │ │ └── util │ │ │ └── block_mask.py │ ├── pruning.py │ ├── pruning_recipes │ │ ├── __init__.py │ │ └── patterns │ │ │ ├── __init__.py │ │ │ ├── pattern.py │ │ │ └── tile_pattern.py │ ├── pruning_v2.py │ ├── pytorch_pruner │ │ ├── __init__.py │ │ ├── logger.py │ │ ├── patterns.py │ │ ├── prune_utils.py │ │ ├── pruner.py │ │ ├── pruning.py │ │ └── scheduler.py │ ├── quantization.py │ ├── scheduler.py │ └── strategy │ │ ├── __init__.py │ │ ├── auto_mixed_precision.py │ │ ├── basic.py │ │ ├── bayesian.py │ │ ├── exhaustive.py │ │ ├── mse.py │ │ ├── mse_v2.py │ │ ├── random.py │ │ ├── strategy.py │ │ └── utils │ │ ├── __init__.py │ │ ├── constant.py │ │ ├── tuning_sampler.py │ │ ├── tuning_space.py │ │ ├── tuning_structs.py │ │ └── utility.py ├── metric │ ├── __init__.py │ ├── bleu.py │ ├── bleu_util.py │ ├── coco_label_map.py │ ├── coco_tools.py │ ├── evaluate_squad.py │ ├── f1.py │ └── metric.py ├── mix_precision.py ├── model │ ├── __init__.py │ ├── base_model.py │ ├── keras_model.py │ ├── model.py │ ├── mxnet_model.py │ ├── nets_factory.py │ ├── onnx_model.py │ ├── tensorflow_model.py │ └── torch_model.py ├── objective.py ├── profiling │ ├── __init__.py │ ├── parser │ │ ├── __init__.py │ │ ├── factory.py │ │ ├── onnx_parser │ │ │ ├── __init__.py │ │ │ ├── factory.py │ │ │ └── parser.py │ │ ├── parser.py │ │ ├── result.py │ │ └── tensorflow_parser │ │ │ ├── __init__.py │ │ │ ├── factory.py │ │ │ └── parser.py │ └── profiler │ │ ├── __init__.py │ │ ├── factory.py │ │ ├── onnxrt_profiler │ │ ├── __init__.py │ │ ├── factory.py │ │ ├── profiler.py │ │ └── utils.py │ │ ├── profiler.py │ │ └── tensorflow_profiler │ │ ├── __init__.py │ │ ├── factory.py │ │ ├── profiler.py │ │ └── utils.py ├── quantization.py ├── strategy │ ├── __init__.py │ ├── auto.py │ ├── auto_mixed_precision.py │ ├── basic.py │ ├── bayesian.py │ ├── conservative.py │ ├── exhaustive.py │ ├── hawq_v2.py │ ├── mse.py │ ├── mse_v2.py │ ├── random.py │ ├── strategy.py │ └── utils │ │ ├── __init__.py │ │ ├── constant.py │ │ ├── tuning_sampler.py │ │ ├── tuning_space.py │ │ ├── tuning_structs.py │ │ └── utility.py ├── template │ ├── __init__.py │ ├── api_doc_example.py │ ├── graph_optimization.yaml │ ├── pruning.yaml │ ├── ptq.yaml │ └── qat.yaml ├── tensorflow │ ├── __init__.py │ ├── algorithms │ │ ├── __init__.py │ │ └── static_quantize │ │ │ ├── __init__.py │ │ │ ├── keras.py │ │ │ ├── keras.yaml │ │ │ ├── keras_utils │ │ │ ├── __init__.py │ │ │ ├── conv2d.py │ │ │ ├── dense.py │ │ │ ├── depthwise_conv2d.py │ │ │ ├── pool2d.py │ │ │ ├── quantizer.py │ │ │ └── separable_conv2d.py │ │ │ └── quantize_entry.py │ ├── quantization │ │ ├── __init__.py │ │ ├── config.py │ │ └── quantize.py │ └── utils.py ├── torch │ ├── __init__.py │ ├── algorithms │ │ ├── __init__.py │ │ ├── weight_only │ │ │ ├── __init__.py │ │ │ ├── gptq.py │ │ │ └── rtn.py │ │ └── weight_only_algos.py │ ├── amp │ │ ├── __init__.py │ │ ├── autocast.py │ │ └── fp8 │ │ │ ├── __init__.py │ │ │ └── functions.py │ ├── quantization │ │ ├── __init__.py │ │ ├── config.py │ │ ├── fp8 │ │ │ ├── __init__.py │ │ │ ├── modules.py │ │ │ ├── observer.py │ │ │ └── quantization_impl.py │ │ ├── layers.py │ │ └── quantize.py │ ├── tune.py │ └── utils │ │ ├── __init__.py │ │ ├── constants.py │ │ └── utility.py ├── training.py ├── utils │ ├── __init__.py │ ├── collect_layer_histogram.py │ ├── constant.py │ ├── create_obj_from_config.py │ ├── kl_divergence.py │ ├── load_huggingface.py │ ├── logger.py │ ├── neural_insights_utils.py │ ├── options.py │ ├── pytorch.py │ ├── utility.py │ └── weights_details.py └── version.py ├── parse_results.py ├── requirements.yaml ├── run_quant.py └── run_quant_slurm.sh /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .vscode 3 | .idea 4 | /venv/ 5 | */__pycache__ 6 | .ipynb_checkpoints/ 7 | *.snapshot 8 | *.csv 9 | *.pb 10 | *.ckpt 11 | *.log 12 | *.swp 13 | *.onnx 14 | *.so 15 | *.egg-info/ 16 | tags 17 | build/ 18 | _build 19 | lpot_workspace/ 20 | .torch/ 21 | node_modules 22 | build_tmp 23 | saved_results/ 24 | nc_workspace/ 25 | results/ -------------------------------------------------------------------------------- /figures/quantized_values.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cornell-zhang/llm-datatypes/b0f1a17190aae75a8badb37ce679652b74a28fca/figures/quantized_values.png -------------------------------------------------------------------------------- /neural_compressor/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Intel® Neural Compressor: An open-source Python library supporting popular model compression techniques.""" 18 | from .version import __version__ 19 | 20 | # we need to set a global 'NA' backend, or Model can't be used 21 | from .config import ( 22 | DistillationConfig, 23 | PostTrainingQuantConfig, 24 | WeightPruningConfig, 25 | QuantizationAwareTrainingConfig, 26 | MixedPrecisionConfig, 27 | ) 28 | from .contrib import * 29 | from .model import * 30 | from .metric import * 31 | from .utils import options 32 | from .utils.utility import set_random_seed, set_tensorboard, set_workspace, set_resume_from 33 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from .adaptor import FRAMEWORKS 19 | from os.path import dirname, basename, isfile, join 20 | import glob 21 | 22 | modules = glob.glob(join(dirname(__file__), "*.py")) 23 | 24 | for f in modules: 25 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 26 | __import__(basename(f)[:-3], globals(), locals(), level=1) 27 | 28 | __all__ = ["FRAMEWORKS"] 29 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/keras_utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/keras_utils/pool2d.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import json 19 | 20 | import tensorflow as tf 21 | from tensorflow import quantization 22 | from tensorflow.keras import activations, backend, constraints, initializers, regularizers 23 | from tensorflow.keras.layers import AveragePooling2D, MaxPooling2D 24 | 25 | 26 | class QAvgPool2D(AveragePooling2D): 27 | def __init__( 28 | self, 29 | pool_size=(2, 2), 30 | strides=None, 31 | padding="valid", 32 | data_format=None, 33 | min_value=-10000, 34 | max_value=10000, 35 | **kwargs 36 | ): 37 | super(QAvgPool2D, self).__init__( 38 | pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, **kwargs 39 | ) 40 | self.min_value = json.loads(min_value) 41 | self.max_value = json.loads(max_value) 42 | 43 | 44 | class QMaxPool2D(MaxPooling2D): 45 | def __init__( 46 | self, 47 | pool_size=(2, 2), 48 | strides=None, 49 | padding="valid", 50 | data_format=None, 51 | min_value=-10000, 52 | max_value=10000, 53 | **kwargs 54 | ): 55 | super(QMaxPool2D, self).__init__( 56 | pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, **kwargs 57 | ) 58 | self.min_value = json.loads(min_value) 59 | self.max_value = json.loads(max_value) 60 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/mxnet_utils/__init__.py: -------------------------------------------------------------------------------- 1 | """Mxnet util init.""" 2 | #!/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2021 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/onnxrt_dml.yaml: -------------------------------------------------------------------------------- 1 | ## Copyright (c) 2021 Intel Corporation 2 | ## 3 | ## Licensed under the Apache License, Version 2.0 (the "License"); 4 | ## you may not use this file except in compliance with the License. 5 | ## You may obtain a copy of the License at 6 | ## 7 | ## http://www.apache.org/licenses/LICENSE-2.0 8 | ## 9 | ## Unless required by applicable law or agreed to in writing, software 10 | ## distributed under the License is distributed on an "AS IS" BASIS, 11 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | ## See the License for the specific language governing permissions and 13 | ## limitations under the License. 14 | ## 15 | # 16 | 17 | - 18 | version: 19 | name: '1.13.1' 20 | int8: &ref_1_13 { 21 | 'static': &ref_1_13_static { 22 | 'Conv': &default_static_qlinear_qdq { 23 | 'weight': &int8_sym_pertensor_minmax { 24 | 'dtype': ['int8'], 25 | 'scheme': ['sym'], 26 | 'granularity': ['per_tensor'], 27 | 'algorithm': ['minmax'] 28 | }, 29 | 'activation': &uint8_asym_pertensor_minmax { 30 | 'dtype': ['uint8'], 31 | 'scheme': ['asym'], 32 | 'granularity': ['per_tensor'], 33 | 'algorithm': ['minmax'] 34 | }, 35 | 'mode': ['QDQ', 'QLinear'] 36 | }, 37 | 'MatMul': { 38 | 'weight': *int8_sym_pertensor_minmax, 39 | 'activation': *uint8_asym_pertensor_minmax, 40 | 'mode': ['QDQ', 'QLinear'] 41 | }, 42 | 'Mul': &default_static_qlinear { 43 | 'weight': *int8_sym_pertensor_minmax, 44 | 'activation': *uint8_asym_pertensor_minmax, 45 | 'mode': ['QLinear'] 46 | }, 47 | 'Relu': *default_static_qlinear_qdq, 48 | 'Clip': *default_static_qlinear_qdq, 49 | 'MaxPool': *default_static_qlinear_qdq, 50 | 'Add': *default_static_qlinear, 51 | }, 52 | } 53 | fp16: &common_fp16 ['Add', 'GlobalAveragePool', 'AveragePool', 'SpaceToDepth', 'Sigmoid', 'Mul', 54 | 'Softmax', 'Gemm', 'MatMul', 'Conv', 'Concat', 'Upsample', 'Pow', 'Sqrt', 'DepthToSpace', 55 | 'Clip', 'BatchNormalization', 'Transpose', 'Softmax', 'AveragePool', 'Squeeze', 'MaxPool', 56 | 'Relu', 'Concat'] 57 | 58 | recipes: &default_optimization 59 | graph_optimization: # from onnxruntime graph_optimization_level 60 | level: ['DISABLE_ALL', 'ENABLE_BASIC', 'ENABLE_EXTENDED', 'ENABLE_ALL'] 61 | 62 | - 63 | version: 64 | name: 'default' 65 | int8: *ref_1_13 66 | recipes: 67 | <<: *default_optimization 68 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/ox_utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Init for ox_utils.""" 18 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/ox_utils/operators/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Operators for onnx model.""" 18 | 19 | from os.path import dirname, basename, isfile, join 20 | import glob 21 | from .ops import OPERATORS, QOPERATORS 22 | 23 | modules = glob.glob(join(dirname(__file__), "*.py")) 24 | 25 | for f in modules: 26 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 27 | __import__(basename(f)[:-3], globals(), locals(), level=1) 28 | 29 | __all__ = ["OPERATORS", "QOPERATORS"] 30 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/ox_utils/operators/argmax.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """ArgMax operator.""" 18 | 19 | from neural_compressor.adaptor.ox_utils.operators.ops import Operator, QOperator, op_registry, qop_registry 20 | 21 | 22 | @op_registry(op_types="ArgMax") 23 | class ArgMaxOperator(Operator): 24 | """ArgMax operator.""" 25 | 26 | def __init__(self, onnx_quantizer, onnx_node): 27 | """Initialization.""" 28 | super(ArgMaxOperator, self).__init__(onnx_quantizer, onnx_node) 29 | 30 | def convert_check(self, convert_format): 31 | """Check if conversion can be done.""" 32 | node = self.node 33 | assert convert_format in ["static"], "convert format for {} should be in ['static']".format(node.op_type) 34 | return True 35 | 36 | def convert(self, convert_format): 37 | """Convert to quantized format.""" 38 | node = self.node 39 | origin_name = node.input[0].split("_argmax_node")[0] 40 | 41 | if origin_name in self.quantizer.quantized_value_map: 42 | node.name = node.name + "_quant" 43 | 44 | 45 | @qop_registry(op_types="ArgMax") 46 | class QArgMaxOperator(QOperator): 47 | """INT8 ArgMax operator.""" 48 | 49 | def __init__(self, onnx_node, children, initializers): 50 | """Initialization.""" 51 | super().__init__(onnx_node, children, initializers) 52 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/ox_utils/operators/norm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Normalization Operator.""" 18 | 19 | import onnx 20 | 21 | from neural_compressor.adaptor.ox_utils.operators.ops import Operator, QOperator, op_registry, qop_registry 22 | from neural_compressor.adaptor.ox_utils.util import attribute_to_kwarg, ms_domain 23 | 24 | 25 | @op_registry(op_types="BatchNormalization") 26 | class BatchNormalizationOperator(Operator): 27 | """BatchNormalization Operator.""" 28 | 29 | def __init__(self, onnx_quantizer, onnx_node): 30 | """Initialization.""" 31 | super(BatchNormalizationOperator, self).__init__(onnx_quantizer, onnx_node) 32 | 33 | def cast(self): 34 | """Cast node.""" 35 | if self.dtype == "bf16": 36 | self.quantizer.cast_inputs(self.node, self.dtype, [0]) 37 | else: 38 | self.quantizer.cast_inputs(self.node, self.dtype) 39 | self.quantizer.cast_outputs(self.node, self.dtype) 40 | 41 | 42 | @op_registry(op_types="LayerNormalization") 43 | class NormalizationOperator(Operator): 44 | """Normalization Operator.""" 45 | 46 | def __init__(self, onnx_quantizer, onnx_node): 47 | """Initialization.""" 48 | super(NormalizationOperator, self).__init__(onnx_quantizer, onnx_node) 49 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/query.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | import logging 18 | from abc import abstractmethod 19 | 20 | 21 | class QueryBackendCapability: 22 | """Base class that defines Query Interface. 23 | 24 | Each adaption layer should implement the inherited class for specific backend on their own. 25 | """ 26 | 27 | def __init__(self): 28 | pass 29 | 30 | @abstractmethod 31 | def get_version(self): 32 | """Get the current backend's version string.""" 33 | raise NotImplementedError 34 | 35 | @abstractmethod 36 | def get_precisions(self): 37 | """Get the supported low precisions, e.g ['int8', 'bf16']""" 38 | raise NotImplementedError 39 | 40 | @abstractmethod 41 | def get_op_types(self): 42 | """Get the op types for specific backend per low precision. 43 | 44 | e.g {'2.3.0': {'int8': ['Conv2D', 'MatMuL']}} 45 | """ 46 | raise NotImplementedError 47 | 48 | @abstractmethod 49 | def get_fuse_patterns(self): 50 | """Get the fusion patterns for specified op type for every specific precision.""" 51 | raise NotImplementedError 52 | 53 | @abstractmethod 54 | def set_quantization_config(self, q_config): 55 | """Set the quantization config to backend. 56 | 57 | Args: 58 | q_config (yaml content?): set the organized quantization configuration to backend. 59 | """ 60 | raise NotImplementedError 61 | 62 | @abstractmethod 63 | def get_quantization_capability(self): 64 | """Get the quantization capability of low precision op types. 65 | 66 | e.g, granularity, scheme and etc. 67 | """ 68 | raise NotImplementedError 69 | 70 | @abstractmethod 71 | def get_mixed_precision_combination(self, unsupported_precisions): 72 | """Get the valid precision combination base on hardware and user' config. 73 | 74 | e.g['fp32', 'bf16', 'int8'] 75 | """ 76 | raise NotImplementedError 77 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | """Tensorflow Adaptor Utils.""" 19 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tensorflow Graph Rewriters.""" 18 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/bf16/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tensorflow BF16 Graph Rewriter.""" 18 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/generic/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tensorflow Generic Graph Rewriters.""" 18 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/generic/convert_nan_to_random.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Convert NAN to random Graph Rewriter.""" 18 | 19 | import numpy as np 20 | from tensorflow.core.framework import attr_value_pb2 21 | from tensorflow.python.framework import dtypes, tensor_util 22 | 23 | from neural_compressor.adaptor.tf_utils.graph_util import GraphAnalyzer 24 | 25 | from ..graph_base import GraphRewriterBase 26 | 27 | 28 | class ConvertNanToRandom(GraphRewriterBase): 29 | """Convert Const node which value consists of NAN to random data.""" 30 | 31 | def do_transformation(self): 32 | """Execute convert NAN to random.""" 33 | cur_graph = GraphAnalyzer() 34 | cur_graph.graph = self.model 35 | 36 | graph_info = cur_graph.parse_graph() 37 | 38 | target_nodes = cur_graph.query_fusion_pattern_nodes([["Const"]]) 39 | 40 | for i in target_nodes: 41 | const_node = graph_info[i[0]].node 42 | const_content = tensor_util.MakeNdarray(const_node.attr["value"].tensor) 43 | if const_content.dtype == np.float32 and np.any(np.isnan(const_content)): 44 | const_node.attr["value"].CopyFrom( 45 | attr_value_pb2.AttrValue( 46 | tensor=tensor_util.make_tensor_proto( 47 | np.random.rand(*const_content.shape), dtypes.float32, const_content.shape 48 | ) 49 | ) 50 | ) 51 | 52 | return cur_graph.dump_graph() 53 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/generic/rename_batch_norm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Rename FusedBatchNorm op to FusedBatchNormV2 Graph Rewriter.""" 18 | 19 | import math 20 | 21 | import numpy as np 22 | from tensorflow.core.framework import attr_value_pb2, node_def_pb2 23 | from tensorflow.python.framework import tensor_util 24 | 25 | from neural_compressor.adaptor.tf_utils.graph_util import GraphAnalyzer 26 | from neural_compressor.adaptor.tf_utils.graph_util import GraphRewriterHelper as Helper 27 | from neural_compressor.utils.utility import dump_elapsed_time 28 | 29 | from ..graph_base import GraphRewriterBase 30 | 31 | 32 | class RenameBatchNormOptimizer(GraphRewriterBase): 33 | """Rename FusedBatchNorm op to FusedBatchNormV2.""" 34 | 35 | @dump_elapsed_time("Pass RenameBatchNormOptimizer") 36 | def do_transformation(self): 37 | """Rename FusedBatchNorm op to FusedBatchNormV2. 38 | 39 | This pass is needed for bf16 conversion. Due to TensorFlow historical reason, 40 | FusedBatchNorm is not a bf16 op but FusedBatchNormV2 is. As the latter is compatible 41 | with the former, changing FusedBatchNorm op to FusedBatchNormV2 op will be able to 42 | convert to bf16 op on the platforms supporting VNNI_BF16 and AMX instructions. 43 | 44 | Returns: 45 | Modified graph with BN ops renamed. 46 | 47 | Raises: 48 | ValueError: If the graph is badly formed. 49 | """ 50 | cur_graph = GraphAnalyzer() 51 | cur_graph.graph = self.model 52 | graph_details = cur_graph.parse_graph() 53 | 54 | for _, v in graph_details.items(): 55 | # for node in cur_graph.graph.node: 56 | if v.node.op == "FusedBatchNorm" or v.node.op == "FusedBatchNormV2": 57 | v.node.op = "FusedBatchNormV3" 58 | v.node.attr["U"].CopyFrom(v.node.attr["T"]) 59 | 60 | return cur_graph.dump_graph() 61 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/generic/split_shared_input.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Split shared input Graph Rewriter.""" 18 | 19 | from tensorflow.core.framework import node_def_pb2 20 | 21 | from neural_compressor.adaptor.tf_utils.graph_util import GraphAnalyzer 22 | from neural_compressor.adaptor.tf_utils.graph_util import GraphRewriterHelper as Helper 23 | from neural_compressor.utils.utility import dump_elapsed_time 24 | 25 | from ..graph_base import GraphRewriterBase 26 | 27 | 28 | class SplitSharedInputOptimizer(GraphRewriterBase): 29 | """Split the shared input if the input node is shared and const.""" 30 | 31 | @dump_elapsed_time("Pass SplitSharedInputOptimizer") 32 | def do_transformation(self): 33 | """Execute splitting the shared input.""" 34 | cur_graph = GraphAnalyzer() 35 | cur_graph.graph = self.model 36 | 37 | graph_info = cur_graph.parse_graph() 38 | 39 | is_shared_input = False 40 | # map of: input_name - op_name 41 | input_map = {} 42 | for node_name in list(graph_info.keys()): 43 | node = graph_info[node_name].node 44 | for _, input_node_name in enumerate(node.input): 45 | if input_node_name.startswith("^"): 46 | continue 47 | if graph_info[Helper.node_name_from_input(input_node_name)].node.op == "Const": 48 | # is shared and current node is not the first one 49 | # sharing the input 50 | if input_node_name in input_map: 51 | is_shared_input = True 52 | input_map[input_node_name].append(node.name) 53 | new_input_node = node_def_pb2.NodeDef() 54 | new_input_node.CopyFrom(graph_info[input_node_name].node) 55 | new_input_node.name = input_node_name + "_nc_share_" + str(len(input_map[input_node_name])) 56 | cur_graph.replace_const_node(new_input_node, [node.name], input_node_name, False) 57 | else: 58 | input_map[input_node_name] = [node.name] 59 | 60 | return cur_graph.dump_graph() if is_shared_input else self.model 61 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/generic/strip_equivalent_nodes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Strip Equivalent Nodes Graph Rewriter.""" 18 | 19 | from neural_compressor.adaptor.tf_utils.util import fix_ref_type_of_graph_def, strip_equivalent_nodes 20 | from neural_compressor.utils import logger 21 | from neural_compressor.utils.utility import dump_elapsed_time 22 | 23 | from ..graph_base import GraphRewriterBase 24 | 25 | 26 | class StripEquivalentNodesOptimizer(GraphRewriterBase): 27 | """Remove the equivalent nodes which have the same inputs and attributes.""" 28 | 29 | def __init__(self, model, output_node_names): 30 | """Initialization.""" 31 | super().__init__(model) 32 | self.output_node_names = output_node_names 33 | 34 | @dump_elapsed_time("Pass StripEquivalentNodesOptimizer") 35 | def do_transformation(self): 36 | """Strip the equivalent nodes in the graph.""" 37 | self.model = fix_ref_type_of_graph_def(self.model) 38 | iter_num = 0 39 | replaced_nodes_type = True 40 | all_replaced_nodes_type = {} 41 | while replaced_nodes_type: 42 | self.model, replaced_nodes_type = strip_equivalent_nodes(self.model, self.output_node_names) 43 | for k, v in replaced_nodes_type.items(): 44 | all_replaced_nodes_type[k] = all_replaced_nodes_type.get(k, 0) + v 45 | iter_num += 1 46 | logger.debug( 47 | f"StripEquivalentNodes[Iter-{iter_num}]-Replaced equivalent node types are {replaced_nodes_type}" 48 | ) 49 | logger.warning("All replaced equivalent node types are {}".format(all_replaced_nodes_type)) 50 | return self.model 51 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/generic/strip_unused_nodes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Strip unused nodes Graph Rewriter.""" 18 | 19 | from neural_compressor.utils.utility import dump_elapsed_time 20 | 21 | from ..graph_base import GraphRewriterBase 22 | 23 | 24 | class StripUnusedNodesOptimizer(GraphRewriterBase): 25 | """Remove the unused nodes in the graph.""" 26 | 27 | def __init__(self, model, input_node_names, output_node_names): 28 | """Initialization.""" 29 | super().__init__(model) 30 | self.input_node_names = input_node_names 31 | self.output_node_names = output_node_names 32 | 33 | @dump_elapsed_time("Pass StripUnusedNodesOptimizer") 34 | def do_transformation(self): 35 | """Execute stripping unused nodes.""" 36 | from neural_compressor.adaptor.tf_utils.util import fix_ref_type_of_graph_def, strip_unused_nodes 37 | 38 | self.model = fix_ref_type_of_graph_def(self.model) 39 | return strip_unused_nodes(self.model, self.input_node_names, self.output_node_names) 40 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/graph_base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Graph Rewrite Base Class.""" 18 | 19 | import logging 20 | from abc import abstractmethod 21 | 22 | 23 | class GraphRewriterBase: 24 | """Graph Rewrite Base class. 25 | 26 | We abstract this base class and define the interface only. 27 | 28 | Args: 29 | object (model): the input model to be converted. 30 | """ 31 | 32 | def __init__(self, model): 33 | """Initialization.""" 34 | self.model = model 35 | self.logger = logging.getLogger("neural_compressor") 36 | 37 | @abstractmethod 38 | def do_transformation(self): 39 | """Base Interface that need to be implemented by each sub class.""" 40 | raise NotImplementedError 41 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/int8/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tensorflow INT8 Graph Rewriters.""" 18 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/int8/post_hostconst_converter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Post HostConst Graph Rewriter.""" 18 | 19 | import os 20 | 21 | from tensorflow.core.framework import graph_pb2, node_def_pb2 22 | 23 | from neural_compressor.utils.utility import dump_elapsed_time 24 | 25 | from ..graph_base import GraphRewriterBase 26 | 27 | 28 | class PostHostConstConverter(GraphRewriterBase): 29 | """Support HostConst as default for all devices, not just for GPU.""" 30 | 31 | @dump_elapsed_time("Pass PostHostConstConverter") 32 | def do_transformation(self): 33 | """Convert Const to HostConst as default.""" 34 | if os.environ.get("DISABLE_HOSTCONST") == "1": 35 | return self.model 36 | output_graph_def = graph_pb2.GraphDef() 37 | for node in self.model.node: 38 | new_node = node_def_pb2.NodeDef() 39 | new_node.CopyFrom(node) 40 | new_node.device = "" 41 | if ( 42 | node.op == "Const" 43 | and node.attr["dtype"].type in [1, 3] 44 | and ( 45 | node.name.endswith("_min") 46 | or node.name.endswith("_max") 47 | or node.name.endswith("_max_only") 48 | or node.name.endswith("_min_only") 49 | ) 50 | ): 51 | new_node.op = "HostConst" 52 | output_graph_def.node.extend([new_node]) 53 | return output_graph_def 54 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/onnx/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tensorflow model export to ONNX model Graph Rewriters.""" 18 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/qdq/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tensorflow QDQ Graph Rewriters.""" 18 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/graph_rewriter/qdq/share_qdq_y_pattern.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Share QDQ for ITEX Y pattern Graph Rewriter.""" 18 | 19 | from neural_compressor.adaptor.tf_utils.graph_util import GraphAnalyzer 20 | from neural_compressor.utils.utility import dump_elapsed_time 21 | 22 | from ..graph_base import GraphRewriterBase 23 | 24 | 25 | class ShareQDQForItexYPatternOptimizer(GraphRewriterBase): 26 | """Insert Q/DQ op before one input of Add to enable Conv/MatMul + BiasAdd + Add + Relu fusion for ITEX. 27 | 28 | Only 1 Q/DQ before Add op need to be inserted. Insert 2 Q/DQ breaks the ITEX fusion pattern. 29 | """ 30 | 31 | @dump_elapsed_time("Pass ShareQDQForItexYPatternOptimizer") 32 | def do_transformation(self): 33 | """Share the QDQ of one output of Relu node with the another output which is Add node.""" 34 | g = GraphAnalyzer() 35 | g.graph = self.model 36 | graph_info = g.parse_graph() 37 | 38 | patterns = [["Relu", "MaxPool"], ["QuantizeV2"], ["Dequantize"]] 39 | matched_nodes = g.query_fusion_pattern_nodes(patterns) 40 | 41 | for i in matched_nodes: 42 | relu_node_name = graph_info[i[0]].node.name 43 | if len(g.node_name_details[relu_node_name].outputs) != 2: 44 | continue 45 | 46 | add_node_name = g.node_name_details[relu_node_name].outputs[0] 47 | quantize_node_name = g.node_name_details[relu_node_name].outputs[1] 48 | if ( 49 | "Add" not in g.node_name_details[add_node_name].node.op 50 | or g.node_name_details[quantize_node_name].node.op != "QuantizeV2" 51 | ): 52 | continue 53 | dequantize_node_name = graph_info[i[2]].node.name 54 | 55 | g.node_name_details[relu_node_name].outputs.remove(add_node_name) 56 | g.node_name_details[dequantize_node_name].outputs.append(add_node_name) 57 | g.node_name_details[add_node_name].node.input.remove(relu_node_name) 58 | g.node_name_details[add_node_name].node.input.append(dequantize_node_name) 59 | 60 | return g.dump_graph() 61 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/quantize_graph/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tensorflow Graph Quantizers.""" 18 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/quantize_graph/qat/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tensorflow QAT Graph Quantizers.""" 18 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/quantize_graph/qat/quantize_layers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tensorflow QAT Graph Quantize Layers.""" 18 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/quantize_graph/qat/quantize_layers/optimize_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Optimize layer config.""" 18 | 19 | from .quantize_layer_add import QuantizeLayerAdd 20 | from .quantize_layer_bn import QuantizeLayerBatchNormalization 21 | 22 | 23 | def config_quantizable_layers(model): 24 | """Configure the quantizable layers.""" 25 | quantize_layer_mapping = {"Add": QuantizeLayerAdd, "BatchNormalization": QuantizeLayerBatchNormalization} 26 | 27 | for layer_class, quantize_layer in quantize_layer_mapping.items(): 28 | quantize_layer_mapping[layer_class] = quantize_layer() 29 | 30 | for layer in model.layers: 31 | if layer.__class__.__name__ in quantize_layer_mapping: 32 | quantize_layer_mapping[layer.__class__.__name__](layer) 33 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/quantize_graph/qat/quantize_layers/quantize_layer_bn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Quantize Layer BatchNormalization Class.""" 18 | 19 | from .quantize_layer_base import QuantizeLayerBase 20 | 21 | 22 | class QuantizeLayerBatchNormalization(QuantizeLayerBase): # pragma: no cover 23 | """The class for quantization of BatchNormalization.""" 24 | 25 | def __init__(self): 26 | """Initialize QuantizeLayerBatchNormalization class.""" 27 | super().__init__() 28 | 29 | def _quantizable_bn(self): 30 | """Check if the input layer meets criteria of quantization. 31 | 32 | Args: 33 | layer (tf.keras.layers.Layer): The input layer. 34 | 35 | Returns: 36 | quantizable (bool): If this layer should be quantized. 37 | """ 38 | input_layer = self._find_input_layers(self.layer) 39 | assert len(input_layer) == 1, "BatchNormalization only has one input." 40 | input_layer_class = input_layer.__class__.__name__ 41 | if "Conv" not in input_layer_class: 42 | return True 43 | 44 | return False 45 | 46 | def __call__(self, layer): 47 | """The main logic of QuantizeLayerBatchNormalization. 48 | 49 | Neural Compressor will enumerate all layers of the input model to check 50 | if there are any layer meeting the criteria. The chosen ones will be marked 51 | as quantizable by QuantizeConfig. 52 | 53 | Args: 54 | layer (tf.keras.layers.Layer): The keras layer to be estimated. 55 | """ 56 | self.layer = layer 57 | if self._quantizable_bn(): 58 | self.quantize_config.add_quantize_recipe({self.layer.name: {"quantize": True}}) 59 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/quantize_graph/qdq/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tensorflow QDQ Graph Quantizers.""" 18 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/tf_utils/transform_graph/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | """Tensorflow Graph Transformers.""" 19 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/torch_utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Torch utils module.""" 18 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/torch_utils/autoround/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from .autoround import AutoRound, AutoOPTRound, AutoAdamRound 15 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/torch_utils/layer_wise_quant/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2023 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Torch layer-wise quantization module.""" 18 | from .utils import load_empty_model 19 | from .quantize import LayerWiseQuant 20 | -------------------------------------------------------------------------------- /neural_compressor/adaptor/torch_utils/mixed_precision.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from neural_compressor.utils.utility import LazyImport 19 | 20 | ipex = LazyImport("intel_extension_for_pytorch") 21 | torch = LazyImport("torch") 22 | 23 | 24 | def ipex_mixed_precision(model, example_inputs=None, device="cpu"): 25 | """The function is used for ipex mixed precision quantization.""" 26 | model.eval() 27 | if device == "xpu": 28 | autocast = torch.xpu.amp.autocast 29 | else: 30 | autocast = torch.cpu.amp.autocast 31 | mp_model = model._model 32 | if example_inputs is None: 33 | assert False, "please provide the correct example_inputs for torchscript mode" 34 | mp_model = ipex.optimize(mp_model, dtype=torch.bfloat16) 35 | 36 | with torch.no_grad(), autocast(): 37 | if isinstance(example_inputs, dict): 38 | try: 39 | mp_model = torch.jit.trace(mp_model, example_kwarg_inputs=example_inputs) 40 | except: 41 | mp_model = torch.jit.trace( 42 | mp_model, example_kwarg_inputs=example_inputs, strict=False, check_trace=False 43 | ) 44 | else: 45 | try: 46 | mp_model = torch.jit.trace(mp_model, example_inputs) 47 | except: 48 | mp_model = torch.jit.trace(mp_model, example_inputs, strict=False) 49 | mp_model = torch.jit.freeze(mp_model.eval()) 50 | model._model = mp_model 51 | return model 52 | -------------------------------------------------------------------------------- /neural_compressor/algorithm/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Fetch all files and init all algorithms.""" 18 | 19 | from .algorithm import ALGORITHMS, Algorithm, AlgorithmScheduler, algorithm_registry 20 | from os.path import dirname, basename, isfile, join 21 | import glob 22 | 23 | modules = glob.glob(join(dirname(__file__), "*.py")) 24 | 25 | for f in modules: 26 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 27 | __import__(basename(f)[:-3], globals(), locals(), level=1) 28 | 29 | 30 | __all__ = ["ALGORITHMS", "Algorithm", "AlgorithmScheduler", "algorithm_registry"] 31 | -------------------------------------------------------------------------------- /neural_compressor/common/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from neural_compressor.common.logger import level, log, info, debug, warn, warning, error, fatal 16 | -------------------------------------------------------------------------------- /neural_compressor/common/utility.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2023 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | # All constants 20 | 21 | # constants for configs 22 | GLOBAL = "global" 23 | LOCAL = "local" 24 | DEFAULT_WHITE_LIST = "*" 25 | EMPTY_WHITE_LIST = None 26 | 27 | # config name 28 | BASE_CONFIG = "base_config" 29 | COMPOSABLE_CONFIG = "composable_config" 30 | RTN_WEIGHT_ONLY_QUANT = "rtn_weight_only_quant" 31 | STATIC_QUANT = "static_quant" 32 | GPTQ = "gptq" 33 | FP8_QUANT = "fp8_quant" 34 | 35 | 36 | from typing import Callable, Union 37 | 38 | OP_NAME_OR_MODULE_TYPE = Union[str, Callable] 39 | -------------------------------------------------------------------------------- /neural_compressor/compression/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | from .callbacks import QuantizationAwareTrainingCallbacks, DistillationCallbacks, PruningCallbacks 19 | -------------------------------------------------------------------------------- /neural_compressor/compression/distillation/__init__.py: -------------------------------------------------------------------------------- 1 | """Distillation init.""" 2 | # !/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2022 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | -------------------------------------------------------------------------------- /neural_compressor/compression/distillation/utility.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """This is an utility file for PyTorch distillation.""" 18 | 19 | from neural_compressor.utils.utility import LazyImport 20 | 21 | torch = LazyImport("torch") 22 | 23 | STUDENT_FEATURES = {} 24 | TEACHER_FEATURES = {} 25 | 26 | 27 | # for adapting fx model 28 | @torch.fx.wrap 29 | def record_output(output, name, output_process, student=False): 30 | """Record layers output. 31 | 32 | It is a help function. 33 | """ 34 | recorded_output = output 35 | if output_process != "": 36 | if isinstance(output, dict) and output_process in output: 37 | recorded_output = output[output_process] 38 | elif isinstance(output, (tuple, list)) and str.isnumeric(output_process): 39 | recorded_output = output[int(output_process)] 40 | elif callable(output_process): 41 | recorded_output = output_process(output) 42 | else: 43 | raise NotImplementedError( 44 | "Current only support get the data with " 45 | + "integer index in case the output is tuple or list and only " 46 | + "need one item or with key in case the output is dict, " 47 | + "or output_process is a function." 48 | ) 49 | if student: 50 | STUDENT_FEATURES[name].append(recorded_output) 51 | else: 52 | TEACHER_FEATURES[name].append(recorded_output) 53 | return output 54 | 55 | 56 | def get_activation(name, output_process="", student=False): 57 | """Get a hook for getting activation.""" 58 | 59 | def hook(model, input, output): 60 | if model.training or not student: 61 | return record_output(output, name, output_process, student=student) 62 | else: 63 | return output 64 | 65 | return hook 66 | -------------------------------------------------------------------------------- /neural_compressor/compression/hpo/__init__.py: -------------------------------------------------------------------------------- 1 | """Hyper-parameter Optimization.""" 2 | #!/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2021 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | from .search_space import * 20 | from .sa_optimizer import * 21 | from .search_algorithms import * 22 | -------------------------------------------------------------------------------- /neural_compressor/compression/pruner/model_slim/__init__.py: -------------------------------------------------------------------------------- 1 | """Slim init.""" 2 | # !/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2022 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | from .auto_slim import parse_auto_slim_config 19 | from .auto_slim import model_slim 20 | -------------------------------------------------------------------------------- /neural_compressor/compression/pruner/model_slim/imgs/auto_slim_feed_forward_network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cornell-zhang/llm-datatypes/b0f1a17190aae75a8badb37ce679652b74a28fca/neural_compressor/compression/pruner/model_slim/imgs/auto_slim_feed_forward_network.png -------------------------------------------------------------------------------- /neural_compressor/compression/pruner/patterns/__init__.py: -------------------------------------------------------------------------------- 1 | """Pruners.""" 2 | # !/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2022 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | from os.path import dirname, basename, isfile, join 20 | import glob 21 | 22 | from .base import PATTERNS 23 | 24 | modules = glob.glob(join(dirname(__file__), "*.py")) 25 | 26 | for f in modules: 27 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 28 | __import__(basename(f)[:-3], globals(), locals(), level=1) 29 | 30 | FRAMEWORK = {"pytorch": "pt", "keras": "keras"} 31 | 32 | 33 | def get_pattern(config, modules, framework="pytorch"): 34 | """Get registered pattern class. 35 | 36 | Get a Pattern object from PATTERNS. 37 | 38 | Args: 39 | config: A config dict object that contains the pattern information. 40 | modules: Torch neural network modules to be pruned with the pattern. 41 | 42 | Returns: 43 | A Pattern object. 44 | 45 | Raises: 46 | AssertionError: Currently only support patterns which have been registered in PATTERNS. 47 | """ 48 | assert framework in FRAMEWORK.keys(), f"does not support {framework}, currently only support {FRAMEWORK.keys()}" 49 | 50 | name = config.pattern 51 | name = name.split("_")[-1] 52 | pattern = FRAMEWORK[framework] 53 | if "x" in name: 54 | pattern += "NxM" 55 | elif ":" in name: 56 | pattern += "N:M" 57 | elif "mha" in name: 58 | pattern += "MHA" 59 | else: 60 | assert False, f"currently only support {PATTERNS.keys()}" 61 | if pattern not in PATTERNS.keys(): 62 | assert False, f"currently only support {PATTERNS.keys()}" 63 | return PATTERNS[pattern](config, modules) 64 | -------------------------------------------------------------------------------- /neural_compressor/compression/pruner/patterns/mha.py: -------------------------------------------------------------------------------- 1 | """MHA patterns.""" 2 | # !/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2022 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | from ..utils import torch 19 | from .base import PytorchBasePattern, register_pattern 20 | 21 | 22 | @register_pattern("ptMHA") 23 | class PatternMHA(PytorchBasePattern): 24 | """Pruning Pattern. 25 | 26 | A Pattern class derived from BasePattern. In this pattern, we calculate head masks for a MHA module 27 | For more info of this pattern, please refer to : 28 | https://github.com/intel/neural-compressor/blob/master/docs/sparsity.md 29 | 30 | Args: 31 | config: A config dict object that contains the pattern information. 32 | 33 | Attributes: 34 | N: The number of elements to be pruned in a weight sequence. 35 | M: The size of the weight sequence. 36 | """ 37 | 38 | def __init__(self, config, modules=None): 39 | self.is_global = config.pruning_scope == "global" 40 | 41 | # only implement three method: get_masks, get_masks_local, get_masks_global 42 | 43 | def get_masks_global(self, scores, target_sparsity_ratio, pre_masks): 44 | # gather all score items into one tensor 45 | if target_sparsity_ratio <= 0.0: 46 | return pre_masks 47 | flatten_score = torch.cat(list(scores.values())).flatten() 48 | k = int(target_sparsity_ratio * flatten_score.numel()) 49 | if k <= 0: 50 | return pre_masks 51 | threshold, _ = torch.kthvalue(flatten_score, k) 52 | head_masks = {} 53 | zero = torch.tensor([0.0]).to(threshold.device) 54 | one = torch.tensor([1.0]).to(threshold.device) 55 | for mha_name, mha_score in scores.items(): 56 | head_masks[mha_name] = torch.where(mha_score <= threshold, zero, one).permute(1, 0) 57 | head_masks[mha_name] = head_masks[mha_name].bool() 58 | return head_masks 59 | -------------------------------------------------------------------------------- /neural_compressor/compression/pruner/pruners/pattern_lock.py: -------------------------------------------------------------------------------- 1 | """Pattern lock pruner.""" 2 | # !/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2022 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | from ..patterns import get_pattern 20 | from .base import KerasBasePruner, PytorchBasePruner, register_pruner 21 | 22 | 23 | @register_pruner("pt_pattern_lock") 24 | class PytorchPatternLockPruner(PytorchBasePruner): 25 | """Pruning Pruner. 26 | 27 | A Pruner class derived from BasePruner. 28 | In this pruner, original model's sparsity pattern will be fixed while training. 29 | This pruner is useful when a user trains a sparse model without changing its original structure. 30 | 31 | Args: 32 | modules: A dict {"module_name": Tensor} that stores the pruning modules' weights. 33 | config: A config dict object that contains the pruner information. 34 | 35 | Attributes: 36 | Inherit from parent class Pruner. 37 | """ 38 | 39 | def __init__(self, config, modules): 40 | """Initialize.""" 41 | super().__init__(config, modules) 42 | self.pattern = get_pattern(self.config, modules) 43 | assert self.config.end_step == self.config.start_step, "pattern_lock pruner only supports one shot mode" 44 | 45 | def update_masks(self, local_step): 46 | """Update the masks at a given local step.""" 47 | if not self.check_is_pruned_step(self.global_step): 48 | return 49 | self.masks = self.pattern.get_pattern_lock_masks(self.modules) 50 | 51 | def on_after_optimizer_step(self): 52 | """Implement after optimizer.step(). 53 | 54 | Prune the model after optimization. 55 | """ 56 | self.mask_weights() 57 | self.global_step += 1 58 | -------------------------------------------------------------------------------- /neural_compressor/conf/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | -------------------------------------------------------------------------------- /neural_compressor/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Built-in strategy for multiple framework backends.""" 18 | from .strategy import * 19 | -------------------------------------------------------------------------------- /neural_compressor/contrib/strategy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Built-in strategy for multiple framework backends.""" 18 | from os.path import dirname, basename, isfile, join 19 | import glob 20 | 21 | modules = glob.glob(join(dirname(__file__), "*.py")) 22 | 23 | for f in modules: 24 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 25 | __import__(basename(f)[:-3], globals(), locals(), level=1) 26 | -------------------------------------------------------------------------------- /neural_compressor/data/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ============================================================================== 18 | """Built-in dataloaders, datasets, transforms, filters for multiple framework backends.""" 19 | 20 | 21 | import neural_compressor.data.datasets 22 | import neural_compressor.data.transforms 23 | from .datasets import Datasets, Dataset, IterableDataset, dataset_registry, TensorflowImageRecord, COCORecordDataset 24 | from .dataloaders import DATALOADERS, DataLoader 25 | from .dataloaders.dataloader import check_dataloader 26 | from .dataloaders.default_dataloader import DefaultDataLoader 27 | from .transforms import TRANSFORMS, BaseTransform, ComposeTransform, transform_registry, Postprocess 28 | from .transforms import LabelShift, BilinearImagenetTransform, TensorflowResizeCropImagenetTransform 29 | from .transforms import TFSquadV1PostTransform, TFSquadV1ModelZooPostTransform 30 | from .transforms import TensorflowResizeWithRatio, ResizeTFTransform, RescaleTFTransform, NormalizeTFTransform 31 | from .transforms import ParseDecodeCocoTransform, TensorflowShiftRescale 32 | 33 | from .filters import FILTERS, Filter, filter_registry, LabelBalanceCOCORecordFilter 34 | 35 | __all__ = [ 36 | "check_dataloader", 37 | "DataLoader", 38 | "DATALOADERS", 39 | "DefaultDataLoader", 40 | "Datasets", 41 | "Dataset", 42 | "IterableDataset", 43 | "COCORecordDataset", 44 | "dataset_registry", 45 | "TensorflowImageRecord", 46 | "TRANSFORMS", 47 | "BaseTransform", 48 | "ComposeTransform", 49 | "transform_registry", 50 | "Postprocess", 51 | "LabelShift", 52 | "ResizeTFTransform", 53 | "RescaleTFTransform", 54 | "TensorflowShiftRescale", 55 | "NormalizeTFTransform", 56 | "ParseDecodeCocoTransform", 57 | "BilinearImagenetTransform", 58 | "TensorflowResizeWithRatio", 59 | "TensorflowResizeCropImagenetTransform", 60 | "FILTERS", 61 | "Filter", 62 | "filter_registry", 63 | "LabelBalanceCOCORecordFilter", 64 | "TFSquadV1PostTransform", 65 | "TFSquadV1ModelZooPostTransform", 66 | ] 67 | -------------------------------------------------------------------------------- /neural_compressor/data/dataloaders/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ============================================================================== 18 | 19 | from .dataloader import DataLoader, DATALOADERS 20 | 21 | __all__ = ["DataLoader", "DATALOADERS"] 22 | -------------------------------------------------------------------------------- /neural_compressor/data/dataloaders/mxnet_dataloader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ============================================================================== 18 | """MXNet Dataloader implementation.""" 19 | 20 | import logging 21 | 22 | from neural_compressor.utils.utility import LazyImport 23 | 24 | from .base_dataloader import BaseDataLoader 25 | 26 | mx = LazyImport("mxnet") 27 | 28 | 29 | class MXNetDataLoader(BaseDataLoader): # pragma: no cover 30 | """Subclass of BaseDataLoader.""" 31 | 32 | def _generate_dataloader( 33 | self, 34 | dataset, 35 | batch_size, 36 | last_batch, 37 | collate_fn, 38 | sampler, 39 | batch_sampler, 40 | num_workers, 41 | pin_memory, 42 | shuffle, 43 | distributed, 44 | ): 45 | """Overwrite _generate_dataloader function.""" 46 | if shuffle: 47 | logging.warning("Shuffle is not supported yet in MXNetDataLoader, " "ignoring shuffle keyword.") 48 | return mx.gluon.data.DataLoader( 49 | dataset, 50 | batch_size=batch_size, 51 | batchify_fn=collate_fn, 52 | last_batch=last_batch, 53 | num_workers=num_workers, 54 | pin_memory=pin_memory, 55 | sampler=sampler, 56 | batch_sampler=batch_sampler, 57 | ) 58 | -------------------------------------------------------------------------------- /neural_compressor/data/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Built-in datasets class for multiple framework backends.""" 18 | 19 | from .coco_dataset import COCORecordDataset 20 | from .dataset import Datasets, Dataset, IterableDataset, dataset_registry, TensorflowImageRecord 21 | from os.path import dirname, basename, isfile, join 22 | import glob 23 | 24 | modules = glob.glob(join(dirname(__file__), "*.py")) 25 | 26 | for f in modules: 27 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 28 | __import__(basename(f)[:-3], globals(), locals(), level=1) 29 | 30 | 31 | __all__ = ["Datasets", "Dataset", "IterableDataset", "dataset_registry", "TensorflowImageRecord", "COCORecordDataset"] 32 | -------------------------------------------------------------------------------- /neural_compressor/data/filters/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Built-in filter.""" 18 | 19 | from .coco_filter import LabelBalanceCOCORecordFilter 20 | from .filter import FILTERS, Filter, filter_registry 21 | from os.path import dirname, basename, isfile, join 22 | import glob 23 | 24 | modules = glob.glob(join(dirname(__file__), "*.py")) 25 | 26 | for f in modules: 27 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 28 | __import__(basename(f)[:-3], globals(), locals(), level=1) 29 | 30 | 31 | __all__ = ["FILTERS", "Filter", "filter_registry", "LabelBalanceCOCORecordFilter"] 32 | -------------------------------------------------------------------------------- /neural_compressor/data/filters/coco_filter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Built-in COCO filter.""" 18 | 19 | from neural_compressor.utils.utility import LazyImport 20 | 21 | from .filter import Filter, filter_registry 22 | 23 | tf = LazyImport("tensorflow") 24 | 25 | 26 | @filter_registry(filter_type="LabelBalanceCOCORecord", framework="tensorflow, tensorflow_itex") 27 | class LabelBalanceCOCORecordFilter(Filter): # pragma: no cover 28 | """The label balance filter for COCO Record.""" 29 | 30 | def __init__(self, size=1): 31 | """Initialize the attribute of class.""" 32 | self.size = size 33 | 34 | def __call__(self, image, label): 35 | """Execute the filter. 36 | 37 | Args: 38 | image: Not used. 39 | label: label of a sample. 40 | """ 41 | return tf.math.equal(len(label[0]), self.size) 42 | 43 | 44 | @filter_registry( 45 | filter_type="LabelBalanceCOCORaw", 46 | framework="tensorflow, \ 47 | tensorflow_itex, pytorch, mxnet, onnxrt_qlinearops, onnxrt_integerops", 48 | ) 49 | class LabelBalanceCOCORawFilter(Filter): # pragma: no cover 50 | """The label balance filter for COCO raw data.""" 51 | 52 | def __init__(self, size=1): 53 | """Initialize the attribute of class.""" 54 | self.size = size 55 | 56 | def __call__(self, image, label): 57 | """Execute the filter. 58 | 59 | Args: 60 | image: Not used. 61 | label: label of a sample. 62 | """ 63 | return len(label) == self.size 64 | -------------------------------------------------------------------------------- /neural_compressor/data/transforms/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ============================================================================== 18 | """Neural Compressor Built-in transforms for multiple framework backends.""" 19 | 20 | from .transform import ( 21 | TRANSFORMS, 22 | BaseTransform, 23 | ComposeTransform, 24 | transform_registry, 25 | ResizeTFTransform, 26 | TensorflowResizeWithRatio, 27 | RescaleTFTransform, 28 | NormalizeTFTransform, 29 | ) 30 | from .transform import TFSquadV1PostTransform, TFSquadV1ModelZooPostTransform 31 | from .coco_transform import ParseDecodeCocoTransform 32 | from .postprocess import Postprocess 33 | from .imagenet_transform import LabelShift, BilinearImagenetTransform, TensorflowResizeCropImagenetTransform 34 | from .imagenet_transform import TensorflowShiftRescale 35 | from os.path import dirname, basename, isfile, join 36 | import glob 37 | 38 | modules = glob.glob(join(dirname(__file__), "*.py")) 39 | 40 | for f in modules: 41 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 42 | __import__(basename(f)[:-3], globals(), locals(), level=1) 43 | 44 | 45 | __all__ = [ 46 | "TRANSFORMS", 47 | "BaseTransform", 48 | "ComposeTransform", 49 | "transform_registry", 50 | "ResizeTFTransform", 51 | "Postprocess", 52 | "LabelShift", 53 | "BilinearImagenetTransform", 54 | "TensorflowResizeCropImagenetTransform", 55 | "RescaleTFTransform", 56 | "NormalizeTFTransform", 57 | "ParseDecodeCocoTransform", 58 | "TensorflowResizeWithRatio", 59 | "TFSquadV1PostTransform", 60 | "TFSquadV1ModelZooPostTransform", 61 | "TensorflowShiftRescale", 62 | ] 63 | -------------------------------------------------------------------------------- /neural_compressor/data/transforms/coco_transform.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 19 | # 20 | # Licensed under the Apache License, Version 2.0 (the "License"); 21 | # you may not use this file except in compliance with the License. 22 | # You may obtain a copy of the License at 23 | # 24 | # http://www.apache.org/licenses/LICENSE-2.0 25 | # 26 | # Unless required by applicable law or agreed to in writing, software 27 | # distributed under the License is distributed on an "AS IS" BASIS, 28 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | # See the License for the specific language governing permissions and 30 | # limitations under the License. 31 | # ============================================================================== 32 | 33 | from neural_compressor.data.transforms import BaseTransform, transform_registry 34 | from neural_compressor.utils import logger 35 | 36 | 37 | # BELOW IS TO BE DEPRECATED! 38 | @transform_registry(transform_type="ParseDecodeCoco", process="preprocess", framework="tensorflow") 39 | class ParseDecodeCocoTransform(BaseTransform): # pragma: no cover 40 | """Coco decoding will be performed automatically from Neural Compressor v1.4.""" 41 | 42 | def __call__(self, sample): 43 | """Convert `ParseDecodeCocoTransform` feature.""" 44 | logger.warning( 45 | "This transform is going to be deprecated, " 46 | "coco decoding will be performed automatically from Neural Compressor v1.4." 47 | ) 48 | return sample 49 | -------------------------------------------------------------------------------- /neural_compressor/data/transforms/postprocess.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Common Postprocess.""" 18 | 19 | 20 | class Postprocess(object): 21 | # class Transform(object): 22 | """Just collect the infos to construct a Postprocess.""" 23 | 24 | def __init__(self, postprocess_cls, name="user_postprocess", **kwargs): 25 | """Initialize `Postprocess` class.""" 26 | self.postprocess_cls = postprocess_cls 27 | self.name = name 28 | self.kwargs = kwargs 29 | -------------------------------------------------------------------------------- /neural_compressor/experimental/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Intel® Neural Compressor: An open-source Python library supporting popular model compression techniques.""" 18 | 19 | from .component import Component 20 | from .quantization import Quantization 21 | from .pruning import Pruning 22 | from .benchmark import Benchmark 23 | from .graph_optimization import Graph_Optimization, GraphOptimization 24 | from .mixed_precision import MixedPrecision 25 | from .model_conversion import ModelConversion 26 | from .distillation import Distillation 27 | from .nas import NAS 28 | from . import export 29 | from .contrib import * 30 | 31 | __all__ = [ 32 | "Component", 33 | "Quantization", 34 | "Pruning", 35 | "Benchmark", 36 | "Graph_Optimization", 37 | "GraphOptimization", 38 | "ModelConversion", 39 | "Distillation", 40 | "NAS", 41 | "MixedPrecision", 42 | "export", 43 | ] 44 | -------------------------------------------------------------------------------- /neural_compressor/experimental/common/__init__.py: -------------------------------------------------------------------------------- 1 | """Intel® Neural Compressor: An open-source Python library supporting common model.""" 2 | #!/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2021 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | from .model import Model 20 | from .dataloader import DataLoader, _generate_common_dataloader 21 | from .postprocess import Postprocess 22 | from .metric import Metric 23 | from .criterion import Criterions 24 | from .optimizer import Optimizers 25 | 26 | __all__ = ["Model", "DataLoader", "Postprocess", "Metric", "_generate_common_dataloader"] 27 | -------------------------------------------------------------------------------- /neural_compressor/experimental/common/metric.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Common Metric just collects the information to construct a Metric.""" 18 | from deprecated import deprecated 19 | 20 | 21 | @deprecated(version="2.0") 22 | class Metric(object): 23 | """A wrapper of the information needed to construct a Metric. 24 | 25 | The metric class should take the outputs of the model as the metric's inputs, 26 | neural_compressor built-in metric always take (predictions, labels) as inputs, it's 27 | recommended to design metric_cls to take (predictions, labels) as inputs. 28 | """ 29 | 30 | def __init__(self, metric_cls, name="user_metric", **kwargs): 31 | """Initialize a Metric with needed information. 32 | 33 | Args: 34 | metric_cls (cls): Should be a sub_class of neural_compressor.metric.BaseMetric, 35 | which takes (predictions, labels) as inputs 36 | name (str, optional): Name for metric. Defaults to 'user_metric'. 37 | """ 38 | self.metric_cls = metric_cls 39 | self.name = name 40 | self.kwargs = kwargs 41 | -------------------------------------------------------------------------------- /neural_compressor/experimental/common/postprocess.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Common Postprocess.""" 18 | from deprecated import deprecated 19 | 20 | 21 | @deprecated(version="2.0") 22 | class Postprocess(object): 23 | # class Transform(object): 24 | """Just collect the infos to construct a Postprocess.""" 25 | 26 | def __init__(self, postprocess_cls, name="user_postprocess", **kwargs): 27 | """Initialize `Postprocess` class.""" 28 | self.postprocess_cls = postprocess_cls 29 | self.name = name 30 | self.kwargs = kwargs 31 | -------------------------------------------------------------------------------- /neural_compressor/experimental/common/torch_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """This is an utility file for PyTorch distillation.""" 18 | from deprecated import deprecated 19 | 20 | from neural_compressor.utils.utility import LazyImport 21 | 22 | torch = LazyImport("torch") 23 | 24 | STUDENT_FEATURES = {} 25 | TEACHER_FEATURES = {} 26 | 27 | 28 | # for adapting fx model 29 | @deprecated(version="2.0") 30 | @torch.fx.wrap 31 | def record_output(output, name, output_process, student=False): 32 | """Record layers output. 33 | 34 | It is a help function. 35 | """ 36 | recorded_output = output 37 | if output_process != "": 38 | if isinstance(output, dict) and output_process in output: 39 | recorded_output = output[output_process] 40 | elif isinstance(output, (tuple, list)) and str.isnumeric(output_process): 41 | recorded_output = output[int(output_process)] 42 | elif callable(output_process): 43 | recorded_output = output_process(output) 44 | else: 45 | raise NotImplementedError( 46 | "Current only support get the data with " 47 | + "integer index in case the output is tuple or list and only " 48 | + "need one item or with key in case the output is dict, " 49 | + "or output_process is a function." 50 | ) 51 | if student: 52 | STUDENT_FEATURES[name].append(recorded_output) 53 | else: 54 | TEACHER_FEATURES[name].append(recorded_output) 55 | return output 56 | 57 | 58 | @deprecated(version="2.0") 59 | def get_activation(name, output_process="", student=False): 60 | """Get a hook for getting activation.""" 61 | 62 | def hook(model, input, output): 63 | if model.training or not student: 64 | return record_output(output, name, output_process, student=student) 65 | else: 66 | return output 67 | 68 | return hook 69 | -------------------------------------------------------------------------------- /neural_compressor/experimental/compression/__init__.py: -------------------------------------------------------------------------------- 1 | """Experimental pruning API.""" 2 | #!/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2023 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | -------------------------------------------------------------------------------- /neural_compressor/experimental/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Built-in strategy for multiple framework backends.""" 18 | from .strategy import * 19 | -------------------------------------------------------------------------------- /neural_compressor/experimental/contrib/strategy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Built-in strategy for multiple framework backends.""" 18 | from os.path import dirname, basename, isfile, join 19 | import glob 20 | 21 | modules = glob.glob(join(dirname(__file__), "*.py")) 22 | for f in modules: 23 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 24 | __import__(basename(f)[:-3], globals(), locals(), level=1) 25 | -------------------------------------------------------------------------------- /neural_compressor/experimental/data/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ============================================================================== 18 | """Built-in dataloaders, datasets, transforms, filters for multiple framework backends.""" 19 | 20 | 21 | from .datasets import Datasets, Dataset, IterableDataset, dataset_registry 22 | from .transforms import TRANSFORMS, BaseTransform, transform_registry 23 | from .dataloaders import DATALOADERS 24 | from .filters import FILTERS, Filter, filter_registry 25 | 26 | __all__ = [ 27 | "DATALOADERS", 28 | "Datasets", 29 | "Dataset", 30 | "IterableDataset", 31 | "dataset_registry", 32 | "TRANSFORMS", 33 | "BaseTransform", 34 | "transform_registry", 35 | "FILTERS", 36 | "Filter", 37 | "filter_registry", 38 | ] 39 | -------------------------------------------------------------------------------- /neural_compressor/experimental/data/dataloaders/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ============================================================================== 18 | """Built-in dataloaders for multiple framework backends.""" 19 | 20 | from .dataloader import DATALOADERS 21 | 22 | __all__ = ["DATALOADERS"] 23 | -------------------------------------------------------------------------------- /neural_compressor/experimental/data/dataloaders/dataloader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ============================================================================== 18 | """Built-in dataloaders for multiple framework backends.""" 19 | from .mxnet_dataloader import MXNetDataLoader 20 | from .onnxrt_dataloader import ONNXRTDataLoader 21 | from .pytorch_dataloader import PyTorchDataLoader 22 | from .tensorflow_dataloader import TensorflowDataLoader 23 | 24 | DATALOADERS = { 25 | "tensorflow": TensorflowDataLoader, 26 | "tensorflow_itex": TensorflowDataLoader, 27 | "keras": TensorflowDataLoader, 28 | "mxnet": MXNetDataLoader, 29 | "pytorch": PyTorchDataLoader, 30 | "pytorch_ipex": PyTorchDataLoader, 31 | "pytorch_fx": PyTorchDataLoader, 32 | "onnxrt_qlinearops": ONNXRTDataLoader, 33 | "onnxrt_integerops": ONNXRTDataLoader, 34 | "onnxrt_qdq": ONNXRTDataLoader, 35 | "onnxruntime": ONNXRTDataLoader, 36 | } 37 | -------------------------------------------------------------------------------- /neural_compressor/experimental/data/dataloaders/mxnet_dataloader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ============================================================================== 18 | """MXNet Dataloader implementation.""" 19 | 20 | import logging 21 | 22 | from deprecated import deprecated 23 | 24 | from neural_compressor.utils.utility import LazyImport 25 | 26 | from .base_dataloader import BaseDataLoader 27 | 28 | mx = LazyImport("mxnet") 29 | 30 | 31 | @deprecated(version="2.0") 32 | class MXNetDataLoader(BaseDataLoader): 33 | """Subclass of BaseDataLoader.""" 34 | 35 | def _generate_dataloader( 36 | self, 37 | dataset, 38 | batch_size, 39 | last_batch, 40 | collate_fn, 41 | sampler, 42 | batch_sampler, 43 | num_workers, 44 | pin_memory, 45 | shuffle, 46 | distributed, 47 | ): 48 | """Overwrite _generate_dataloader function.""" 49 | if shuffle: 50 | logging.warning("Shuffle is not supported yet in MXNetDataLoader, " "ignoring shuffle keyword.") 51 | return mx.gluon.data.DataLoader( 52 | dataset, 53 | batch_size=batch_size, 54 | batchify_fn=collate_fn, 55 | last_batch=last_batch, 56 | num_workers=num_workers, 57 | pin_memory=pin_memory, 58 | sampler=sampler, 59 | batch_sampler=batch_sampler, 60 | ) 61 | -------------------------------------------------------------------------------- /neural_compressor/experimental/data/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Built-in datasets class for multiple framework backends.""" 18 | 19 | from .dataset import Datasets, Dataset, IterableDataset, dataset_registry 20 | from os.path import dirname, basename, isfile, join 21 | import glob 22 | 23 | modules = glob.glob(join(dirname(__file__), "*.py")) 24 | 25 | for f in modules: 26 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 27 | __import__(basename(f)[:-3], globals(), locals(), level=1) 28 | 29 | 30 | __all__ = ["Datasets", "Dataset", "IterableDataset", "dataset_registry"] 31 | -------------------------------------------------------------------------------- /neural_compressor/experimental/data/filters/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Built-in filter.""" 18 | 19 | from .filter import FILTERS, Filter, filter_registry 20 | from os.path import dirname, basename, isfile, join 21 | import glob 22 | 23 | modules = glob.glob(join(dirname(__file__), "*.py")) 24 | 25 | for f in modules: 26 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 27 | __import__(basename(f)[:-3], globals(), locals(), level=1) 28 | 29 | 30 | __all__ = ["FILTERS", "Filter", "filter_registry"] 31 | -------------------------------------------------------------------------------- /neural_compressor/experimental/data/filters/coco_filter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Built-in COCO filter.""" 18 | 19 | from neural_compressor.utils.utility import LazyImport 20 | 21 | from .filter import Filter, filter_registry 22 | 23 | tf = LazyImport("tensorflow") 24 | 25 | 26 | @filter_registry(filter_type="LabelBalanceCOCORecord", framework="tensorflow, tensorflow_itex") 27 | class LabelBalanceCOCORecordFilter(Filter): 28 | """The label balance filter for COCO Record.""" 29 | 30 | def __init__(self, size=1): 31 | """Initialize the attribute of class.""" 32 | self.size = size 33 | 34 | def __call__(self, image, label): 35 | """Execute the filter. 36 | 37 | Args: 38 | image: Not used. 39 | label: label of a sample. 40 | """ 41 | return tf.math.equal(len(label[0]), self.size) 42 | 43 | 44 | @filter_registry( 45 | filter_type="LabelBalanceCOCORaw", 46 | framework="tensorflow, \ 47 | tensorflow_itex, pytorch, mxnet, onnxrt_qlinearops, onnxrt_integerops", 48 | ) 49 | class LabelBalanceCOCORawFilter(Filter): 50 | """The label balance filter for COCO raw data.""" 51 | 52 | def __init__(self, size=1): 53 | """Initialize the attribute of class.""" 54 | self.size = size 55 | 56 | def __call__(self, image, label): 57 | """Execute the filter. 58 | 59 | Args: 60 | image: Not used. 61 | label: label of a sample. 62 | """ 63 | return len(label) == self.size 64 | -------------------------------------------------------------------------------- /neural_compressor/experimental/data/transforms/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # ============================================================================== 18 | """Neural Compressor Built-in transforms for multiple framework backends.""" 19 | 20 | from .transform import TRANSFORMS, BaseTransform, transform_registry 21 | from os.path import dirname, basename, isfile, join 22 | import glob 23 | 24 | modules = glob.glob(join(dirname(__file__), "*.py")) 25 | 26 | for f in modules: 27 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 28 | __import__(basename(f)[:-3], globals(), locals(), level=1) 29 | 30 | 31 | __all__ = ["TRANSFORMS", "BaseTransform", "transform_registry"] 32 | -------------------------------------------------------------------------------- /neural_compressor/experimental/export/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Intel Neural Compressor Export.""" 18 | 19 | from .torch2onnx import torch_to_fp32_onnx, torch_to_int8_onnx 20 | from .qlinear2qdq import onnx_qlinear_to_qdq 21 | from .tf2onnx import tf_to_fp32_onnx, tf_to_int8_onnx 22 | -------------------------------------------------------------------------------- /neural_compressor/experimental/metric/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Intel Neural Compressor Metric.""" 18 | 19 | from .metric import METRICS, BaseMetric, metric_registry 20 | from os.path import dirname, basename, isfile, join 21 | import glob 22 | 23 | modules = glob.glob(join(dirname(__file__), "*.py")) 24 | 25 | for f in modules: 26 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 27 | __import__(basename(f)[:-3], globals(), locals(), level=1) 28 | 29 | 30 | __all__ = ["METRICS", "BaseMetric", "metric_registry"] 31 | -------------------------------------------------------------------------------- /neural_compressor/experimental/metric/coco_label_map.py: -------------------------------------------------------------------------------- 1 | # 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | # 19 | """The dict mapping category IDs to its names of labels.""" 20 | 21 | category_map = { 22 | 1: "person", 23 | 2: "bicycle", 24 | 3: "car", 25 | 4: "motorcycle", 26 | 5: "airplane", 27 | 6: "bus", 28 | 7: "train", 29 | 8: "truck", 30 | 9: "boat", 31 | 10: "traffic light", 32 | 11: "fire hydrant", 33 | 13: "stop sign", 34 | 14: "parking meter", 35 | 15: "bench", 36 | 16: "bird", 37 | 17: "cat", 38 | 18: "dog", 39 | 19: "horse", 40 | 20: "sheep", 41 | 21: "cow", 42 | 22: "elephant", 43 | 23: "bear", 44 | 24: "zebra", 45 | 25: "giraffe", 46 | 27: "backpack", 47 | 28: "umbrella", 48 | 31: "handbag", 49 | 32: "tie", 50 | 33: "suitcase", 51 | 34: "frisbee", 52 | 35: "skis", 53 | 36: "snowboard", 54 | 37: "sports ball", 55 | 38: "kite", 56 | 39: "baseball bat", 57 | 40: "baseball glove", 58 | 41: "skateboard", 59 | 42: "surfboard", 60 | 43: "tennis racket", 61 | 44: "bottle", 62 | 46: "wine glass", 63 | 47: "cup", 64 | 48: "fork", 65 | 49: "knife", 66 | 50: "spoon", 67 | 51: "bowl", 68 | 52: "banana", 69 | 53: "apple", 70 | 54: "sandwich", 71 | 55: "orange", 72 | 56: "broccoli", 73 | 57: "carrot", 74 | 58: "hot dog", 75 | 59: "pizza", 76 | 60: "donut", 77 | 61: "cake", 78 | 62: "chair", 79 | 63: "couch", 80 | 64: "potted plant", 81 | 65: "bed", 82 | 67: "dining table", 83 | 70: "toilet", 84 | 72: "tv", 85 | 73: "laptop", 86 | 74: "mouse", 87 | 75: "remote", 88 | 76: "keyboard", 89 | 77: "cell phone", 90 | 78: "microwave", 91 | 79: "oven", 92 | 80: "toaster", 93 | 81: "sink", 94 | 82: "refrigerator", 95 | 84: "book", 96 | 85: "clock", 97 | 86: "vase", 98 | 87: "scissors", 99 | 88: "teddy bear", 100 | 89: "hair drier", 101 | 90: "toothbrush", 102 | } 103 | -------------------------------------------------------------------------------- /neural_compressor/experimental/nas/__init__.py: -------------------------------------------------------------------------------- 1 | """NAS module.""" 2 | #!/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2021 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | from .basic_nas import BasicNAS 20 | from .dynas import DyNAS 21 | from .nas import NAS 22 | -------------------------------------------------------------------------------- /neural_compressor/experimental/pruner_legacy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Legacy pruner module.""" 18 | 19 | from os.path import dirname, basename, isfile, join 20 | import glob 21 | from .pruner import PRUNERS 22 | 23 | modules = glob.glob(join(dirname(__file__), "*.py")) 24 | 25 | for f in modules: 26 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 27 | __import__(basename(f)[:-3], globals(), locals(), level=1) 28 | 29 | __all__ = ["PRUNERS"] 30 | -------------------------------------------------------------------------------- /neural_compressor/experimental/pruner_legacy/pattern_lock.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Pattern lock pruner.""" 18 | from deprecated import deprecated 19 | 20 | from .pruner import Pruner, pruner_registry 21 | 22 | 23 | @deprecated(version="2.0") 24 | @pruner_registry 25 | class PatternLockPruner(Pruner): 26 | """Pattern lock pruner class. 27 | 28 | Args: 29 | model (object): The original model (currently PyTorchModel instance). 30 | local_config (Conf): configs specific for this pruning instance. 31 | global_config (Conf): global configs which may be overwritten by local_config. 32 | """ 33 | 34 | def __init__(self, model, local_config, global_config): 35 | """Initialize the attributes.""" 36 | super(PatternLockPruner, self).__init__(model, local_config, global_config) 37 | self.compute_mask() 38 | 39 | def on_epoch_begin(self, epoch): 40 | """Be called on the beginning of epochs.""" 41 | pass 42 | 43 | def on_step_begin(self, batch_id): 44 | """Be called on the beginning of steps.""" 45 | pass 46 | 47 | def on_epoch_end(self): 48 | """Be called on the end of epochs.""" 49 | pass 50 | 51 | def on_step_end(self): 52 | """Update weights.""" 53 | self.update_weights() 54 | 55 | def compute_mask(self): 56 | """Compute masks according to current sparsity pattern.""" 57 | for weight in self.weights: 58 | tensor = self.model.get_weight(weight) 59 | if len(tensor.shape) in self.tensor_dims: 60 | self.masks[weight] = tensor == 0.0 61 | 62 | def update_weights(self): 63 | """Update weights according to the masks.""" 64 | for weight in self.weights: 65 | if weight in self.masks: 66 | self.model.prune_weights_(weight, self.masks[weight]) 67 | -------------------------------------------------------------------------------- /neural_compressor/experimental/pruner_legacy/util/block_mask.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Block mask.""" 18 | -------------------------------------------------------------------------------- /neural_compressor/experimental/pruning_recipes/__init__.py: -------------------------------------------------------------------------------- 1 | """patterns.""" 2 | #!/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2021 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | from .patterns import PATTERNS, patterns 21 | 22 | __all__ = [ 23 | "PATTERNS", 24 | "patterns", 25 | ] 26 | -------------------------------------------------------------------------------- /neural_compressor/experimental/pruning_recipes/patterns/__init__.py: -------------------------------------------------------------------------------- 1 | """Different patterns.""" 2 | #!/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2021 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | from os.path import dirname, basename, isfile, join 20 | import glob 21 | from .pattern import PATTERNS 22 | 23 | modules = glob.glob(join(dirname(__file__), "*.py")) 24 | 25 | for f in modules: 26 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 27 | __import__(basename(f)[:-3], globals(), locals(), level=1) 28 | 29 | patterns = PATTERNS() 30 | __all__ = ["PATTERNS"] 31 | -------------------------------------------------------------------------------- /neural_compressor/experimental/pytorch_pruner/__init__.py: -------------------------------------------------------------------------------- 1 | """PyTorch Pruner module.""" 2 | #!/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2022 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | -------------------------------------------------------------------------------- /neural_compressor/experimental/pytorch_pruner/logger.py: -------------------------------------------------------------------------------- 1 | """Logger module.""" 2 | #!/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | # 5 | # Copyright (c) 2022 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | from ...utils import logger 20 | -------------------------------------------------------------------------------- /neural_compressor/experimental/strategy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Intel Neural Compressor Strategy.""" 18 | 19 | from .strategy import EXP_STRATEGIES 20 | from os.path import dirname, basename, isfile, join 21 | import glob 22 | 23 | modules = glob.glob(join(dirname(__file__), "*.py")) 24 | 25 | for f in modules: 26 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 27 | __import__(basename(f)[:-3], globals(), locals(), level=1) 28 | 29 | __all__ = ["EXP_STRATEGIES"] 30 | -------------------------------------------------------------------------------- /neural_compressor/experimental/strategy/exhaustive.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """The exhaustive tuning strategy.""" 18 | from deprecated import deprecated 19 | 20 | from .strategy import TuneStrategy, strategy_registry 21 | from .utils.tuning_sampler import OpWiseTuningSampler 22 | 23 | 24 | @deprecated(version="2.0") 25 | @strategy_registry 26 | class ExhaustiveTuneStrategy(TuneStrategy): 27 | """The exhaustive tuning strategy.""" 28 | 29 | def next_tune_cfg(self): 30 | """Generate and yield the next tuning config using exhaustive search in tuning space. 31 | 32 | It sequentially traverse all possible quantization tuning configurations 33 | in a tuning space. From the perspective of the impact on performance, 34 | we currently only traverse all possible quantization tuning configs. 35 | Same reason as Bayesian, fallback datatypes are not included for now. 36 | 37 | Returns: 38 | tune_config (dict): A dict containing the tuning configuration for quantization. 39 | """ 40 | tuning_space = self.tuning_space 41 | calib_sampling_size_lst = tuning_space.root_item.get_option_by_name("calib_sampling_size").options 42 | for calib_sampling_size in calib_sampling_size_lst: 43 | op_item_dtype_dict, quant_mode_wise_items, initial_op_tuning_cfg = self.initial_tuning_cfg() 44 | op_wise_tuning_sampler = OpWiseTuningSampler( 45 | tuning_space, [], [], op_item_dtype_dict, initial_op_tuning_cfg 46 | ) 47 | for op_tuning_cfg in op_wise_tuning_sampler: 48 | op_tuning_cfg["calib_sampling_size"] = calib_sampling_size 49 | yield op_tuning_cfg 50 | return 51 | -------------------------------------------------------------------------------- /neural_compressor/experimental/strategy/random.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """The random tuning strategy.""" 18 | import numpy as np 19 | from deprecated import deprecated 20 | 21 | from ...utils import logger 22 | from .strategy import TuneStrategy, strategy_registry 23 | from .utils.tuning_sampler import FallbackTuningSampler, OpWiseTuningSampler 24 | from .utils.tuning_structs import OpTuningConfig 25 | 26 | 27 | @deprecated(version="2.0") 28 | @strategy_registry 29 | class RandomTuneStrategy(TuneStrategy): 30 | """The random tuning strategy.""" 31 | 32 | def next_tune_cfg(self): 33 | """Generate and yield the next tuning config by random searching in tuning space. 34 | 35 | Random strategy is used to randomly choose quantization tuning configurations 36 | from the tuning space. As with the Exhaustive strategy, it also only considers 37 | quantization tuning configs to generate a better-performance quantized model. 38 | 39 | Returns: 40 | tune_config (dict): A dict containing the tuning configuration for quantization. 41 | """ 42 | tuning_space = self.tuning_space 43 | op_item_dtype_dict, quant_mode_wise_items, initial_op_tuning_cfg = self.initial_tuning_cfg() 44 | op_wise_tuning_sampler = OpWiseTuningSampler(tuning_space, [], [], op_item_dtype_dict, initial_op_tuning_cfg) 45 | op_tuning_cfg_lst = list(op_wise_tuning_sampler) 46 | op_tuning_cfg_cnt = len(op_tuning_cfg_lst) 47 | calib_sampling_size_lst = tuning_space.root_item.get_option_by_name("calib_sampling_size").options 48 | calib_sampling_size_cnt = len(calib_sampling_size_lst) 49 | while True: 50 | calib_index = np.random.choice(calib_sampling_size_cnt) 51 | calib_sampling_size = calib_sampling_size_lst[calib_index] 52 | op_tuning_cfg_index = np.random.choice(op_tuning_cfg_cnt) 53 | op_tuning_cfg = op_tuning_cfg_lst[op_tuning_cfg_index] 54 | op_tuning_cfg["calib_sampling_size"] = calib_sampling_size 55 | yield op_tuning_cfg 56 | return 57 | -------------------------------------------------------------------------------- /neural_compressor/experimental/strategy/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | """Intel Neural Compressor Strategy Utils.""" 19 | 20 | from .tuning_sampler import TuningSampler, OpWiseTuningSampler, OpTypeWiseTuningSampler, FallbackTuningSampler 21 | from .tuning_structs import OpTuningConfig 22 | from .tuning_space import TuningItem, TuningSpace 23 | -------------------------------------------------------------------------------- /neural_compressor/experimental/strategy/utils/constant.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2023 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Strategy constant.""" 18 | 19 | PRECISION_SET = { 20 | "bf16", 21 | "fp16", 22 | "fp32", 23 | } 24 | QUANT_MODE_SET = {"static", "dynamic"} 25 | QUNAT_BIT_SET = {"int8", "uint8", "int4", "uint4"} 26 | 27 | TUNING_ITEMS_LST = [ 28 | ("activation", "scheme"), 29 | ("activation", "algorithm"), 30 | ("activation", "granularity"), 31 | ("weight", "scheme"), 32 | ("weight", "algorithm"), 33 | ("weight", "granularity"), 34 | "sampling_size", 35 | ] 36 | 37 | PRECISION_SET_V2_0 = {"fp32", "bf16"} 38 | 39 | auto_query_order = ["static", "dynamic", "bf16", "fp16", "fp32"] 40 | static_query_order = ["static", "bf16", "fp16", "fp32"] 41 | dynamic_query_order = ["dynamic", "bf16", "fp16", "fp32"] 42 | 43 | 44 | FALLBACK_RECIPES_SET = { 45 | "first_conv_or_matmul_quantization", 46 | "last_conv_or_matmul_quantization" "pre_post_process_quantization", 47 | } 48 | -------------------------------------------------------------------------------- /neural_compressor/experimental/strategy/utils/utility.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Tuning utility.""" 18 | 19 | 20 | from collections import OrderedDict 21 | 22 | from deprecated import deprecated 23 | 24 | 25 | @deprecated(version="2.0") 26 | class OrderedDefaultDict(OrderedDict): 27 | """Ordered default dict.""" 28 | 29 | def __missing__(self, key): 30 | """Initialize value for the missing key.""" 31 | self[key] = value = OrderedDefaultDict() 32 | return value 33 | 34 | 35 | @deprecated(version="2.0") 36 | def extract_data_type(data_type: str) -> str: 37 | """Extract data type and signed from data type. 38 | 39 | Args: 40 | data_type: The original data type such as uint8, int8. 41 | 42 | Returns: 43 | (signed or unsigned, data type without signed) 44 | """ 45 | return ("signed", data_type) if data_type[0] != "u" else ("unsigned", data_type[1:]) 46 | 47 | 48 | @deprecated(version="2.0") 49 | def reverted_data_type(signed_flag: str, data_type: str) -> str: 50 | """Revert the data type.""" 51 | return data_type if signed_flag == "signed" else "u" + data_type 52 | 53 | 54 | @deprecated(version="2.0") 55 | def get_adaptor_name(adaptor): 56 | """Get adaptor name. 57 | 58 | Args: 59 | adaptor: adaptor instance. 60 | """ 61 | adaptor_name = type(adaptor).__name__.lower() 62 | adaptor_name_lst = ["onnx", "tensorflow", "pytorch"] 63 | for name in adaptor_name_lst: 64 | if adaptor_name.startswith(name): 65 | return name 66 | return "" 67 | -------------------------------------------------------------------------------- /neural_compressor/metric/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Intel Neural Compressor Metric.""" 18 | 19 | from .metric import ( 20 | METRICS, 21 | Metric, 22 | BaseMetric, 23 | TensorflowTopK, 24 | metric_registry, 25 | COCOmAPv2, 26 | SquadF1, 27 | GeneralTopK, 28 | register_customer_metric, 29 | ) 30 | from os.path import dirname, basename, isfile, join 31 | import glob 32 | 33 | modules = glob.glob(join(dirname(__file__), "*.py")) 34 | 35 | for f in modules: 36 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 37 | __import__(basename(f)[:-3], globals(), locals(), level=1) 38 | 39 | 40 | __all__ = [ 41 | "METRICS", 42 | "Metric", 43 | "BaseMetric", 44 | "TensorflowTopK", 45 | "metric_registry", 46 | "COCOmAPv2", 47 | "SquadF1", 48 | "GeneralTopK", 49 | "register_customer_metric", 50 | ] 51 | -------------------------------------------------------------------------------- /neural_compressor/metric/coco_label_map.py: -------------------------------------------------------------------------------- 1 | # 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | # 19 | """The dict mapping category IDs to its names of labels.""" 20 | 21 | category_map = { 22 | 1: "person", 23 | 2: "bicycle", 24 | 3: "car", 25 | 4: "motorcycle", 26 | 5: "airplane", 27 | 6: "bus", 28 | 7: "train", 29 | 8: "truck", 30 | 9: "boat", 31 | 10: "traffic light", 32 | 11: "fire hydrant", 33 | 13: "stop sign", 34 | 14: "parking meter", 35 | 15: "bench", 36 | 16: "bird", 37 | 17: "cat", 38 | 18: "dog", 39 | 19: "horse", 40 | 20: "sheep", 41 | 21: "cow", 42 | 22: "elephant", 43 | 23: "bear", 44 | 24: "zebra", 45 | 25: "giraffe", 46 | 27: "backpack", 47 | 28: "umbrella", 48 | 31: "handbag", 49 | 32: "tie", 50 | 33: "suitcase", 51 | 34: "frisbee", 52 | 35: "skis", 53 | 36: "snowboard", 54 | 37: "sports ball", 55 | 38: "kite", 56 | 39: "baseball bat", 57 | 40: "baseball glove", 58 | 41: "skateboard", 59 | 42: "surfboard", 60 | 43: "tennis racket", 61 | 44: "bottle", 62 | 46: "wine glass", 63 | 47: "cup", 64 | 48: "fork", 65 | 49: "knife", 66 | 50: "spoon", 67 | 51: "bowl", 68 | 52: "banana", 69 | 53: "apple", 70 | 54: "sandwich", 71 | 55: "orange", 72 | 56: "broccoli", 73 | 57: "carrot", 74 | 58: "hot dog", 75 | 59: "pizza", 76 | 60: "donut", 77 | 61: "cake", 78 | 62: "chair", 79 | 63: "couch", 80 | 64: "potted plant", 81 | 65: "bed", 82 | 67: "dining table", 83 | 70: "toilet", 84 | 72: "tv", 85 | 73: "laptop", 86 | 74: "mouse", 87 | 75: "remote", 88 | 76: "keyboard", 89 | 77: "cell phone", 90 | 78: "microwave", 91 | 79: "oven", 92 | 80: "toaster", 93 | 81: "sink", 94 | 82: "refrigerator", 95 | 84: "book", 96 | 85: "clock", 97 | 86: "vase", 98 | 87: "scissors", 99 | 88: "teddy bear", 100 | 89: "hair drier", 101 | 90: "toothbrush", 102 | } 103 | -------------------------------------------------------------------------------- /neural_compressor/model/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Built-in model for multiple framework backends.""" 18 | 19 | from .model import MODELS, Model 20 | from .base_model import BaseModel 21 | 22 | __all__ = ["MODELS", "Model", "BaseModel"] 23 | -------------------------------------------------------------------------------- /neural_compressor/model/base_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Base model for multiple framework backends.""" 18 | 19 | from abc import abstractmethod 20 | 21 | 22 | class BaseModel: 23 | """Base class of all neural_compressor.model, will play graph role.""" 24 | 25 | def __init__(self, model, **kwargs): 26 | """Initialize a BaseModel. 27 | 28 | Args: 29 | model (object): raw model format. For Tensorflow model, could be path to frozen pb file, 30 | path to ckpt or savedmodel folder, loaded estimator/graph_def/graph/keras model object. 31 | For PyTorch model, it's torch.nn.model instance. For MXNet model, it's mxnet.symbol.Symbol 32 | or gluon.HybirdBlock instance. For ONNX model, it's path to onnx model or loaded ModelProto 33 | model object. 34 | """ 35 | self.component = None 36 | 37 | @property 38 | def model(self): 39 | """Return model itself.""" 40 | raise NotImplementedError 41 | 42 | @property 43 | def graph_info(self): 44 | """Return a dict with content 'Node: Node_type'.""" 45 | raise NotImplementedError 46 | 47 | @abstractmethod 48 | def save(self, root, *args, **kwargs): 49 | """Abstract method of model saving.""" 50 | raise NotImplementedError 51 | 52 | @abstractmethod 53 | def export( 54 | self, 55 | save_path: str, 56 | conf, 57 | ): 58 | """Abstract method of model conversion to ONNX.""" 59 | raise NotImplementedError 60 | 61 | @abstractmethod 62 | def framework(self): 63 | """Abstract method of model framework.""" 64 | raise NotImplementedError 65 | -------------------------------------------------------------------------------- /neural_compressor/model/mxnet_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | """Class for MXNet model.""" 19 | 20 | import os 21 | 22 | from neural_compressor.utils import logger 23 | from neural_compressor.utils.utility import LazyImport 24 | 25 | from .base_model import BaseModel 26 | 27 | mx = LazyImport("mxnet") 28 | 29 | 30 | class MXNetModel(BaseModel): 31 | """Build MXNet model.""" 32 | 33 | def __init__(self, model, **kwargs): 34 | """Initialize a MXNet model. 35 | 36 | Args: 37 | model (mxnet model): model path 38 | """ 39 | # (TODO) MXNet does not support recover model from tuning history currently 40 | self.q_config = None 41 | self._model = model 42 | self.calib_cache = {} 43 | 44 | def framework(self): 45 | """Return framework.""" 46 | return "mxnet" 47 | 48 | @property 49 | def model(self): 50 | """Return model itself.""" 51 | return self._model 52 | 53 | @model.setter 54 | def model(self, model): 55 | """Set model.""" 56 | self._model = model 57 | 58 | def save(self, root=None): 59 | """Save MXNet model.""" 60 | if root is None: 61 | from neural_compressor import config as cfg 62 | 63 | root = cfg.default_workspace 64 | root = os.path.abspath(os.path.expanduser(root)) 65 | os.makedirs(os.path.dirname(root), exist_ok=True) 66 | 67 | if isinstance(self._model, mx.gluon.HybridBlock): 68 | self._model.export(root, remove_amp_cast=False) 69 | logger.info("Save quantized hybrid block model to {}.".format(root)) 70 | else: 71 | symnet, args, auxs = self._model 72 | symnet = symnet.as_nd_ndarray() 73 | args = {k: v.as_nd_ndarray() for k, v in args.items()} 74 | auxs = {k: v.as_nd_ndarray() for k, v in auxs.items()} 75 | mx.model.save_checkpoint(root, 0, symnet, args, auxs, remove_amp_cast=False) 76 | logger.info("Save quantized symbol model to {}.".format(root)) 77 | -------------------------------------------------------------------------------- /neural_compressor/profiling/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Intel Neural Compressor Profiling.""" 16 | -------------------------------------------------------------------------------- /neural_compressor/profiling/parser/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /neural_compressor/profiling/parser/factory.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Profiling parser class factory.""" 16 | from typing import Optional 17 | 18 | from neural_compressor.model import BaseModel 19 | from neural_compressor.profiling.parser.onnx_parser.factory import OnnxrtParserFactory 20 | from neural_compressor.profiling.parser.parser import ProfilingParser 21 | from neural_compressor.profiling.parser.tensorflow_parser.factory import TensorFlowParserFactory 22 | 23 | 24 | class ParserFactory: 25 | """Parser factory.""" 26 | 27 | @staticmethod 28 | def get_parser( 29 | model: BaseModel, 30 | logs: list, 31 | ) -> Optional[ProfilingParser]: 32 | """Get parser for specified framework. 33 | 34 | Args: 35 | model: model to be profiled 36 | logs: list of path to logs 37 | 38 | Returns: 39 | ProfilingParser instance if model is supported else None 40 | """ 41 | framework_parser = { 42 | "tensorflow": TensorFlowParserFactory.get_parser, 43 | "onnxruntime": OnnxrtParserFactory.get_parser, 44 | } 45 | 46 | parser = framework_parser.get(model.framework(), None) 47 | if parser is None: 48 | raise Exception(f"Profiling Parser for '{model.framework()}' framework is not supported.") 49 | return parser(logs) 50 | -------------------------------------------------------------------------------- /neural_compressor/profiling/parser/onnx_parser/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /neural_compressor/profiling/parser/onnx_parser/factory.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Parser class factory.""" 16 | 17 | from typing import List, Optional 18 | 19 | from neural_compressor.profiling.parser.onnx_parser.parser import OnnxProfilingParser 20 | 21 | 22 | class OnnxrtParserFactory: 23 | """Parser factory.""" 24 | 25 | @staticmethod 26 | def get_parser( 27 | logs: List, 28 | *args, 29 | **kwargs, 30 | ) -> Optional[OnnxProfilingParser]: 31 | """Get ProfilingParser for specified workload. 32 | 33 | Args: 34 | logs: list of path to logs 35 | *args: list with additional arguments 36 | **kwargs: dict with named arguments 37 | 38 | Returns: 39 | OnnxProfilingParser instance if model is supported else None 40 | """ 41 | return OnnxProfilingParser(logs) 42 | -------------------------------------------------------------------------------- /neural_compressor/profiling/parser/parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Parsers for log files.""" 17 | from abc import ABC, abstractmethod 18 | from typing import List 19 | 20 | from neural_compressor.profiling.parser.result import ProfilingResult 21 | 22 | 23 | class ProfilingParser(ABC): 24 | """Parser class is responsible for parsing profiling log files.""" 25 | 26 | def __init__(self, logs: list) -> None: 27 | """Initialize parser. 28 | 29 | Args: 30 | logs: list of path to logs 31 | 32 | Returns: 33 | None 34 | """ 35 | self._logs = logs 36 | self._results: List[ProfilingResult] = [] 37 | 38 | @property 39 | def results(self) -> List[ProfilingResult]: 40 | """Get profiling results. 41 | 42 | Returns: list of ProfilingResult entries 43 | """ 44 | return self._results 45 | 46 | def add_result(self, result: ProfilingResult) -> None: 47 | """Add result to list. 48 | 49 | Args: 50 | result: ProfilingResult object 51 | 52 | Returns: 53 | None 54 | """ 55 | self._results.append(result) 56 | 57 | @abstractmethod 58 | def process(self) -> List[dict]: 59 | """Process profiling logs. 60 | 61 | Returns: 62 | list of dicts with processed profiling results 63 | """ 64 | raise NotImplementedError 65 | 66 | def _serialize_results(self) -> list: 67 | """Serialize Results list. 68 | 69 | Returns: 70 | list of serialized results 71 | """ 72 | sorted_results = sorted( 73 | self.results, 74 | key=lambda result: result.total_execution_time, 75 | reverse=True, 76 | ) 77 | return [result.serialize() for result in sorted_results] 78 | -------------------------------------------------------------------------------- /neural_compressor/profiling/parser/result.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Profiling Result class.""" 17 | 18 | 19 | class ProfilingResult: 20 | """Profiling result class.""" 21 | 22 | def __init__( 23 | self, 24 | node_name: str, 25 | total_execution_time: int, 26 | accelerator_execution_time: int, 27 | cpu_execution_time: int, 28 | op_run: int, 29 | op_defined: int, 30 | ) -> None: 31 | """Create profiling result instance. 32 | 33 | Args: 34 | node_name: name of the node 35 | total_execution_time: sum of cpu and accelerator execution times in us 36 | accelerator_execution_time: accelerator execution time in us 37 | cpu_execution_time: cpu execution time in us 38 | op_run: number of run OPs 39 | op_defined: number of defined OPs 40 | 41 | Returns: 42 | None 43 | """ 44 | self.node_name = node_name 45 | self.total_execution_time = total_execution_time 46 | self.accelerator_execution_time = accelerator_execution_time 47 | self.cpu_execution_time = cpu_execution_time 48 | self.op_run = op_run 49 | self.op_defined = op_defined 50 | 51 | def serialize(self) -> dict: 52 | """Serialize profiling result. 53 | 54 | Returns: 55 | Serialized profiling result 56 | """ 57 | return { 58 | "node_name": self.node_name, 59 | "total_execution_time": self.total_execution_time, 60 | "accelerator_execution_time": self.accelerator_execution_time, 61 | "cpu_execution_time": self.cpu_execution_time, 62 | "op_run": self.op_run, 63 | "op_defined": self.op_defined, 64 | } 65 | -------------------------------------------------------------------------------- /neural_compressor/profiling/parser/tensorflow_parser/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /neural_compressor/profiling/parser/tensorflow_parser/factory.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Parser class factory.""" 16 | 17 | from typing import List, Optional 18 | 19 | from neural_compressor.profiling.parser.tensorflow_parser.parser import TensorFlowProfilingParser 20 | 21 | 22 | class TensorFlowParserFactory: 23 | """Parser factory.""" 24 | 25 | @staticmethod 26 | def get_parser( 27 | logs: List, 28 | *args, 29 | **kwargs, 30 | ) -> Optional[TensorFlowProfilingParser]: 31 | """Get ProfilingParser for specified workload. 32 | 33 | Args: 34 | logs: list of path to logs 35 | *args: list with additional arguments 36 | **kwargs: dict with named arguments 37 | 38 | Returns: 39 | TensorFlowProfilingParser instance if model is supported else None 40 | """ 41 | return TensorFlowProfilingParser(logs) 42 | -------------------------------------------------------------------------------- /neural_compressor/profiling/profiler/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /neural_compressor/profiling/profiler/factory.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2021-2022 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Profiling class factory.""" 16 | from typing import Optional 17 | 18 | from neural_compressor.data.dataloaders.base_dataloader import BaseDataLoader 19 | from neural_compressor.model import BaseModel 20 | from neural_compressor.profiling.profiler.onnxrt_profiler.factory import ProfilerFactory as OnnxrtProfilerFactory 21 | from neural_compressor.profiling.profiler.profiler import Profiler 22 | from neural_compressor.profiling.profiler.tensorflow_profiler.factory import ( 23 | ProfilerFactory as TensorflowProfilerFactory, 24 | ) 25 | 26 | 27 | class ProfilerFactory: 28 | """Profiling factory.""" 29 | 30 | @staticmethod 31 | def get_profiler( 32 | model: BaseModel, 33 | dataloader: BaseDataLoader, 34 | log_file: Optional[str] = None, 35 | ) -> Optional[Profiler]: 36 | """Get profiling for specified framework. 37 | 38 | Args: 39 | model: model to be profiled 40 | dataloader: DataLoader object 41 | log_file: optional path to log file 42 | 43 | Returns: 44 | Profiler instance if model is supported else None 45 | """ 46 | framework_profilers = { 47 | "tensorflow": TensorflowProfilerFactory.get_profiler, 48 | "onnxruntime": OnnxrtProfilerFactory.get_profiler, 49 | } 50 | 51 | profiler = framework_profilers.get(model.framework(), None) 52 | if profiler is None: 53 | raise Exception(f"Profiling for '{model.framework()}' framework is not supported.") 54 | return profiler(**locals()) 55 | -------------------------------------------------------------------------------- /neural_compressor/profiling/profiler/onnxrt_profiler/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /neural_compressor/profiling/profiler/onnxrt_profiler/factory.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Profiling class factory.""" 16 | 17 | from typing import Optional 18 | 19 | from neural_compressor.data.dataloaders.onnxrt_dataloader import ONNXRTDataLoader 20 | from neural_compressor.model.onnx_model import ONNXModel 21 | from neural_compressor.profiling.profiler.onnxrt_profiler.profiler import Profiler 22 | 23 | 24 | class ProfilerFactory: 25 | """Profiler factory.""" 26 | 27 | @staticmethod 28 | def get_profiler( 29 | model: ONNXModel, 30 | dataloader: ONNXRTDataLoader, 31 | log_file: Optional[str] = None, 32 | *args, 33 | **kwargs, 34 | ) -> Profiler: 35 | """Get profiling for specified framework. 36 | 37 | Args: 38 | model: model to be profiled 39 | dataloader: DataLoader object 40 | log_file: optional path to log file 41 | 42 | Returns: 43 | Profiler instance if model is supported else None 44 | """ 45 | return Profiler(model, dataloader, log_file) 46 | -------------------------------------------------------------------------------- /neural_compressor/profiling/profiler/onnxrt_profiler/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """ONNX profiler utils.""" 16 | 17 | from typing import Any 18 | 19 | 20 | def create_onnx_config(ort: Any, intra_num_of_threads: int, inter_num_of_threads: int) -> Any: 21 | """Create tensorflow config. 22 | 23 | Args: 24 | ort: onnxruntime library 25 | intra_num_of_threads: number of threads used within an individual op for parallelism 26 | inter_num_of_threads: number of threads used for parallelism between independent operations 27 | 28 | Returns: 29 | ONNX SessionOptions object 30 | """ 31 | options = ort.SessionOptions() 32 | options.enable_profiling = True 33 | options.intra_op_num_threads = intra_num_of_threads 34 | options.inter_op_num_threads = inter_num_of_threads 35 | return options 36 | -------------------------------------------------------------------------------- /neural_compressor/profiling/profiler/profiler.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Generic profiler.""" 16 | 17 | 18 | class Profiler: 19 | """Profiler class.""" 20 | 21 | def __init__(self, input_graph: str) -> None: 22 | """Profiler constructor. 23 | 24 | Args: 25 | input_graph: path to model. 26 | 27 | Returns: 28 | None 29 | """ 30 | self.input_graph = input_graph 31 | 32 | def profile_model(self, *args, **kwargs) -> None: 33 | """Profile model.""" 34 | raise NotImplementedError 35 | -------------------------------------------------------------------------------- /neural_compressor/profiling/profiler/tensorflow_profiler/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /neural_compressor/profiling/profiler/tensorflow_profiler/factory.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2021-2022 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Profiling class factory.""" 16 | 17 | from typing import Optional 18 | 19 | from neural_compressor.data.dataloaders.tensorflow_dataloader import TensorflowDataLoader 20 | from neural_compressor.model.tensorflow_model import TensorflowBaseModel 21 | from neural_compressor.profiling.profiler.profiler import Profiler 22 | from neural_compressor.profiling.profiler.tensorflow_profiler.profiler import Profiler as FrozenPbProfiler 23 | 24 | 25 | class ProfilerFactory: 26 | """Profiler factory.""" 27 | 28 | @staticmethod 29 | def get_profiler( 30 | model: TensorflowBaseModel, 31 | dataloader: TensorflowDataLoader, 32 | log_file: Optional[str] = None, 33 | *args, 34 | **kwargs, 35 | ) -> Profiler: 36 | """Get profiling for specified framework. 37 | 38 | Args: 39 | model: model to be profiled 40 | dataloader: DataLoader object 41 | log_file: optional path to log file 42 | 43 | Returns: 44 | Profiler instance if model is supported else None 45 | """ 46 | framework_profilers = { 47 | "frozen_pb": FrozenPbProfiler, 48 | } 49 | 50 | profiler = framework_profilers.get(model.model_type, None) 51 | if profiler is None: 52 | raise Exception(f"Profiling for '{model.model_type}' model type is not supported.") 53 | return profiler(model, dataloader, log_file) 54 | -------------------------------------------------------------------------------- /neural_compressor/strategy/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Intel Neural Compressor Strategy.""" 18 | 19 | from .strategy import STRATEGIES 20 | from os.path import dirname, basename, isfile, join 21 | import glob 22 | 23 | modules = glob.glob(join(dirname(__file__), "*.py")) 24 | 25 | for f in modules: 26 | if isfile(f) and not f.startswith("__") and not f.endswith("__init__.py"): 27 | __import__(basename(f)[:-3], globals(), locals(), level=1) 28 | 29 | __all__ = ["STRATEGIES"] 30 | -------------------------------------------------------------------------------- /neural_compressor/strategy/exhaustive.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """The exhaustive tuning strategy.""" 18 | from .strategy import TuneStrategy, strategy_registry 19 | from .utils.tuning_sampler import OpWiseTuningSampler 20 | 21 | 22 | @strategy_registry 23 | class ExhaustiveTuneStrategy(TuneStrategy): 24 | """The exhaustive tuning strategy.""" 25 | 26 | def next_tune_cfg(self): 27 | """Generate and yield the next tuning config using exhaustive search in tuning space. 28 | 29 | It sequentially traverse all possible quantization tuning configurations 30 | in a tuning space. From the perspective of the impact on performance, 31 | we currently only traverse all possible quantization tuning configs. 32 | Same reason as Bayesian, fallback datatypes are not included for now. 33 | 34 | Returns: 35 | tune_config (dict): A dict containing the tuning configuration for quantization. 36 | """ 37 | tuning_space = self.tuning_space 38 | calib_sampling_size_lst = tuning_space.root_item.get_option_by_name("calib_sampling_size").options 39 | for calib_sampling_size in calib_sampling_size_lst: 40 | op_item_dtype_dict, quant_mode_wise_items, initial_op_tuning_cfg = self.initial_tuning_cfg() 41 | op_wise_tuning_sampler = OpWiseTuningSampler( 42 | tuning_space, [], [], op_item_dtype_dict, initial_op_tuning_cfg 43 | ) 44 | for op_tuning_cfg in op_wise_tuning_sampler: 45 | op_tuning_cfg["calib_sampling_size"] = calib_sampling_size 46 | yield op_tuning_cfg 47 | -------------------------------------------------------------------------------- /neural_compressor/strategy/random.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """The random tuning strategy.""" 18 | 19 | import numpy as np 20 | 21 | from .strategy import TuneStrategy, strategy_registry 22 | from .utils.tuning_sampler import OpWiseTuningSampler 23 | 24 | 25 | @strategy_registry 26 | class RandomTuneStrategy(TuneStrategy): 27 | """The random tuning strategy.""" 28 | 29 | def next_tune_cfg(self): 30 | """Generate and yield the next tuning config by random searching in tuning space. 31 | 32 | Random strategy is used to randomly choose quantization tuning configurations 33 | from the tuning space. As with the Exhaustive strategy, it also only considers 34 | quantization tuning configs to generate a better-performance quantized model. 35 | 36 | Returns: 37 | tune_config (dict): A dict containing the tuning configuration for quantization. 38 | """ 39 | tuning_space = self.tuning_space 40 | op_item_dtype_dict, quant_mode_wise_items, initial_op_tuning_cfg = self.initial_tuning_cfg() 41 | op_wise_tuning_sampler = OpWiseTuningSampler(tuning_space, [], [], op_item_dtype_dict, initial_op_tuning_cfg) 42 | op_tuning_cfg_lst = list(op_wise_tuning_sampler) 43 | op_tuning_cfg_cnt = len(op_tuning_cfg_lst) 44 | calib_sampling_size_lst = tuning_space.root_item.get_option_by_name("calib_sampling_size").options 45 | calib_sampling_size_cnt = len(calib_sampling_size_lst) 46 | while True: 47 | calib_index = np.random.choice(calib_sampling_size_cnt) 48 | calib_sampling_size = calib_sampling_size_lst[calib_index] 49 | op_tuning_cfg_index = np.random.choice(op_tuning_cfg_cnt) 50 | op_tuning_cfg = op_tuning_cfg_lst[op_tuning_cfg_index] 51 | op_tuning_cfg["calib_sampling_size"] = calib_sampling_size 52 | yield op_tuning_cfg 53 | return 54 | -------------------------------------------------------------------------------- /neural_compressor/strategy/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | """Intel Neural Compressor Strategy Utils.""" 19 | 20 | from .tuning_sampler import TuningSampler, OpWiseTuningSampler, OpTypeWiseTuningSampler, FallbackTuningSampler 21 | from .tuning_structs import OpTuningConfig 22 | from .tuning_space import TuningItem, TuningSpace 23 | -------------------------------------------------------------------------------- /neural_compressor/strategy/utils/constant.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2023 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Strategy constant.""" 18 | 19 | 20 | PRECISION_LIST = ["bf16", "fp16", "fp32"] 21 | QUANT_MODE_SET = {"static", "dynamic"} 22 | LOWER_BIT_LIST = ["int4"] 23 | 24 | TUNING_ITEMS_LST = [ 25 | ("activation", "scheme"), 26 | ("activation", "algorithm"), 27 | ("activation", "granularity"), 28 | ("weight", "scheme"), 29 | ("weight", "algorithm"), 30 | ("weight", "granularity"), 31 | "sampling_size", 32 | ] 33 | WEIGHT_ONLY_TUNING_ITEMS_LST = [ 34 | ("activation", "scheme"), 35 | ("activation", "algorithm"), 36 | ("activation", "granularity"), 37 | ("weight", "scheme"), 38 | ("weight", "algorithm"), 39 | ("weight", "granularity"), 40 | ("weight", "bits"), 41 | ("weight", "group_size"), 42 | ("weight", "dtype"), 43 | "sampling_size", 44 | ] 45 | 46 | PRECISION_SET_V2_0 = {"fp32", "bf16", "fp16"} 47 | 48 | auto_query_order = ["static", "dynamic", "bf16", "fp16", "fp32"] 49 | static_query_order = ["static", "bf16", "fp16", "fp32"] 50 | dynamic_query_order = ["dynamic", "bf16", "fp16", "fp32"] 51 | auto_query_order_o0 = ["bf16", "fp16", "fp32", "static", "dynamic"] 52 | weight_only_query_order = ["weight_only", "fp32"] 53 | 54 | 55 | FALLBACK_RECIPES_SET = { 56 | "first_conv_or_matmul_quantization", 57 | "last_conv_or_matmul_quantization", 58 | "pre_post_process_quantization", 59 | } 60 | 61 | 62 | WOQ_TUNING_ALGOS = { 63 | "RTN_G32ASYM": {"algorithm": "RTN", "group_size": 32, "scheme": "asym"}, 64 | "GPTQ_G32ASYM": {"algorithm": "GPTQ", "group_size": 32, "scheme": "asym"}, 65 | "GPTQ_G32ASYM_DISABLE_LAST_MATMUL": {"algorithm": "GPTQ", "group_size": 32, "scheme": "asym"}, 66 | "GPTQ_G128ASYM": {"algorithm": "GPTQ", "group_size": 128, "scheme": "asym"}, 67 | "AWQ_G32ASYM": {"algorithm": "AWQ", "group_size": 32, "scheme": "asym"}, 68 | } 69 | -------------------------------------------------------------------------------- /neural_compressor/template/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /neural_compressor/tensorflow/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from neural_compressor.tensorflow.utils import register_algo 16 | from neural_compressor.tensorflow.algorithms import static_quantize_entry 17 | from neural_compressor.tensorflow.quantization import quantize_model, StaticQuantConfig, get_default_static_quant_config 18 | -------------------------------------------------------------------------------- /neural_compressor/tensorflow/algorithms/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from neural_compressor.tensorflow.algorithms.static_quantize import static_quantize_entry 17 | -------------------------------------------------------------------------------- /neural_compressor/tensorflow/algorithms/static_quantize/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from neural_compressor.tensorflow.algorithms.static_quantize.quantize_entry import static_quantize_entry 16 | -------------------------------------------------------------------------------- /neural_compressor/tensorflow/algorithms/static_quantize/keras_utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | -------------------------------------------------------------------------------- /neural_compressor/tensorflow/algorithms/static_quantize/keras_utils/pool2d.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | import json 19 | 20 | import tensorflow as tf 21 | from tensorflow import quantization 22 | from tensorflow.keras import activations, backend, constraints, initializers, regularizers 23 | from tensorflow.keras.layers import AveragePooling2D, MaxPooling2D 24 | 25 | 26 | class QAvgPool2D(AveragePooling2D): 27 | def __init__( 28 | self, 29 | pool_size=(2, 2), 30 | strides=None, 31 | padding="valid", 32 | data_format=None, 33 | min_value=-10000, 34 | max_value=10000, 35 | **kwargs 36 | ): 37 | super(QAvgPool2D, self).__init__( 38 | pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, **kwargs 39 | ) 40 | self.min_value = json.loads(min_value) 41 | self.max_value = json.loads(max_value) 42 | 43 | 44 | class QMaxPool2D(MaxPooling2D): 45 | def __init__( 46 | self, 47 | pool_size=(2, 2), 48 | strides=None, 49 | padding="valid", 50 | data_format=None, 51 | min_value=-10000, 52 | max_value=10000, 53 | **kwargs 54 | ): 55 | super(QMaxPool2D, self).__init__( 56 | pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, **kwargs 57 | ) 58 | self.min_value = json.loads(min_value) 59 | self.max_value = json.loads(max_value) 60 | -------------------------------------------------------------------------------- /neural_compressor/tensorflow/quantization/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from neural_compressor.tensorflow.quantization.quantize import quantize_model 16 | from neural_compressor.tensorflow.quantization.config import StaticQuantConfig, get_default_static_quant_config 17 | -------------------------------------------------------------------------------- /neural_compressor/tensorflow/quantization/quantize.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import Any, Callable 16 | 17 | import tensorflow as tf 18 | 19 | from neural_compressor.common.base_config import BaseConfig 20 | from neural_compressor.common.logger import Logger 21 | from neural_compressor.common.utility import STATIC_QUANT 22 | from neural_compressor.tensorflow.quantization.config import parse_config_from_dict 23 | from neural_compressor.tensorflow.utils import algos_mapping 24 | 25 | logger = Logger().get_logger() 26 | 27 | 28 | def quantize_model( 29 | model: tf.keras.Model, quant_config: BaseConfig, calib_dataloader: Callable = None, calib_iteration: int = 100 30 | ) -> tf.keras.Model: 31 | """The main entry to quantize model. 32 | 33 | Args: 34 | model: a fp32 model to be quantized. 35 | quant_config: a quantization configuration. 36 | calib_dataloader: a data loader for calibration. 37 | calib_iteration: the iteration of calibration. 38 | 39 | Returns: 40 | q_model: the quantized model. 41 | """ 42 | if isinstance(quant_config, dict): 43 | quant_config = parse_config_from_dict(quant_config) 44 | logger.info("Parsed dict to construct the quantization config.") 45 | else: 46 | assert isinstance( 47 | quant_config, BaseConfig 48 | ), "Please pass a dict or config instance as the quantization configuration." 49 | logger.info(f"Quantize model with config: \n {quant_config.to_json_string()} \n") 50 | 51 | # select quantization algo according to config 52 | # TODO (Yi) support combine more than one algo 53 | if quant_config.name == STATIC_QUANT: 54 | quant_fn = algos_mapping[quant_config.name] 55 | else: 56 | raise NotImplementedError("Currently, only the basic algorithm is being ported.") 57 | qmodel = quant_fn(model, quant_config, calib_dataloader, calib_iteration) 58 | return qmodel 59 | -------------------------------------------------------------------------------- /neural_compressor/torch/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from neural_compressor.torch.utils.utility import register_algo 16 | from neural_compressor.torch.algorithms import rtn_quantize_entry, gptq_quantize_entry 17 | 18 | from neural_compressor.torch.quantization import ( 19 | quantize, 20 | RTNWeightQuantConfig, 21 | get_default_rtn_config, 22 | GPTQConfig, 23 | get_default_gptq_config, 24 | ) 25 | 26 | from neural_compressor.torch.tune import autotune, TuningConfig, get_default_tune_config 27 | -------------------------------------------------------------------------------- /neural_compressor/torch/algorithms/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from neural_compressor.torch.algorithms.weight_only_algos import rtn_quantize_entry 17 | from neural_compressor.torch.algorithms.weight_only_algos import gptq_quantize_entry 18 | -------------------------------------------------------------------------------- /neural_compressor/torch/algorithms/weight_only/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /neural_compressor/torch/algorithms/weight_only_algos.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from typing import Dict, Tuple 17 | 18 | import torch 19 | 20 | from neural_compressor.common.logger import Logger 21 | from neural_compressor.common.utility import GPTQ, RTN_WEIGHT_ONLY_QUANT 22 | from neural_compressor.torch.quantization.config import GPTQConfig, RTNWeightQuantConfig 23 | from neural_compressor.torch.utils.utility import fetch_module, register_algo, set_module 24 | 25 | logger = Logger().get_logger() 26 | 27 | 28 | ###################### RTN Algo Entry ################################## 29 | @register_algo(name=RTN_WEIGHT_ONLY_QUANT) 30 | def rtn_quantize_entry( 31 | model: torch.nn.Module, configs_mapping: Dict[Tuple[str, callable], RTNWeightQuantConfig], *args, **kwargs 32 | ) -> torch.nn.Module: 33 | """The main entry to apply rtn quantization.""" 34 | from .weight_only.rtn import apply_rtn_on_single_module 35 | 36 | for (op_name, op_type), quant_config in configs_mapping.items(): 37 | original_module = fetch_module(model, op_name) 38 | if original_module is None: 39 | continue 40 | logger.info(f"Apply RTN on module: {op_name}, {original_module}") 41 | rtn_module = apply_rtn_on_single_module(original_module, quant_config) 42 | set_module(model, op_name, rtn_module) 43 | return model 44 | 45 | 46 | ###################### GPTQ Algo Entry ################################## 47 | @register_algo(name=GPTQ) 48 | def gptq_quantize_entry( 49 | model: torch.nn.Module, configs_mapping: Dict[Tuple[str, callable], GPTQConfig], *args, **kwargs 50 | ) -> torch.nn.Module: 51 | logger.info("Quantize model with the GPTQ algorithm.") 52 | from .weight_only.gptq import apply_gptq_quantize 53 | 54 | model, quantization_perm = apply_gptq_quantize(model=model, configs_mapping=configs_mapping, *args, **kwargs) 55 | # Assign the gptq config as an attribute of model 56 | model._gptq_quantization_perm = quantization_perm 57 | return model 58 | -------------------------------------------------------------------------------- /neural_compressor/torch/amp/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .autocast import autocast 16 | -------------------------------------------------------------------------------- /neural_compressor/torch/amp/fp8/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /neural_compressor/torch/quantization/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from neural_compressor.torch.quantization.quantize import quantize, quantize_dynamic 16 | from neural_compressor.torch.quantization.config import ( 17 | RTNWeightQuantConfig, 18 | get_default_rtn_config, 19 | GPTQConfig, 20 | get_default_gptq_config, 21 | ) 22 | -------------------------------------------------------------------------------- /neural_compressor/torch/quantization/fp8/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .quantization_impl import quantize_dynamic, quantize 16 | -------------------------------------------------------------------------------- /neural_compressor/torch/tune.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import Callable, Dict, List, Optional, Tuple, Union 16 | 17 | import torch 18 | 19 | from neural_compressor.common.base_tune import BaseTuningConfig, FrameworkWrapper, Tuner, tuning_objectives 20 | from neural_compressor.common.logger import Logger 21 | from neural_compressor.torch.quantization.config import GPTQConfig, RTNWeightQuantConfig 22 | 23 | logger = Logger().get_logger() 24 | 25 | 26 | def get_default_tuning_config(): 27 | # TODO (Yi) support it in the next PR 28 | return None 29 | 30 | 31 | class TorchWrapper(FrameworkWrapper): 32 | """Concrete implementation of `FrameworkWrapper` for PyTorch models.""" 33 | 34 | def __init__( 35 | self, model: torch.nn.Module, run_fn: Optional[Callable] = None, run_args: Optional[Tuple] = None 36 | ) -> None: 37 | super().__init__(model) 38 | self.run_fn = run_fn 39 | self.run_args = run_args 40 | 41 | def apply(self, quant_config): 42 | """The entry to apply quantization algorithms on a given a model.""" 43 | logger.info(f"apply quant_config: {quant_config}.") 44 | from neural_compressor.torch import quantize 45 | 46 | q_model = quantize(model=self.model, quant_config=quant_config, run_fn=self.run_fn, run_args=self.run_args) 47 | return q_model 48 | 49 | 50 | class TuningConfig(BaseTuningConfig): 51 | def __init__(self, quant_configs=None, timeout=0, max_trials=100): 52 | super().__init__(quant_configs, timeout, max_trials) 53 | 54 | 55 | def autotune( 56 | model: torch.nn.Module, 57 | tune_config: TuningConfig, 58 | eval_fns: Optional[Union[Dict, List[Dict]]] = None, 59 | run_fn=None, 60 | run_args=None, 61 | ): 62 | tuning_objectives.set_eval_fn_registry(eval_fns) 63 | torch_wrapper = TorchWrapper(model, run_fn, run_args) 64 | tuner = Tuner(tune_config=tune_config, tuning_objectives=tuning_objectives, fwk_wrapper=torch_wrapper) 65 | best_qmodel = tuner.search() 66 | return best_qmodel 67 | 68 | 69 | def get_default_tune_config(): 70 | # TODO use the registered default tuning config in the next PR 71 | return TuningConfig(quant_configs=[GPTQConfig(weight_bits=[4, 8]), RTNWeightQuantConfig(weight_bits=[4, 8])]) 72 | -------------------------------------------------------------------------------- /neural_compressor/torch/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Intel Corporation 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /neural_compressor/torch/utils/constants.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # double quant params 18 | 19 | DOUBLE_QUANT_CONFIGS = { 20 | "GGML_TYPE_Q4_K": { 21 | "weight_dtype": "int", 22 | "weight_bits": 4, 23 | "weight_group_size": 32, 24 | "double_quant_bits": 6, 25 | "double_quant_dtype": "int", 26 | "double_quant_sym": True, 27 | "double_quant_group_size": 8, 28 | }, 29 | "BNB": { 30 | "weight_dtype": "nf4", 31 | "weight_bits": 4, 32 | "weight_group_size": 32, 33 | "double_quant_bits": 8, 34 | "double_quant_dtype": "int", 35 | "double_quant_sym": False, 36 | "double_quant_group_size": 256, 37 | }, 38 | } 39 | -------------------------------------------------------------------------------- /neural_compressor/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Utils: provide useful methods and auxiliary functionalities.""" 18 | 19 | from .collect_layer_histogram import LayerHistogramCollector 20 | from .logger import log, info, debug, warn, warning, error, fatal 21 | from .options import OPTIONS 22 | from .utility import alias_param 23 | 24 | __all__ = [ 25 | "LayerHistogramCollector", 26 | "log", 27 | "info", 28 | "debug", 29 | "warn", 30 | "warning", 31 | "error", 32 | "fatal", 33 | "OPTIONS", 34 | "alias_param", 35 | ] 36 | -------------------------------------------------------------------------------- /neural_compressor/utils/options.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """ONNX options.""" 18 | 19 | from ..conf.dotdict import DotDict 20 | 21 | 22 | class onnxrt: 23 | """ONNX helper configuration.""" 24 | 25 | graph_optimization = DotDict({"level": None, "gemm2matmul": True}) 26 | qdq_setting = DotDict( 27 | {"OpTypesToExcludeOutputQuantizatioin": [], "AddQDQPairToWeight": False, "DedicatedQDQPair": False} 28 | ) 29 | 30 | 31 | OPTIONS = { 32 | "tensorflow": None, 33 | "tensorflow_itex": None, 34 | "pytorch": None, 35 | "pytorch_fx": None, 36 | "pytorch_ipex": None, 37 | "mxnet": None, 38 | "onnxrt_integerops": onnxrt, 39 | "onnxrt_qlinearops": onnxrt, 40 | "onnxrt_qdq": onnxrt, 41 | "onnxruntime": onnxrt, 42 | } 43 | -------------------------------------------------------------------------------- /neural_compressor/utils/weights_details.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright (c) 2023 Intel Corporation 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """The Weights details class.""" 16 | from typing import Any, Dict, Optional 17 | 18 | import numpy as np 19 | 20 | from neural_compressor.utils.utility import mse_metric_gap 21 | 22 | PRECISION = 5 23 | 24 | 25 | class WeightsDetails: 26 | """Weights details class.""" 27 | 28 | def __init__(self, op_name: str, input_tensor_data, optimized_tensor_data): 29 | """Initialize Weights details.""" 30 | self.op_name: str = op_name 31 | self.mse: Optional[float] = mse_metric_gap( 32 | input_tensor_data, 33 | optimized_tensor_data, 34 | ) 35 | self.input_stats: WeightsStatistics = WeightsStatistics(input_tensor_data) 36 | self.optimized_stats: WeightsStatistics = WeightsStatistics(optimized_tensor_data) 37 | 38 | def serialize(self) -> Dict[str, Any]: 39 | """Serialize Weights details.""" 40 | return { 41 | "OP name": self.op_name, 42 | "MSE": self.mse, 43 | "Input model": self.input_stats.serialize(), 44 | "Optimized model": self.optimized_stats.serialize(), 45 | } 46 | 47 | 48 | class WeightsStatistics: 49 | """Weights statistics class.""" 50 | 51 | def __init__(self, tensor_data) -> None: 52 | """Initialize Weights details. 53 | 54 | Args: 55 | tensor_data: numpy array with tensor data 56 | 57 | Returns: 58 | None 59 | """ 60 | self.min: float = np.min(tensor_data) 61 | self.max: float = np.max(tensor_data) 62 | self.mean: float = np.mean(tensor_data) 63 | self.std: float = np.std(tensor_data) 64 | self.var: float = np.var(tensor_data) 65 | 66 | def serialize(self) -> Dict[str, Any]: 67 | """Serialize Weights details. 68 | 69 | Returns: 70 | Dictionary with serialized WeightsStatistics object 71 | """ 72 | return { 73 | "Min weight": round(self.min, PRECISION), 74 | "Max weight": round(self.max, PRECISION), 75 | "Mean": round(self.mean, PRECISION), 76 | "Standard deviation": round(self.std, PRECISION), 77 | "Variance": round(self.var, PRECISION), 78 | } 79 | -------------------------------------------------------------------------------- /neural_compressor/version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2021 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """Intel® Neural Compressor: An open-source Python library supporting popular model compression techniques.""" 18 | __version__ = "2.5" 19 | -------------------------------------------------------------------------------- /parse_results.py: -------------------------------------------------------------------------------- 1 | import re 2 | import sys 3 | import glob 4 | import os 5 | 6 | def parse_acc_values(file_path): 7 | # Pattern to match acc values directly 8 | pattern = re.compile(r'\|\s*(?:[\w_]+\s*\|\s*\d+\s*\|)?acc\s*\|\s*([\d.]+)\s*\|') 9 | acc_values = [] 10 | 11 | with open(file_path, 'r') as file: 12 | for line in file: 13 | match = pattern.search(line) 14 | if match: 15 | acc_values.append(match.group(1)) 16 | 17 | return acc_values 18 | 19 | def main(input_directory, output_file): 20 | # Find all *.out files in the input directory and sort them alphabetically 21 | input_files = sorted(glob.glob(os.path.join(input_directory, '*.out'))) 22 | 23 | with open(output_file, 'w') as out_file: 24 | for file_path in input_files: 25 | acc_values = parse_acc_values(file_path) 26 | # Include the filename (without the path) before the values 27 | file_name = os.path.basename(file_path) 28 | out_file.write(file_name + '\n' + ','.join(acc_values) + '\n\n') 29 | 30 | # Example usage: python script.py /path/to/input/directory /path/to/output.csv 31 | if __name__ == "__main__": 32 | input_directory_path = sys.argv[1] # First argument is the input directory path 33 | output_file_path = sys.argv[2] # Second argument is the output file path 34 | 35 | main(input_directory_path, output_file_path) 36 | -------------------------------------------------------------------------------- /requirements.yaml: -------------------------------------------------------------------------------- 1 | name: llm-datatypes 2 | channels: 3 | - pytorch 4 | - anaconda 5 | - nvidia 6 | - defaults 7 | dependencies: 8 | - python=3.9 9 | - pytorch-cuda=12.1 10 | - pytorch 11 | - datasets 12 | - schema 13 | - pip: 14 | - py-cpuinfo 15 | - opencv-python 16 | - scikit-learn 17 | - pycocotools 18 | - intel_extension_for_transformers==1.3 19 | - git+https://github.com/EleutherAI/lm-evaluation-harness.git@cc9778fbe4fa1a709be2abed9deb6180fd40e7e2 20 | -------------------------------------------------------------------------------- /run_quant_slurm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #SBATCH --job-name=job_name 3 | #SBATCH -o /home/user/slurm_logs/%x_%A_%a.out 4 | #SBATCH -e /home/user/slurm_logs/%x_%A_%a.err 5 | #SBATCH --mail-type=ALL 6 | #SBATCH --mail-user=user@email.com 7 | #SBATCH --mem=120000 8 | #SBATCH -n 2 9 | #SBATCH -t 12:00:00 10 | #SBATCH --gres=gpu:1 11 | 12 | MODELS=("facebook/opt-1.3b" "facebook/opt-6.7b" "01-ai/Yi-6B" "mistralai/Mistral-7B-v0.1" "bigscience/bloom-7b1" "meta-llama/Llama-2-7b-hf" "microsoft/phi-2" ) 13 | MODEL_PATH=${MODELS[$SLURM_ARRAY_TASK_ID-1]} 14 | TIMESTAMP=$(date +%Y%m%d-%H%M%S) 15 | VARIABLE="int4" 16 | INTERPRETER="/path/to/interpreter" 17 | 18 | INTERPRETER run_clm_no_trainer.py \ 19 | --model ${MODEL_PATH} --batch_size=4 \ 20 | --output_dir="./results/$(basename ${MODEL_PATH})_${VARIABLE}_${TIMESTAMP}" \ 21 | --tasks lambada_openai hellaswag winogrande piqa boolq arc_challenge \ 22 | --quantize --enable_activation --sq \ 23 | --bits=4 --dtype=${VARIABLE} --group_size=128 --algo=RTN \ 24 | 25 | # Delete the quantized model 26 | rm -rf "./results/$(basename ${MODEL_PATH})_${VARIABLE}_${TIMESTAMP}" --------------------------------------------------------------------------------