├── .gitignore ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── analysis ├── abstract_interpretation.py └── inference.py ├── analysis_main.py ├── appendix.pdf ├── docs ├── analysis.md ├── dynamic_tool.md ├── imgs │ ├── fig0.png │ ├── fig1.png │ └── fig2.png ├── overview.md ├── parse.md └── troubleshoot.md ├── dynamic_tool ├── TFNBDetector.py ├── TFNBUtils.py └── Yuhao Zhang-undergraduate dissertation.pdf ├── main.py ├── parse ├── parse_format_text.py ├── parse_graph.py └── specified_ranges.py ├── requirements.txt ├── solver.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.pdf 3 | *.pbtxt 4 | *.gv 5 | *.xml 6 | *.iml 7 | *.txt -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to DEBAR 2 | 3 | You can contribute to BEBAR by 4 | 5 | 1. Contribute by implementing the abstract interpretations of unhandled TensorFlow APIs. Please see the last section of [Analysis](./docs/analysis.md). 6 | 7 | 2. Contribute by providing more specified weights and inputs ranges. Please see the last section of [Parse](./docs/parse.md). 8 | 9 | 3. Troubleshoot by creating issues. Please see [Troubleshoot](./docs/troubleshoot.md). 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3 2 | FROM tensorflow/tensorflow:1.13.1-py3 3 | 4 | WORKDIR /usr/src/app 5 | 6 | COPY requirements.txt ./ 7 | RUN pip install --no-cache-dir -r requirements.txt 8 | 9 | RUN apt-get update && apt-get install -y \ 10 | curl \ 11 | unzip 12 | 13 | RUN curl -L -o a.zip 'https://drive.google.com/uc?export=download&id=1GBHFd-fPIBWqJOpIC8ZO8g3F1LoIZYNn' 14 | RUN unzip a.zip 15 | 16 | COPY . . 17 | # Reproduce the results of our paper. 18 | # CMD [ "python", "./main.py", "./computation_graphs_and_TP_list/computation_graphs"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2020 The Authors: Yuhao Zhang, Luyao Ren, Liqian Chen, Yingfei Xiong, Shing-Chi Cheung, Tao Xie. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DEBAR: *DE*tecting Numerical *B*ugs in Neural Network *AR*chitectures 2 | 3 | 4 | This repository contains the implementation and the evaluation of our upcoming ESEC/FSE 2020 paper: Detecting Numerical Bugs in Neural Network Architectures. 5 | 6 | DEBAR can detect numerical bugs in neural networks at the architecture level (without concrete weights and inputs, before the training session of the neural network). 7 | 8 | We have created pull requests to fix the numerical bugs that we found in open source repositories. And some of them are accepted and merged: 9 | 10 | * https://github.com/tensorflow/models/pull/8223 11 | 12 | * https://github.com/tensorflow/models/pull/8221 13 | 14 | 15 | ## Collected Datasets 16 | 17 | We share our two collected datasets and evaluation results [online](https://drive.google.com/uc?export=download&id=1GBHFd-fPIBWqJOpIC8ZO8g3F1LoIZYNn). 18 | 19 | The first dataset is a set of 9 buggy architectures collected by existing studies. The buggy architectures come from two studies: eight architectures were collected by a previous [empirical study on TensorFlow bugs](https://github.com/ForeverZyh/TensorFlow-Program-Bugs) (Github/StackOverflow-IPS-id.pbtxt) and one architecture was obtained from the study that proposes and evaluates [TensorFuzz](https://github.com/brain-research/tensorfuzz/blob/master/bugs/collection_bug.py) (TensorFuzz.pbtxt). 20 | 21 | The second dataset contains 48 architectures from a large collection of research projects in TensorFlow Models repository. Overall, our second dataset contains a great diversity of neural architectures like CNN, RNN, GAN, HMM, and so on. Please note that we have no knowledge about whether the architectures in this dataset contain numerical bugs when collecting the dataset. 22 | 23 | For every architecture in two datasets, we extract the computation graph by using a TensorFlow API. Each extracted computation graph is represented by a Protocol Buffer file, which provides the operations (nodes) and the data flow relations (edges). 24 | 25 | ## Setups 26 | 27 | There are two ways you can run DEBAR: 28 | 29 | 1. Run in docker. 30 | 2. Run in virtual environments with virtualenv or conda. 31 | 32 | ### Setups for docker 33 | 34 | Install docker and type the following command to build the image. 35 | 36 | ```bash 37 | docker pull yuhaoz/debar:main 38 | ``` 39 | 40 | Then type the following command to start a bash into the image. 41 | 42 | ```bash 43 | docker run -it yuhaoz/debar:main bash 44 | ``` 45 | 46 | ### Setups for virtual environments with virtualenv or conda 47 | 48 | #### Environment 49 | 50 | DEBAR runs on python3 (>=3.5). 51 | 52 | We encourage users to use virtual environments such as virtualenv or conda. Make sure you are in a virtual environment and then follow the steps: 53 | 54 | ```bash 55 | pip install -r requirements.txt 56 | ``` 57 | 58 | The current implementation of DEBAR only supports detecting numerical bugs in static computation graphs in TensorFlow. If you want a GPU version of TensorFlow, which can accelerate the loading process of protocol buffer files into (GPU) memory. 59 | 60 | ```bash 61 | pip install tensorflow-gpu==1.13.1 62 | ``` 63 | 64 | or a CPU version: 65 | 66 | ```bash 67 | pip install tensorflow==1.13.1 68 | ``` 69 | 70 | DEBAR has a dependency on TensorFlow v1 but is not compatible with TensorFlow v2. You may also notice that DEBAR has a dependency of z3-solver, it is due to some legacy during development which may be removed later. 71 | 72 | #### Dataset 73 | 74 | We share our two collected datasets and evaluation results [online](https://drive.google.com/uc?export=download&id=1GBHFd-fPIBWqJOpIC8ZO8g3F1LoIZYNn). You can manually download from the link, or 75 | 76 | ```bash 77 | curl -L -o dataset.zip 'https://drive.google.com/uc?export=download&id=1GBHFd-fPIBWqJOpIC8ZO8g3F1LoIZYNn' 78 | ``` 79 | 80 | ## Running DEBAR 81 | 82 | ```bash 83 | python analysis_main.py PBTXT_FILE [unbounded_weight/unbounded_input] 84 | ``` 85 | 86 | The above command shows how to run DEBAR. The first argument to `analysis_main.py` is the Protocol Buffer file describing the target computation graph. 87 | 88 | The second argument is a [optional] flag denoting whether to specify the range of the weights and the range of the inputs. 89 | 90 | * The default value (do not pass the second argument) means to specify the range of the weights and the range of the inputs. 91 | * `unbounded_weight` means to specify the range of the inputs, but leave the weights unbounded, which means the ranges of weights will be set to `[-inf,+inf]`. 92 | * `unbounded_input` means to specify the range of the weights, but leave the inputs unbounded, which means the ranges of inputs will be set to `[-inf,+inf]`. 93 | 94 | The specification of ranges of weights/inputs can be given in two ways: 95 | 96 | * Input to the console: During running, DEBAR will prompt the name of the node denoting weights/inputs, if the node name does not exist in `./parse/specified_ranges.py`. Then users can input the specified ranges into the console. 97 | * `./parse/specified_ranges.py`: Manually store the ranges in `./parse/specified_ranges.py` for future reproduction. Please see the documentation [Parse](./docs/parse.md) for more information. 98 | 99 | The recommended way of specifying ranges is first trying to input to the console and then manually store the ranges in `./parse/specified_ranges.py` if future reproduction is needed. 100 | 101 | ### Example 102 | 103 | In the working directory of docker image, you can type the following command to get the result of `TensorFuzz`. 104 | 105 | ```bash 106 | python analysis_main.py ./computation_graphs_and_TP_list/computation_graphs/TensorFuzz.pbtxt 107 | ``` 108 | 109 | Our tool will report the following: 110 | 111 | ``` 112 | (225, 178110) 113 | Exp Exp 114 | warnings 115 | Exp Exp_1 116 | warnings 117 | RealDiv truediv 118 | warnings 119 | Log Log 120 | warnings 121 | TensorFuzz , all: 4 warnings: 4 safe: 0 122 | ``` 123 | 124 | , which means there are 4 unsafe operations in total. DEBAR generates warnings for all of them. DEBAR will output the operation and the name of the node, e.g., `Exp Exp_1` means the operation is `Exp` and the name of the node is `Exp_1`. DEBAR will also output the basic information of the architecture: `(225, 178110)` means that there are 225 operations and 178110 parameters in the architecture. 125 | 126 | ## Reproduce Evaluation in our Paper 127 | 128 | Please type the following command, which is supposed to reproduce the evaluation results in our paper. 129 | 130 | ```bash 131 | python main.py ./computation_graphs_and_TP_list/computation_graphs/ 132 | ``` 133 | 134 | The above command (typically running 30-60mins) will only report one summary line for each architecture. For example, it will report the following summary line for the architecture `TensorFuzz`: 135 | 136 | ``` 137 | TensorFuzz , all: 4 warnings: 4 safe: 0 in time: 2.64 138 | ``` 139 | 140 | And the full output will be stored at `./results.txt`. 141 | 142 | The `safe` number corresponds to the column #6 (DEBAR-TN) in Table 1 in our ESEC/FSE2020 paper and the `warnings` number corresponds to the sum of column #5 (TP) and column #7 (DEBAR-FP) in Table 1. 143 | 144 | Notice that we manually classify the warnings to true positives and false positives. The result and reason for each warning are reported in `./computation_graphs_and_TP_list/true_positives.csv` (inside the collected datasets). 145 | 146 | ### Other Results 147 | 148 | We have reproduced the results of DEBAR in Table 1 in our ESEC/FSE2020 paper. There are other results `Array smashing` (Table 1), `Sole Interval Abstraction` (Table 1), and `Array Expansion` (Table 3). Because they are different settings from DEBAR, we create 3 individual tags for these results. 149 | 150 | * `Array Smashing` has the tag `smashing-affine`. 151 | You can checkout to tag `smashing-affine` and then build the docker image again, or a more convenience way is pulling our docker image using 152 | 153 | ```bash 154 | docker pull yuhaoz/debar:smashing-affine 155 | ``` 156 | 157 | * `Sole Interval Abstraction` has the tag `partition-wo-affine`. 158 | You can checkout to tag `partition-wo-affine` and then build the docker image again, or a more convenience way is pulling our docker image using 159 | 160 | ```bash 161 | docker pull yuhaoz/debar:partition-wo-affine 162 | ``` 163 | 164 | * `Array Expansion` has the tag `expansion-affine`. 165 | You can checkout to tag `expansion-affine` and then build the docker image again, or a more convenience way is pulling our docker image using 166 | 167 | ```bash 168 | docker pull yuhaoz/debar:expansion-affine 169 | ``` 170 | 171 | Notice that `expansion-affine` needs a 30-mins timeout. Instead, we manually comment out the corresponding model names in the `./parse/specified_ranges.py`. 172 | 173 | 174 | ## Published Work 175 | 176 | Yuhao Zhang, Luyao Ren, Liqian Chen, Yingfei Xiong, Shing-Chi Cheung, Tao Xie. Detecting Numerical Bugs in Neural Network Architectures. 177 | 178 | 179 | 180 | For more information, please refer to the documentation under the `docs/` directory. 181 | 182 | * [Overview](./docs/overview.md) 183 | * [Analysis](./docs/analysis.md) 184 | * [Parse](./docs/parse.md) 185 | 186 | * [Troubleshoot](./docs/troubleshoot.md) 187 | * [DynamicTool](./docs/dynamic_tool.md) 188 | 189 | -------------------------------------------------------------------------------- /analysis/abstract_interpretation.py: -------------------------------------------------------------------------------- 1 | class AbstractInterpretation: 2 | def __init__(self, size=None, value=None, dtype=None, constraints=None, array=None): 3 | # the shape of the tensor extracted from the protocal buffer file. 4 | self.size = size 5 | # the interval abstraction stored in a Range object or a numpy concrete value. 6 | self.value = value 7 | self.constraints = constraints 8 | # the data type of the tensor extracted from the protocal buffer file. 9 | self.dtype = dtype 10 | # the tensor partition stored in a Array object. 11 | self.array = array 12 | 13 | # check whether some of the fields are None, which indicates that dataflow analysis cannot infer this abstracted 14 | # value due to unimplemented TensorFlow APIs. 15 | def has_none(self): 16 | return self.size is None or self.value is None or self.dtype is None 17 | 18 | # gets the i-th index of all the fields and returns a new AbstractInterpretation object. 19 | # returns self if i is None. 20 | def index_of(self, i): 21 | if i is None: 22 | return self 23 | if self.has_none(): 24 | return AbstractInterpretation() 25 | 26 | return AbstractInterpretation(size=self.size[i], value=self.value[i], dtype=self.dtype[i], 27 | constraints=self.constraints, array=self.array[i]) 28 | 29 | def __str__(self): 30 | return "size: %s\nvalue: %s\ndtype: %s\nconstraints: %s\n array blocks: %s\n" % ( 31 | str(self.size), str(self.value), str(self.dtype), str(self.constraints), str(self.array)) 32 | -------------------------------------------------------------------------------- /analysis/inference.py: -------------------------------------------------------------------------------- 1 | import math 2 | import copy 3 | import warnings 4 | import numpy as np 5 | from itertools import product 6 | 7 | from analysis.abstract_interpretation import AbstractInterpretation 8 | import parse.parse_format_text as parse_format_text 9 | from solver import Range, Array 10 | from utils import OVERFLOW_LIMIT, UNDERFLOW_LIMIT, resolve_type 11 | 12 | turn_on_bool = False 13 | length_unknown = 1e3 14 | 15 | 16 | # infer size from a and b under assumption that a = b, even though one of them might be unknown, i.e., equals to ? 17 | def real_size(a, b): 18 | if str(a) == "?" and str(b) == "?": 19 | raise AssertionError("cannot infer ? size") 20 | elif str(a) == "?": 21 | return int(b) 22 | else: 23 | return int(a) 24 | 25 | 26 | # the abstract interpretation of identity. 27 | def identity(args, node=None): 28 | try: 29 | return args[0].value if isinstance(args[0].value, Range) else Range(left=resolve_type(np.min(args[0].value)), 30 | right=resolve_type(np.max(args[0].value))) 31 | except: # if it is not able to get the range (e.g., it is a zero-size array) 32 | return None 33 | 34 | 35 | # the abstract interpretation of joining of a list of interval abstractions. 36 | def packtorange(args, node): 37 | maxs = [] 38 | mins = [] 39 | for arg in args: 40 | if isinstance(arg.value, Range): 41 | maxs.append(arg.value.right) 42 | mins.append(arg.value.left) 43 | elif arg.value.size > 0: 44 | maxs.append(resolve_type(np.max(arg.value))) 45 | mins.append(resolve_type(np.min(arg.value))) 46 | 47 | if None in maxs or None in mins: 48 | return None 49 | return Range(left=np.min(mins), right=np.max(maxs)) 50 | 51 | 52 | # returns an unbounded interval abstraction with [-inf, +inf] 53 | def dumy(): 54 | return Range(left=-OVERFLOW_LIMIT, right=OVERFLOW_LIMIT) 55 | 56 | 57 | def safeexp(X): 58 | UPPER_BOUND = 100 59 | try: 60 | ans = [] 61 | for x in X: 62 | ans.append(min(math.exp(min(x, UPPER_BOUND)), OVERFLOW_LIMIT)) 63 | return np.array(ans) 64 | except: 65 | return min(math.exp(min(X, UPPER_BOUND)), OVERFLOW_LIMIT) 66 | 67 | 68 | def safesqrt(X): 69 | try: 70 | ans = [] 71 | for x in X: 72 | if x < 0: 73 | ans.append(0) 74 | else: 75 | ans.append(math.sqrt(x)) 76 | return np.array(ans) 77 | except: 78 | if X < 0: 79 | return 0 80 | else: 81 | return math.sqrt(X) 82 | 83 | 84 | def safepow(X, Y): 85 | UPPER_BOUND = 100 86 | try: 87 | ans = [] 88 | for (x, y) in zip(X, Y): 89 | try: 90 | ans.append(min(math.pow(x, y), OVERFLOW_LIMIT)) 91 | except: 92 | ans.append(OVERFLOW_LIMIT) 93 | return np.array(ans) 94 | except: 95 | try: 96 | return min(math.pow(X, Y), OVERFLOW_LIMIT) 97 | except: 98 | return OVERFLOW_LIMIT 99 | 100 | 101 | def safelgamma(X): 102 | try: 103 | ans = [] 104 | for x in X: 105 | if x <= UNDERFLOW_LIMIT: 106 | ans.append(OVERFLOW_LIMIT) 107 | else: 108 | ans.append(math.lgamma(x)) 109 | return np.array(ans) 110 | except: 111 | if X <= UNDERFLOW_LIMIT: 112 | return OVERFLOW_LIMIT 113 | else: 114 | return math.lgamma(X) 115 | 116 | 117 | def safesoftplus(X): 118 | UPPER_BOUND = 100 119 | try: 120 | ans = [] 121 | for x in X: 122 | if X > UPPER_BOUND: 123 | ans.append(X) 124 | else: 125 | ans.append(np.log1p(np.exp(X))) 126 | return np.array(ans) 127 | except: 128 | if X > UPPER_BOUND: 129 | return X 130 | else: 131 | return np.log1p(np.exp(X)) 132 | 133 | 134 | # contains the abstract interpretations of TensorFlow APIs used in interval abstraction + tensor smashing. 135 | class InferValue: 136 | @staticmethod 137 | def abs(args: list, node): 138 | assert len(args) == 1 139 | if isinstance(args[0].value, Range): 140 | left_sq = np.abs(args[0].value.left) 141 | right_sq = np.abs(args[0].value.right) 142 | min_sq = min(left_sq, right_sq) 143 | max_sq = max(left_sq, right_sq) 144 | cond = args[0].value.left <= 0 and args[0].value.right >= 0 145 | return Range(left=0 if cond else min_sq, right=max_sq) 146 | else: 147 | return np.abs(args[0].value) 148 | 149 | @staticmethod 150 | def add(args: list, node): 151 | assert len(args) == 2 152 | if args[0].value is None or args[1].value is None: 153 | return None 154 | if isinstance(args[0].value, Range) or isinstance(args[1].value, Range): 155 | x = identity([args[0]], node) 156 | y = identity([args[1]], node) 157 | return Range(left=x.left + y.left, right=x.right + y.right) 158 | else: 159 | return args[0].value + args[1].value 160 | 161 | @staticmethod 162 | def addn(args: list, node): 163 | assert len(args) > 0 164 | if len(args) == 1: 165 | return args[0].value 166 | else: 167 | s = InferValue.add([args[0], args[1]], node) 168 | for i in range(2, len(args)): 169 | s = InferValue.add([AbstractInterpretation(value=s), args[i]], node) 170 | return s 171 | 172 | @staticmethod 173 | def all(args: list, node): 174 | if not turn_on_bool: 175 | return Range(left=False, right=True) 176 | raise NotImplementedError 177 | 178 | @staticmethod 179 | def any(args: list, node): 180 | if not turn_on_bool: 181 | return Range(left=False, right=True) 182 | raise NotImplementedError 183 | 184 | @staticmethod 185 | def argmax(args: list, node): 186 | assert len(args) == 2 187 | try: 188 | return Range(left=0, right=int(args[0].size[int(args[1].value)]) - 1) 189 | except: 190 | return Range(left=0, right=length_unknown) 191 | 192 | @staticmethod 193 | def assign(args: list, node): 194 | assert len(args) == 2 195 | if args[0].value is None: 196 | return args[1].value 197 | else: 198 | return args[0].value 199 | 200 | def assignadd(args: list, node): 201 | y = identity([args[1]], node) 202 | tmp = dumy() 203 | if y.left >= 0: 204 | tmp.left = args[0].value.left 205 | if y.right <= 0: 206 | tmp.right = args[0].value.right 207 | return tmp 208 | 209 | def assignsub(args: list, node): 210 | y = identity([args[1]], node) 211 | tmp = dumy() 212 | if y.left <= 0: 213 | tmp.left = args[0].value.left 214 | if y.right >= 0: 215 | tmp.right = args[0].value.right 216 | return tmp 217 | 218 | @staticmethod 219 | def avgpool(args: list, node): 220 | assert len(args) == 1 221 | return identity(args, node) 222 | 223 | @staticmethod 224 | def batchmatmul(args: list, node): 225 | assert len(args) == 2 226 | x = copy.deepcopy(args[0]) 227 | y = copy.deepcopy(args[1]) 228 | x.size = x.size[1:] 229 | y.size = y.size[1:] 230 | return InferValue.matmul([x, y], node) 231 | 232 | @staticmethod 233 | def batchtospacend(args: list, node): 234 | assert len(args) == 3 235 | return args[0].value 236 | 237 | @staticmethod 238 | def spacetobatchnd(args: list, node): 239 | assert len(args) == 3 240 | return args[0].value 241 | 242 | @staticmethod 243 | def biasadd(args: list, node): 244 | assert len(args) == 2 and len(args[1].size) == 1 and ( 245 | str(args[0].size[-1]) == "?" or str(args[1].size[0]) or args[0].size[-1] == args[1].size[0]) 246 | return Range(left=args[0].value.left + args[1].value.left, 247 | right=args[0].value.right + args[1].value.right) 248 | 249 | @staticmethod 250 | def broadcastargs(args: list, node): 251 | assert len(args) == 2 252 | return args[0].value 253 | 254 | @staticmethod 255 | def cast(args: list, node): 256 | # tf.int64: 9; tf.int32: 3; tf.int16: 5; tf.int8: 6; 257 | # tf.uint64: 23; tf.uint32: 22; tf.uint16: 17; tf.uint8: 4; 258 | # tf.float64 2; tf.float32: 1; tf.float16: 19; 259 | # tf.bool: 10; 260 | assert len(args) == 1 261 | bool_proto = [10] 262 | int_proto = [9, 3, 5, 6] + [23, 22, 17, 4] 263 | float_proto = [2, 1, 19] 264 | attrs = node.attr 265 | if int(attrs['SrcT'].type) in bool_proto and int(attrs['DstT'].type) in int_proto + float_proto: 266 | return Range(left=0, right=1) 267 | elif int(attrs['SrcT'].type) in int_proto + float_proto and int(attrs['DstT'].type) in [10]: 268 | return Range(left=False, right=True) 269 | elif int(attrs['SrcT'].type) in int_proto and int(attrs['DstT'].type) in int_proto: 270 | return args[0].value 271 | elif int(attrs['SrcT'].type) in float_proto and int(attrs['DstT'].type) in float_proto: 272 | return args[0].value 273 | elif int(attrs['SrcT'].type) in int_proto and int(attrs['DstT'].type) in float_proto: 274 | return args[0].value 275 | elif int(attrs['SrcT'].type) in float_proto and int(attrs['DstT'].type) in int_proto: 276 | return InferValue.floor(args, node) 277 | else: 278 | raise NotImplementedError("%s -> %s not implemented!" % (attrs['SrcT'].type, attrs['DstT'].type)) 279 | 280 | @staticmethod 281 | def checknumerics(args: list, node): 282 | assert len(args) == 1 283 | return args[0].value 284 | 285 | @staticmethod 286 | def cholesky(args: list, node): 287 | return dumy() 288 | 289 | @staticmethod 290 | def clipbyvalue(args: list, node): 291 | assert len(args) == 3 292 | if isinstance(args[0].value, Range): 293 | return Range(left=max(args[0].value.left, 294 | float(args[1].value) if not isinstance(args[1].value, Range) else args[1].value.left), 295 | right=min(args[0].value.right, 296 | float(args[2].value) if not isinstance(args[2].value, Range) else args[ 297 | 2].value.right)) 298 | else: 299 | return np.minimum(np.maximum(args[0].value, args[1].value), args[2].value) 300 | 301 | @staticmethod 302 | def concatv2(args: list, node): 303 | any_range = False 304 | for x in args: 305 | if isinstance(x.value, Range): 306 | any_range = True 307 | break 308 | 309 | if not any_range: 310 | return np.concatenate([x.value for x in args[:-1]], axis=np.int32(args[-1].value)) 311 | else: 312 | return packtorange(args[:-1], node) 313 | 314 | @staticmethod 315 | def const(args: list, node): 316 | assert len(args) == 0 317 | return getattr(parse_format_text, node.op.lower())(node) 318 | 319 | @staticmethod 320 | def conv2d(args: list, node): 321 | assert len(args) == 2 322 | ind = 1 323 | for x in args[1].size[:-1]: 324 | ind *= int(x) 325 | x = identity([args[0]], node) 326 | y = identity([args[1]], node) 327 | ends = [x.left * y.left * ind, x.left * y.right * ind, 328 | x.right * y.left * ind, x.right * y.right * ind] 329 | return Range(left=min(ends), right=max(ends)) 330 | 331 | @staticmethod 332 | def conv2dbackpropinput(args: list, node): 333 | return Range(left=-1, right=1) 334 | return getattr(parse_format_text, "variablev2")(node) 335 | 336 | @staticmethod 337 | def depthwiseconv2dnative(args: list, node): 338 | assert len(args) == 2 339 | ind = 1 340 | for x in args[1].size[:2]: 341 | ind *= int(x) 342 | ends = [args[0].value.left * args[1].value.left * ind, args[0].value.left * args[1].value.right * ind, 343 | args[0].value.right * args[1].value.left * ind, args[0].value.right * args[1].value.right * ind] 344 | return Range(left=min(ends), right=max(ends)) 345 | 346 | @staticmethod 347 | def diag(args: list, node): 348 | assert len(args) == 1 349 | tmp = packtorange(args, node) 350 | return Range(left=min(0, tmp.left), right=max(0, tmp.right)) 351 | 352 | @staticmethod 353 | def dynamicstitch(args: list, node): 354 | assert len(args) % 2 == 0 355 | datas = args[len(args) // 2:] 356 | return packtorange(datas, node) 357 | 358 | @staticmethod 359 | def enter(args: list, node): 360 | assert len(args) == 1 361 | return args[0].value 362 | 363 | @staticmethod 364 | def equal(args: list, node): 365 | if not turn_on_bool: 366 | return Range(left=False, right=True) 367 | raise NotImplementedError 368 | 369 | @staticmethod 370 | def exit(args: list, node): 371 | return InferValue.identity(args, node) 372 | 373 | @staticmethod 374 | def expanddims(args: list, node): 375 | if not isinstance(args[0].value, Range) and not isinstance(args[1].value, Range): 376 | return np.expand_dims(args[0].value, axis=np.int32(args[1].value)) 377 | else: 378 | return identity(args, node) 379 | 380 | @staticmethod 381 | def fifoqueuev2(args: list, node): 382 | return InferValue.randomshufflequeuev2(args, node) 383 | 384 | @staticmethod 385 | def fill(args: list, node): 386 | assert len(args) == 2 387 | if not isinstance(args[0].value, Range) and not isinstance(args[1].value, Range): 388 | ret = np.empty(args[0].value) 389 | ret.fill(args[1].value) 390 | return ret 391 | else: 392 | return identity([args[1]]) 393 | 394 | @staticmethod 395 | def floor(args: list, node): 396 | assert len(args) == 1 397 | if isinstance(args[0].value, Range): 398 | return Range(left=math.floor(args[0].value.left), right=math.floor(args[0].value.right)) 399 | else: 400 | return np.floor(args[0].value) 401 | 402 | @staticmethod 403 | def fusedbatchnorm(args: list, node): 404 | assert len(args) == 5 405 | # x, scale, offset, mean, variance 406 | epsilon = float(node.attr['epsilon'].f) 407 | is_training = node.attr["is_training"].b 408 | 409 | x = identity([args[0]], node) 410 | mean = identity([args[1]], node) 411 | variance = identity([args[2]], node) + epsilon 412 | 413 | if not is_training: 414 | offset = identity([args[3]], node) 415 | scale = identity([args[4]], node) 416 | ends_scale_variance = [scale.left / variance.left, scale.right / variance.left, 417 | scale.left / variance.right, 418 | scale.right / variance.right] 419 | 420 | ends = [(x.left - mean.right) * end for end in ends_scale_variance] + [ 421 | (x.right - mean.left) * end for end in ends_scale_variance] 422 | 423 | return [Range(left=min(ends) + offset.left, right=max(ends) + offset.right), 424 | dumy(), dumy(), dumy(), dumy()] 425 | else: 426 | ends_scale_variance = [1 / variance.left, 1 / variance.right] 427 | 428 | ends = [(x.left - mean.right) * end for end in ends_scale_variance] + [ 429 | (x.right - mean.left) * end for end in ends_scale_variance] 430 | 431 | return [Range(left=min(ends), right=max(ends)), dumy(), dumy(), dumy(), dumy()] 432 | 433 | @staticmethod 434 | def gathernd(args: list, node): 435 | assert len(args) == 2 436 | return identity(args, node) 437 | 438 | @staticmethod 439 | def gatherv2(args: list, node): 440 | assert len(args) == 3 441 | return identity(args, node) 442 | 443 | @staticmethod 444 | def greater(args: list, node): 445 | if not turn_on_bool: 446 | return Range(left=False, right=True) 447 | raise NotImplementedError 448 | 449 | @staticmethod 450 | def greaterequal(args: list, node): 451 | if not turn_on_bool: 452 | return Range(left=False, right=True) 453 | raise NotImplementedError 454 | 455 | @staticmethod 456 | def identity(args: list, node): 457 | assert len(args) == 1 458 | return args[0].value 459 | 460 | @staticmethod 461 | def isfinite(args: list, node): 462 | assert len(args) == 1 463 | return args[0].value 464 | 465 | @staticmethod 466 | def iteratorgetnext(args: list, node): 467 | assert len(args) == 1 468 | return args[0].value 469 | 470 | @staticmethod 471 | def iteratorv2(args: list, node): 472 | assert len(args) == 0 473 | return getattr(parse_format_text, node.op.lower())(node) 474 | 475 | @staticmethod 476 | def leakyrelu(args: list, node): 477 | assert len(args) == 1 478 | alpha = node.attr["alpha"].f 479 | 480 | def leaky_relu(x): 481 | if x >= 0: 482 | return x 483 | else: 484 | return alpha * x 485 | 486 | if isinstance(args[0].value, Range): 487 | return Range(left=leaky_relu(args[0].value.left), right=leaky_relu(args[0].value.right)) 488 | else: 489 | return leaky_relu(args[0].value) 490 | 491 | @staticmethod 492 | def l2loss(args: list, node): 493 | assert len(args) == 1 494 | return InferValue.square(args, node) * 0.5 495 | 496 | @staticmethod 497 | def less(args: list, node): 498 | if not turn_on_bool: 499 | return Range(left=False, right=True) 500 | raise NotImplementedError 501 | 502 | @staticmethod 503 | def lessequal(args: list, node): 504 | if not turn_on_bool: 505 | return Range(left=False, right=True) 506 | raise NotImplementedError 507 | 508 | @staticmethod 509 | def lgamma(args: list, node): 510 | assert len(args) == 1 511 | if isinstance(args[0].value, Range): 512 | ends = [safelgamma(args[0].value.left), safelgamma(args[0].value.right)] 513 | return Range(left=min(ends), right=max(ends)) 514 | else: 515 | return safelgamma(args[0].value) 516 | 517 | @staticmethod 518 | def linspace(args: list, node): 519 | assert len(args) == 3 520 | if isinstance(args[0].value, Range) or isinstance(args[1].value, Range) or isinstance(args[2].value, Range): 521 | return packtorange(args[:-1], node) 522 | else: 523 | return np.linspace(args[0].value, args[1].value, args[2].value) 524 | 525 | @staticmethod 526 | def logicaland(args: list, node): 527 | if not turn_on_bool: 528 | return Range(left=False, right=True) 529 | raise NotImplementedError 530 | 531 | @staticmethod 532 | def logicalnot(args: list, node): 533 | if not turn_on_bool: 534 | return Range(left=False, right=True) 535 | raise NotImplementedError 536 | 537 | @staticmethod 538 | def logicalor(args: list, node): 539 | if not turn_on_bool: 540 | return Range(left=False, right=True) 541 | raise NotImplementedError 542 | 543 | @staticmethod 544 | def loguniformcandidatesampler(args: list, node): 545 | assert len(args) == 1 546 | ind = int(node.attr["range_max"].i) 547 | num = int(node.attr["num_sampled"].i) 548 | return [Range(left=0, right=ind - 1), Range(left=UNDERFLOW_LIMIT * 10, right=num), 549 | Range(left=UNDERFLOW_LIMIT * 10, right=num)] 550 | 551 | @staticmethod 552 | def loopcond(args: list, node): 553 | return InferValue.identity(args, node) 554 | 555 | @staticmethod 556 | def matmul(args: list, node): 557 | assert len(args) == 2 558 | try: 559 | len(args[0].size) == len(args[1].size) 560 | except: 561 | return dumy() 562 | assert len(args[0].size) == len(args[1].size) 563 | for i in range(len(args[0].size) - 2): 564 | assert str(args[0].size[i]) == "?" or str(args[1].size[i]) == "?" or args[0].size[i] == args[1].size[i] 565 | ind = real_size(args[0].size[-1], args[1].size[-2]) 566 | if not isinstance(args[0].value, Range) and not isinstance(args[1].value, Range): 567 | return np.matmul(args[0].value, args[1].value) 568 | else: 569 | x = identity([args[0]], node) 570 | y = identity([args[1]], node) 571 | ends = [x.left * y.left * ind, x.left * y.right * ind, x.right * y.left * ind, x.right * y.right * ind] 572 | return Range(left=min(ends), right=max(ends)) 573 | 574 | @staticmethod 575 | def matrixdiag(args: list, node): 576 | assert len(args) == 1 577 | tmp = packtorange(args, node) 578 | return Range(left=min(0, tmp.left), right=max(0, tmp.right)) 579 | 580 | @staticmethod 581 | def matrixbandpart(args: list, node): 582 | assert len(args) == 3 583 | tmp = packtorange(args[:1], node) 584 | return Range(left=min(tmp.left, 0), right=max(tmp.right, 0)) 585 | 586 | @staticmethod 587 | def matrixdiagpart(args: list, node): 588 | assert len(args) == 1 589 | return args[0].value 590 | 591 | @staticmethod 592 | def max(args: list, node): 593 | assert len(args) == 2 594 | return args[0].value 595 | 596 | @staticmethod 597 | def maxpool(args: list, node): 598 | assert len(args) == 1 599 | return args[0].value 600 | 601 | @staticmethod 602 | def maximum(args: list, node): 603 | assert len(args) == 2 604 | x = args[0].value 605 | y = args[1].value 606 | if isinstance(x, Range) and isinstance(y, Range): 607 | return Range(left=max(x.left, y.left), right=max(x.right, y.right)) 608 | elif not isinstance(x, Range) and not isinstance(y, Range): 609 | return np.maximum(x, y) 610 | else: 611 | if isinstance(y, Range): 612 | x, y = y, x 613 | y = resolve_type(np.max(y)) 614 | return Range(left=max(x.left, y), right=max(x.right, y)) 615 | 616 | @staticmethod 617 | def mean(args: list, node): 618 | assert len(args) == 2 619 | return identity(args, node) 620 | 621 | @staticmethod 622 | def merge(args: list, node): 623 | tmp = packtorange(args, node) 624 | max_index = int(node.attr["N"].i) 625 | return_index = Range(left=0, right=max_index - 1) 626 | if isinstance(tmp, tuple): 627 | raise AssertionError 628 | else: 629 | return [tmp, return_index] 630 | 631 | @staticmethod 632 | def min(args: list, node): 633 | assert len(args) == 2 634 | return args[0].value 635 | 636 | @staticmethod 637 | def minimum(args: list, node): 638 | assert len(args) == 2 639 | x = args[0].value 640 | y = args[1].value 641 | if isinstance(x, Range) and isinstance(y, Range): 642 | return Range(left=min(x.left, y.left), right=min(x.right, y.right)) 643 | elif not isinstance(x, Range) and not isinstance(y, Range): 644 | return np.minimum(x, y) 645 | else: 646 | if isinstance(y, Range): 647 | x, y = y, x 648 | y = resolve_type(np.min(y)) 649 | return Range(left=min(x.left, y), right=min(x.right, y)) 650 | 651 | @staticmethod 652 | def mul(args: list, node): 653 | assert len(args) == 2 654 | if args[0].value is None or args[1].value is None: 655 | return None 656 | if isinstance(args[1].value, Range) or isinstance(args[0].value, Range): 657 | x = identity([args[0]], node) 658 | y = identity([args[1]], node) 659 | ends = [x.left * y.left, x.left * y.right, x.right * y.left, x.right * y.right] 660 | return Range(left=min(ends), right=max(ends)) 661 | else: 662 | return args[0].value * args[1].value 663 | 664 | def multinomial(args: list, node): 665 | assert len(args) == 2 666 | return Range(left=0, right=1) 667 | 668 | @staticmethod 669 | def neg(args: list, node): 670 | assert len(args) == 1 671 | if isinstance(args[0].value, Range): 672 | return Range(left=-args[0].value.right, right=-args[0].value.left) 673 | else: 674 | return -args[0].value 675 | 676 | @staticmethod 677 | def nonmaxsuppressionv3(args: list, node): 678 | assert len(args) == 5 679 | try: 680 | ind = int(args[1].size[0]) 681 | return Range(left=0, right=ind - 1) 682 | except: 683 | return Range(left=0, right=length_unknown) 684 | 685 | @staticmethod 686 | def notequal(args: list, node): 687 | if not turn_on_bool: 688 | return Range(left=False, right=True) 689 | raise NotImplementedError 690 | 691 | @staticmethod 692 | def onehot(args: list, node): 693 | assert len(args) == 4 694 | return Range(left=min([args[2].value, args[3].value]), 695 | right=max([args[2].value, args[3].value])) 696 | 697 | @staticmethod 698 | def oneshotiterator(args: list, node): 699 | assert len(args) == 0 700 | return getattr(parse_format_text, node.op.lower())(node) 701 | 702 | @staticmethod 703 | def pack(args: list, node): 704 | any_range = False 705 | for x in args: 706 | if isinstance(x.value, Range): 707 | any_range = True 708 | break 709 | 710 | if not any_range: 711 | return np.stack([x.value for x in args], axis=int(node.attr["axis"].i)) 712 | else: 713 | return packtorange(args, node) 714 | 715 | @staticmethod 716 | def pad(args: list, node): 717 | return identity(args, node) 718 | 719 | @staticmethod 720 | def paddingfifoqueuev2(args: list, node): 721 | return InferValue.randomshufflequeuev2(args, node) 722 | 723 | @staticmethod 724 | def parsesingleexample(args: list, node): 725 | assert len(args) == 3 726 | return [Range(left=0, right=length_unknown) for _ in range(20)] 727 | 728 | @staticmethod 729 | def placeholder(args: list, node): 730 | assert len(args) == 0 731 | return getattr(parse_format_text, node.op.lower())(node) 732 | 733 | @staticmethod 734 | def placeholderwithdefault(args: list, node): 735 | assert len(args) == 1 736 | tmp = getattr(parse_format_text, 'placeholder')(node) 737 | if isinstance(args[0].value, Range): 738 | return Range(left=min(args[0].value.left, tmp.left), right=max(args[0].value.right, tmp.right)) 739 | else: 740 | return Range(left=min(args[0].value, tmp.left), right=max(args[0].value, tmp.right)) 741 | 742 | @staticmethod 743 | def pow(args: list, node): 744 | assert len(args) == 2 745 | if isinstance(args[0].value, Range) and isinstance(args[1].value, Range): 746 | return Range(left=safepow(args[0].value.left, args[1].value.left), 747 | right=safepow(args[0].value.right, args[1].value.right)) 748 | elif isinstance(args[0].value, Range): 749 | return Range(left=safepow(args[0].value.left, args[1].value), 750 | right=safepow(args[0].value.right, args[1].value)) 751 | elif isinstance(args[1].value, Range): 752 | return Range(left=safepow(args[0].value, args[1].value.left), 753 | right=safepow(args[0].value, args[1].value.right)) 754 | else: 755 | return safepow(args[0].value, args[1].value) 756 | 757 | @staticmethod 758 | def prod(args: list, node): 759 | assert len(args) == 2 760 | if args[0].value is None: 761 | return None 762 | if isinstance(args[0].value, Range) or isinstance(args[1].value, Range): 763 | try: 764 | ind = int(args[0].size[int(args[1].value)]) 765 | return Range(left=safepow(args[0].value.left, ind), right=safepow(args[0].value.right, ind)) 766 | except: 767 | ind = Range(left=0, right=length_unknown) 768 | t = InferValue.pow([args[0], AbstractInterpretation(value=ind, dtype=3, size=[])], node) 769 | if isinstance(t, tuple): 770 | raise AssertionError 771 | else: 772 | return t 773 | else: 774 | axises = np.int32(args[1].value) 775 | return np.prod(args[0].value, axis=tuple(axises) if len(axises.shape) > 0 else axises) 776 | 777 | @staticmethod 778 | def queuedequeuemanyv2(args: list, node): 779 | assert len(args) == 2 780 | return args[0].value 781 | 782 | @staticmethod 783 | def randomshuffle(args: list, node): 784 | assert len(args) == 1 785 | return identity(args, node) 786 | 787 | @staticmethod 788 | def randomshufflequeuev2(args: list, node): 789 | assert len(args) == 0 790 | return getattr(parse_format_text, "oneshotiterator")(node) 791 | 792 | @staticmethod 793 | def randomstandardnormal(args: list, node): 794 | assert len(args) == 1 795 | return Range(left=UNDERFLOW_LIMIT * 10, right=1) 796 | 797 | @staticmethod 798 | def randomuniform(args: list, node): 799 | assert len(args) == 1 800 | return Range(left=UNDERFLOW_LIMIT * 10, right=1) 801 | 802 | @staticmethod 803 | def range(args: list, node): 804 | assert len(args) == 3 805 | all_single_np = True 806 | for arg in args: 807 | if isinstance(arg.value, Range) or len(np.array(arg.value).shape) > 0: 808 | all_single_np = False 809 | break 810 | 811 | if not all_single_np: 812 | left = args[0].value.left if isinstance(args[0].value, Range) else np.min(args[0].value) 813 | right = args[1].value.right if isinstance(args[1].value, Range) else np.max(args[1].value) 814 | return Range(left=left, right=right) 815 | else: 816 | return np.arange(args[0].value, args[1].value, args[2].value) 817 | 818 | @staticmethod 819 | def rank(args: list, node): 820 | assert len(args) == 1 821 | try: 822 | return int(args[0].size) 823 | except: 824 | return Range(left=1, right=length_unknown) 825 | 826 | @staticmethod 827 | def readvariableop(args: list, node): 828 | assert len(args) == 1 829 | return args[0].value 830 | 831 | @staticmethod 832 | def realdiv(args: list, node): 833 | assert len(args) == 2 834 | x = args[0].value 835 | y = args[1].value 836 | if not isinstance(x, Range): 837 | x = np.reshape(x, -1) 838 | if not isinstance(y, Range): 839 | y = np.reshape(y, -1) 840 | if isinstance(x, Range) and isinstance(y, Range): 841 | if y.left > 0 or y.right < 0: 842 | ends = [x.left / y.left, x.left / y.right, x.right / y.left, x.right / y.right] 843 | return Range(left=np.min(ends), right=np.max(ends)) 844 | else: 845 | return Range(left=-OVERFLOW_LIMIT, right=OVERFLOW_LIMIT) 846 | elif not isinstance(y, Range): # x can be a Range or a np.array 847 | if isinstance(x, Range): 848 | ends = [x.left / yy for yy in y] + [x.right / yy for yy in y] 849 | return Range(left=np.min(ends), right=np.max(ends)) 850 | else: 851 | return x * (1 / y) 852 | else: # if y is a Range, whatever x is, we have to end up with a Range, but we can do it precisely when x is a float 853 | if y.left > 0 or y.right < 0: 854 | ends = [xx / y.left for xx in x] + [xx / y.right for xx in x] 855 | return Range(left=np.min(ends), right=np.max(ends)) 856 | else: 857 | return Range(left=-OVERFLOW_LIMIT, right=OVERFLOW_LIMIT) 858 | 859 | @staticmethod 860 | def relu(args: list, node): 861 | assert len(args) == 1 862 | return Range(left=max([args[0].value.left, 0]), 863 | right=max([args[0].value.right, 0])) 864 | 865 | @staticmethod 866 | def relu6(args: list, node): 867 | assert len(args) == 1 868 | return Range(left=min(max(args[0].value.left, 0), 6), 869 | right=min(max(args[0].value.right, 0), 6)) 870 | 871 | @staticmethod 872 | def reshape(args: list, node): 873 | assert len(args) == 2 874 | if not isinstance(args[0].value, Range) and not isinstance(args[1].value, Range): 875 | return np.reshape(args[0].value, np.int32(args[1].value)) 876 | else: 877 | return identity(args, node) 878 | 879 | @staticmethod 880 | def resizearea(args: list, node): 881 | assert len(args) == 2 882 | return args[0].value 883 | 884 | @staticmethod 885 | def resizebilinear(args: list, node): 886 | assert len(args) == 2 887 | return args[0].value 888 | 889 | @staticmethod 890 | def resizenearestneighbor(args: list, node): 891 | assert len(args) == 2 892 | return args[0].value 893 | 894 | @staticmethod 895 | def resourcegather(args: list, node): 896 | assert len(args) == 2 897 | return identity(args, node) 898 | 899 | @staticmethod 900 | def reversev2(args: list, node): 901 | assert len(args) == 2 902 | return identity(args, node) 903 | 904 | @staticmethod 905 | def round(args: list, node): 906 | assert len(args) == 1 907 | if isinstance(args[0].value, Range): 908 | return Range(left=np.round(args[0].value.left), right=np.round(args[0].value.right)) 909 | return np.round(args[0].value) 910 | 911 | @staticmethod 912 | def rsqrt(args: list, node): 913 | assert len(args) == 1 914 | if isinstance(args[0].value, Range): 915 | left = safesqrt(args[0].value.left) 916 | right = safesqrt(args[0].value.right) 917 | if left == 0 or right == 0: 918 | return dumy() 919 | else: 920 | return Range(left=1 / right, right=1 / left) 921 | else: 922 | return 1 / safesqrt(args[0].value) 923 | 924 | @staticmethod 925 | def select(args: list, node): 926 | assert len(args) == 3 927 | if not isinstance(args[0].value, Range): 928 | raise NotImplementedError("not implemented when the condition is known") 929 | 930 | x = identity([args[1]], node) 931 | y = identity([args[2]], node) 932 | if not turn_on_bool: 933 | return Range(left=min(x.left, y.left), right=max(x.right, y.right)) 934 | raise NotImplementedError 935 | 936 | @staticmethod 937 | def shape(args: list, node): 938 | assert len(args) == 1 939 | try: 940 | return [int(x) for x in args[0].size] 941 | except: 942 | return Range(left=1, right=length_unknown) 943 | 944 | @staticmethod 945 | def sign(args: list, node): 946 | assert len(args) == 1 947 | if isinstance(args[0].value, Range): 948 | return Range(left=np.sign(args[0].value.left), right=np.sign(args[0].value.right)) 949 | else: 950 | return np.sign(args[0].value) 951 | 952 | @staticmethod 953 | def size(args: list, node): 954 | assert len(args) == 1 955 | try: 956 | ele = 1 957 | for x in args[0].size: 958 | ele *= int(x) 959 | if ele < 0: 960 | return Range(left=0, right=length_unknown) 961 | else: 962 | return ele 963 | except: 964 | return Range(left=0, right=length_unknown) 965 | 966 | @staticmethod 967 | def slice(args: list, node): 968 | assert len(args) == 3 969 | try: 970 | return args[0].value[ 971 | tuple(slice(a, a + b) if b >= 0 else slice(a, None) for a, b in zip(args[1].value, args[2].value))] 972 | except: 973 | return identity(args, node) 974 | 975 | @staticmethod 976 | def sparsetodense(args: list, node): 977 | assert len(args) == 4 978 | return Range(left=0, right=1) 979 | 980 | @staticmethod 981 | def split(args: list, node): 982 | assert len(args) == 2 983 | nums = int(node.attr["num_split"].i) 984 | if nums == 1: 985 | return identity(args[1:], node) 986 | else: 987 | return [identity(args[1:], node) for _ in range(nums)] 988 | 989 | @staticmethod 990 | def sqrt(args: list, node): 991 | assert len(args) == 1 992 | if isinstance(args[0].value, Range): 993 | left = safesqrt(args[0].value.left) 994 | right = safesqrt(args[0].value.right) 995 | 996 | return Range(left=left, right=right) 997 | else: 998 | return safesqrt(args[0].value) 999 | 1000 | @staticmethod 1001 | def square(args: list, node): 1002 | assert len(args) == 1 1003 | if isinstance(args[0].value, Range): 1004 | abs_value = InferValue.abs(args, node) 1005 | return Range(left=abs_value.left * abs_value.left, right=abs_value.right * abs_value.right) 1006 | else: 1007 | return args[0].value * args[0].value 1008 | 1009 | @staticmethod 1010 | def squareddifference(args: list, node): 1011 | assert len(args) == 2 1012 | value1 = (args[0].value.left - args[1].value.right) * (args[0].value.left - args[1].value.right) 1013 | value2 = (args[0].value.right - args[1].value.left) * (args[0].value.right - args[1].value.left) 1014 | return InferValue.square([AbstractInterpretation(value=Range(left=value1, right=value2))], node) 1015 | 1016 | @staticmethod 1017 | def squeeze(args: list, node): 1018 | assert len(args) == 1 1019 | return identity(args, node) 1020 | 1021 | @staticmethod 1022 | def stopgradient(args: list, node): 1023 | return InferValue.identity(args, node) 1024 | 1025 | @staticmethod 1026 | def stridedslice(args: list, node): 1027 | return identity(args, node) 1028 | 1029 | @staticmethod 1030 | def sub(args: list, node): 1031 | assert len(args) == 2 1032 | if isinstance(args[0].value, Range) or isinstance(args[1].value, Range): 1033 | x = identity([args[0]], node) 1034 | y = identity([args[1]], node) 1035 | return Range(left=x.left - y.right, right=x.right - y.left) 1036 | else: 1037 | return args[0].value - args[1].value 1038 | 1039 | @staticmethod 1040 | def sum(args: list, node): 1041 | assert len(args) == 2 1042 | if args[0].value is None: 1043 | return None 1044 | if isinstance(args[0].value, Range) or isinstance(args[1].value, Range): 1045 | try: 1046 | ind = int(args[0].size[int(args[1].value)]) 1047 | return Range(left=args[0].value.left * ind, right=args[0].value.right * ind) 1048 | except: 1049 | ind = Range(left=1, right=1e6) 1050 | t = InferValue.mul([args[0], AbstractInterpretation(value=ind, dtype=3, size=[])], node) 1051 | if isinstance(t, tuple): 1052 | raise AssertionError 1053 | else: 1054 | return t 1055 | else: 1056 | axises = np.int32(args[1].value) 1057 | return np.sum(args[0].value, axis=tuple(axises) if len(axises.shape) > 0 else axises) 1058 | 1059 | @staticmethod 1060 | def switch(args: list, node): 1061 | assert len(args) == 2 1062 | return [args[0].value, args[0].value] 1063 | 1064 | @staticmethod 1065 | def tensorarraygatherv3(args: list, node): 1066 | assert len(args) == 3 1067 | return args[0].value 1068 | 1069 | @staticmethod 1070 | def tensorarrayv3(args: list, node): 1071 | assert len(args) == 1 1072 | return [dumy(), dumy()] 1073 | 1074 | @staticmethod 1075 | def tensorarrayreadv3(args: list, node): 1076 | assert len(args) == 3 1077 | return args[0].value 1078 | 1079 | @staticmethod 1080 | def tensorarrayscatterv3(args: list, node): 1081 | assert len(args) == 4 1082 | if isinstance(args[2].value, Range): 1083 | return args[0].value 1084 | else: 1085 | return args[0].value 1086 | 1087 | @staticmethod 1088 | def tensorarraysizev3(args: list, node): 1089 | assert len(args) == 2 1090 | return int(args[0].size[0]) 1091 | 1092 | @staticmethod 1093 | def tensorarraywritev3(args: list, node): 1094 | assert len(args) == 4 1095 | return InferValue.tensorarrayscatterv3(args, node) 1096 | 1097 | @staticmethod 1098 | def tile(args: list, node): 1099 | assert len(args) == 2 1100 | if not isinstance(args[0].value, Range) and not isinstance(args[1].value, Range): 1101 | return np.tile(args[0].value, np.int32(args[1].value)) 1102 | else: 1103 | return identity(args, node) 1104 | 1105 | @staticmethod 1106 | def topkv2(args: list, node): 1107 | assert len(args) == 2 1108 | try: 1109 | ind = int(args[0].size[-1]) 1110 | value = Range(left=0, right=ind - 1) 1111 | except: 1112 | value = Range(left=0, right=length_unknown) 1113 | return [identity(args, node), value] 1114 | 1115 | @staticmethod 1116 | def transpose(args: list, node): 1117 | assert len(args) == 2 1118 | try: 1119 | return np.transpose(args[0].value, np.int32(args[1].value)) 1120 | except: 1121 | return identity(args, node) 1122 | 1123 | @staticmethod 1124 | def unpack(args: list, node): 1125 | assert len(args) == 1 1126 | nums = int(node.attr["num"].i) 1127 | axis = int(node.attr["axis"].i) 1128 | if not isinstance(args[0].value, Range): 1129 | assert args[0].value.shape[axis] == nums 1130 | if nums == 1: 1131 | index = [slice(None) for _ in range(len(args[0].value.shape))] 1132 | index[axis] = 0 1133 | return args[0].value[index] 1134 | else: 1135 | ret = [] 1136 | for i in range(nums): 1137 | index = [slice(None) for _ in range(len(args[0].value.shape))] 1138 | index[axis] = i 1139 | ret.append(args[0].value[index]) 1140 | 1141 | return ret 1142 | else: 1143 | if nums == 1: 1144 | return identity(args, node) 1145 | else: 1146 | return [identity(args, node) for _ in range(nums)] 1147 | 1148 | @staticmethod 1149 | def varhandleop(args: list, node): 1150 | assert len(args) == 0 1151 | return getattr(parse_format_text, "variablev2")(node) 1152 | 1153 | @staticmethod 1154 | def variable(args: list, node): 1155 | assert len(args) == 0 1156 | return getattr(parse_format_text, "variablev2")(node) 1157 | 1158 | @staticmethod 1159 | def variablev2(args: list, node): 1160 | assert len(args) == 0 1161 | return getattr(parse_format_text, node.op.lower())(node) 1162 | 1163 | @staticmethod 1164 | def where(args: list, node): 1165 | assert len(args) == 1 1166 | try: 1167 | x = np.max(args[0].size) 1168 | return Range(left=0, right=x - 1) 1169 | except: 1170 | return Range(left=0, right=length_unknown - 1) 1171 | 1172 | @staticmethod 1173 | def zeroslike(args: list, node): 1174 | assert len(args) == 1 1175 | try: 1176 | if len(args[0].size) == 0: 1177 | return 0 1178 | except: 1179 | pass 1180 | 1181 | return Range(left=0, right=0) 1182 | 1183 | @staticmethod 1184 | def floormod(args: list, node): 1185 | def mod(x, y): 1186 | return x - math.floor(x / y) * y 1187 | 1188 | assert len(args) == 2 1189 | try: 1190 | x = float(args[0].value) 1191 | except: 1192 | x = identity([args[0]], node) 1193 | try: 1194 | y = float(args[1].value) 1195 | except: 1196 | y = identity([args[1]], node) 1197 | 1198 | if isinstance(x, Range) and isinstance(y, Range): 1199 | if y.left > 0 or y.right < 0: 1200 | ends = [mod(x.left, y.left), mod(x.left, y.right), mod(x.right, y.left), mod(x.right, y.right)] 1201 | return Range(left=min(ends), right=max(ends)) 1202 | else: 1203 | return Range(left=-OVERFLOW_LIMIT, right=OVERFLOW_LIMIT) 1204 | elif not isinstance(y, Range): 1205 | return x * (1 / y) 1206 | else: 1207 | if y.left > 0 or y.right < 0: 1208 | ends = [mod(x, y.left), mod(x, y.right)] 1209 | return Range(left=min(ends), right=max(ends)) 1210 | else: 1211 | return Range(left=-OVERFLOW_LIMIT, right=OVERFLOW_LIMIT) 1212 | 1213 | @staticmethod 1214 | def iteratortostringhandle(args: list, node): 1215 | warnings.warn("iteratortostringhandle not implemented", RuntimeWarning) 1216 | 1217 | @staticmethod 1218 | def noop(args: list, node): 1219 | warnings.warn("noop not implemented", RuntimeWarning) 1220 | 1221 | @staticmethod 1222 | def restorev2(args: list, node): 1223 | warnings.warn("restorev2 not implemented", RuntimeWarning) 1224 | 1225 | @staticmethod 1226 | def savev2(args: list, node): 1227 | warnings.warn("savev2 not implemented", RuntimeWarning) 1228 | 1229 | # non linear operations: 1230 | @staticmethod 1231 | def sin(args: list, node): 1232 | assert len(args) == 1 1233 | return Range(left=-1, right=1) 1234 | 1235 | def cos(args: list, node): 1236 | assert len(args) == 1 1237 | return Range(left=-1, right=1) 1238 | 1239 | @staticmethod 1240 | def log(args: list, node): 1241 | assert len(args) == 1 1242 | if isinstance(args[0].value, Range): 1243 | if args[0].value.left <= 0: 1244 | return Range(left=-OVERFLOW_LIMIT, right=math.log(args[0].value.right)) 1245 | else: 1246 | return Range(left=math.log(args[0].value.left), right=math.log(args[0].value.right)) 1247 | else: 1248 | return np.log(args[0].value) 1249 | 1250 | @staticmethod 1251 | def log1p(args: list, node): 1252 | assert len(args) == 1 1253 | if isinstance(args[0].value, Range): 1254 | if args[0].value.left <= -1: 1255 | return Range(left=-OVERFLOW_LIMIT, right=np.log1p(args[0].value.right)) 1256 | else: 1257 | return Range(left=np.log1p(args[0].value.left), right=np.log1p(args[0].value.right)) 1258 | else: 1259 | return np.log1p(args[0].value) 1260 | 1261 | @staticmethod 1262 | def softplus(args: list, node): 1263 | assert len(args) == 1 1264 | if isinstance(args[0].value, Range): 1265 | return Range(left=safesoftplus(args[0].value.left), right=safesoftplus(args[0].value.right)) 1266 | else: 1267 | return safesoftplus(args[0].value) 1268 | 1269 | @staticmethod 1270 | def exp(args: list, node): 1271 | assert len(args) == 1 1272 | if isinstance(args[0].value, Range): 1273 | return Range(left=safeexp(args[0].value.left), right=safeexp(args[0].value.right)) 1274 | else: 1275 | return safeexp(args[0].value) 1276 | 1277 | @staticmethod 1278 | def softmax(args: list, node): 1279 | assert len(args) == 1 1280 | try: 1281 | ind = int(args[0].size[-1]) 1282 | except: 1283 | ind = None 1284 | 1285 | if isinstance(args[0].value, Range): 1286 | min_ele = safeexp(args[0].value.left) 1287 | max_ele = safeexp(args[0].value.right) 1288 | if max_ele >= OVERFLOW_LIMIT or min_ele == 0: 1289 | left = 0 1290 | elif ind is not None: 1291 | left = min_ele / ((ind - 1) * max_ele + min_ele) 1292 | else: 1293 | left = min_ele / ((length_unknown - 1) * max_ele + min_ele) 1294 | if max_ele >= OVERFLOW_LIMIT or min_ele == 0: 1295 | right = 1 1296 | elif ind is not None: 1297 | right = max_ele / ((ind - 1) * min_ele + max_ele) 1298 | else: 1299 | right = max_ele / (min_ele + max_ele) 1300 | return Range(left=left, right=right) 1301 | else: 1302 | tmp_exp = np.exp(args[0].value) 1303 | return tmp_exp / np.sum(tmp_exp) 1304 | 1305 | @staticmethod 1306 | def sigmoid(args: list, node): 1307 | assert len(args) == 1 1308 | if isinstance(args[0].value, Range): 1309 | return Range(left=1 / (1 + safeexp(-args[0].value.left)), right=1 / (1 + safeexp(-args[0].value.right))) 1310 | else: 1311 | return 1 / (1 + safeexp(-args[0].value)) 1312 | 1313 | @staticmethod 1314 | def tanh(args: list, node): 1315 | assert len(args) == 1 1316 | if isinstance(args[0].value, Range): 1317 | return Range(left=np.tanh(args[0].value.left), right=np.tanh(args[0].value.right)) 1318 | else: 1319 | return np.tanh(args[0].value) 1320 | 1321 | 1322 | # contains the abstract interpretations of TensorFlow APIs used in the tensor partition and the linear affine relation. 1323 | class InferArray: 1324 | @staticmethod 1325 | def add(args: list, node): 1326 | try: 1327 | len(args[0].size) == len(args[1].size) 1328 | except: 1329 | return None 1330 | assert len(args) == 2 and len(args[0].size) == len(args[1].size) 1331 | ind = len(args[0].size) 1332 | for i in range(ind): 1333 | try: 1334 | l1 = int(args[0].size[i]) 1335 | except: 1336 | l1 = -1 1337 | try: 1338 | l2 = int(args[1].size[i]) 1339 | except: 1340 | l2 = -1 1341 | assert l1 == l2 1342 | 1343 | ret = Array("tmp", args[0].size) 1344 | ret.block_to_symbol = dict() 1345 | ret.index_slices = Array.join_index_slices(args[0].array.index_slices, args[1].array.index_slices) 1346 | keys0 = args[0].array.get_corresponding_keys(ret.index_slices) 1347 | keys1 = args[1].array.get_corresponding_keys(ret.index_slices) 1348 | i = 0 1349 | for indexes in product(*ret.index_slices): 1350 | ret.block_to_symbol[tuple(indexes)] = keys0[i] + keys1[i] 1351 | i += 1 1352 | 1353 | return ret 1354 | 1355 | def sub(args: list, node): 1356 | try: 1357 | len(args[0].size) == len(args[1].size) 1358 | except: 1359 | return None 1360 | assert len(args) == 2 and len(args[0].size) == len(args[1].size) 1361 | ind = len(args[0].size) 1362 | for i in range(ind): 1363 | try: 1364 | l1 = int(args[0].size[i]) 1365 | except: 1366 | l1 = -1 1367 | try: 1368 | l2 = int(args[1].size[i]) 1369 | except: 1370 | l2 = -1 1371 | assert l1 == l2 1372 | 1373 | ret = Array("tmp", args[0].size) 1374 | ret.block_to_symbol = dict() 1375 | ret.index_slices = Array.join_index_slices(args[0].array.index_slices, args[1].array.index_slices) 1376 | keys0 = args[0].array.get_corresponding_keys(ret.index_slices) 1377 | keys1 = args[1].array.get_corresponding_keys(ret.index_slices) 1378 | i = 0 1379 | for indexes in product(*ret.index_slices): 1380 | ret.block_to_symbol[tuple(indexes)] = keys0[i] - keys1[i] 1381 | i += 1 1382 | 1383 | return ret 1384 | 1385 | @staticmethod 1386 | def concatv2(args: list, node): 1387 | assert len(args) > 1 1388 | if len(args) - 1 > 10: 1389 | return None 1390 | try: 1391 | len(args[0].size) == len(args[1].size) 1392 | except: 1393 | return None 1394 | concat_ind = int(args[-1].value) 1395 | for i in range(1, len(args) - 1): 1396 | assert len(args[0].size) == len(args[i].size) 1397 | for j in range(len(args[i].size)): 1398 | try: 1399 | int(args[0].size[j]) 1400 | int(args[i].size[j]) 1401 | except: 1402 | return None 1403 | if j != concat_ind: 1404 | assert int(args[0].size[j]) == int(args[i].size[j]) 1405 | 1406 | ret = Array("tmp", args[0].size) 1407 | ret.block_to_symbol = dict() 1408 | index_slices = [] 1409 | for arg in args[:-1]: 1410 | index_slices.append(copy.deepcopy(arg.array.index_slices)) 1411 | index_slices[-1][concat_ind] = [None] 1412 | 1413 | ret.index_slices = index_slices[0] 1414 | for i in range(1, len(args) - 1): 1415 | ret.index_slices = Array.join_index_slices(ret.index_slices, index_slices[i]) 1416 | tmp_ret_index_slices = copy.deepcopy(ret.index_slices) 1417 | ret.index_slices[concat_ind] = [] 1418 | split_point = 0 1419 | for i in range(len(args) - 1): 1420 | tmp_ret_index_slices[concat_ind] = args[i].array.index_slices[concat_ind] 1421 | ret.index_slices[concat_ind] += list(np.array(args[i].array.index_slices[concat_ind]) + split_point) 1422 | tmp_keys = args[i].array.get_corresponding_keys(tmp_ret_index_slices) 1423 | tmp_ret_index_slices[concat_ind] = list(np.array(args[i].array.index_slices[concat_ind]) + split_point) 1424 | split_point += int(args[i].array.index_slices[concat_ind][-1]) 1425 | ii = 0 1426 | for indexes in product(*tmp_ret_index_slices): 1427 | ret.block_to_symbol[tuple(indexes)] = tmp_keys[ii] 1428 | ii += 1 1429 | 1430 | return ret 1431 | 1432 | @staticmethod 1433 | def identity(args: list, node): 1434 | assert len(args) == 1 1435 | return args[0].array 1436 | 1437 | @staticmethod 1438 | def zeroslike(args: list, node): 1439 | assert len(args) == 1 1440 | ret = Array("tmp", args[0].size) 1441 | if len(ret.block_to_symbol.keys()) == 0: 1442 | return None 1443 | x = list(ret.block_to_symbol.keys())[0] 1444 | ret.block_to_symbol[x].value = {} 1445 | ret.block_to_symbol[x].map_to_index = {} 1446 | 1447 | return ret 1448 | 1449 | @staticmethod 1450 | def relu(args: list, node): 1451 | # right now it will abort when it encounters relu(z=x-y). 1452 | # A better approach is to set it to relu(z) instead of aborting. 1453 | assert len(args) == 1 1454 | ret = copy.deepcopy(args[0].array) 1455 | ret.block_to_symbol = {} 1456 | for x in args[0].array.block_to_symbol: 1457 | ret.block_to_symbol[x] = args[0].array.block_to_symbol[x].relu() 1458 | return ret 1459 | 1460 | @staticmethod 1461 | def maximum(args: list, node): 1462 | try: 1463 | len(args[0].size) == len(args[1].size) 1464 | except: 1465 | return None 1466 | assert len(args) == 2 and len(args[0].size) == len(args[1].size) 1467 | one_value = list(args[1].array.block_to_symbol.values()) 1468 | if len(one_value) == 1 and len(one_value[0].value) == 0: 1469 | return InferArray.relu([args[0]], node) 1470 | one_value = list(args[0].array.block_to_symbol.values()) 1471 | if len(one_value) == 1 and len(one_value[0].value) == 0: 1472 | return InferArray.relu([args[1]], node) 1473 | 1474 | @staticmethod 1475 | def neg(args: list, node): 1476 | assert len(args) == 1 1477 | ret = copy.deepcopy(args[0].array) 1478 | for x in ret.block_to_symbol: 1479 | ret.block_to_symbol[x].neg() 1480 | 1481 | return ret 1482 | 1483 | @staticmethod 1484 | def pack(args: list, node): 1485 | assert len(args) >= 1 1486 | if len(args) > 10: 1487 | return None 1488 | pack_ind = int(node.attr["axis"].i) 1489 | for i in range(1, len(args)): 1490 | try: 1491 | len(args[0].size) == len(args[i].size) 1492 | except: 1493 | return None 1494 | assert len(args[0].size) == len(args[i].size) 1495 | for j in range(len(args[i].size)): 1496 | try: 1497 | int(args[0].size[j]) 1498 | int(args[i].size[j]) 1499 | except: 1500 | return None 1501 | assert int(args[0].size[j]) == int(args[i].size[j]) 1502 | 1503 | ret = Array("tmp", args[0].size) 1504 | ret.block_to_symbol = dict() 1505 | index_slices = [] 1506 | for arg in args: 1507 | index_slices.append(copy.deepcopy(arg.array.index_slices)) 1508 | ret.index_slices = index_slices[0] 1509 | for i in range(1, len(args)): 1510 | ret.index_slices = Array.join_index_slices(ret.index_slices, index_slices[i]) 1511 | tmp_ret_index_slices = copy.deepcopy(ret.index_slices) 1512 | ret.index_slices = ret.index_slices[:pack_ind] + [[]] + ret.index_slices[pack_ind:] 1513 | 1514 | for i in range(len(args)): 1515 | ret.index_slices[pack_ind] += [i + 1] 1516 | tmp_keys = args[i].array.get_corresponding_keys(tmp_ret_index_slices) 1517 | ii = 0 1518 | for indexes in product(*tmp_ret_index_slices): 1519 | tmp_key = list(indexes) 1520 | tmp_key = tmp_key[:pack_ind] + [i + 1] + tmp_key[pack_ind:] 1521 | ret.block_to_symbol[tuple(tmp_key)] = tmp_keys[ii].add_pack_ind(pack_ind) 1522 | ii += 1 1523 | 1524 | return ret 1525 | 1526 | @staticmethod 1527 | def transpose(args: list, node): 1528 | assert len(args) == 2 1529 | assert not isinstance(args[1].value, Range) 1530 | ret = Array("tmp", args[0].size) 1531 | ret.index_slices = [] 1532 | ret.block_to_symbol = {} 1533 | perm = np.array(args[1].value) 1534 | for x in perm: 1535 | ret.index_slices.append(args[0].array.index_slices[x]) 1536 | for indexes in product(*args[0].array.index_slices): 1537 | new_indexes = () 1538 | for x in perm: 1539 | new_indexes += (indexes[x],) 1540 | 1541 | ret.block_to_symbol[new_indexes] = args[0].array.block_to_symbol[tuple(indexes)].transpose(perm) 1542 | 1543 | return ret 1544 | 1545 | @staticmethod 1546 | def unpack(args: list, node): 1547 | assert len(args) == 1 1548 | axis = int(node.attr["axis"].i) 1549 | index_slices = copy.deepcopy(args[0].array.index_slices) 1550 | try: 1551 | if int(args[0].size[axis]) > 10: 1552 | return None 1553 | except: 1554 | return None 1555 | 1556 | rets = [] 1557 | for i in range(int(args[0].size[axis])): 1558 | rets.append(Array("tmp", args[0].size)) 1559 | rets[-1].index_slices = index_slices[:axis] + index_slices[axis + 1:] 1560 | rets[-1].block_to_symbol = {} 1561 | 1562 | length = index_slices[axis][-1] 1563 | index_slices[axis] = list(range(1, length + 1)) # e.g., 4 -> [1,2,3,4] 1564 | tmp_keys = args[0].array.get_corresponding_keys(index_slices) 1565 | ii = 0 1566 | for indexes in product(*index_slices): 1567 | tmp_key = list(indexes) 1568 | which = indexes[axis] - 1 1569 | tmp_key = tmp_key[:axis] + tmp_key[axis + 1:] 1570 | rets[which].block_to_symbol[tuple(tmp_key)] = tmp_keys[ii].remove_unpack_axis(axis) 1571 | ii += 1 1572 | 1573 | return rets if len(rets) > 1 else rets[0] 1574 | -------------------------------------------------------------------------------- /analysis_main.py: -------------------------------------------------------------------------------- 1 | import z3 2 | import math 3 | import sys 4 | import os 5 | 6 | from parse.parse_graph import Graph 7 | import parse.parse_format_text 8 | from parse.specified_ranges import SpecifiedRanges 9 | from solver import Range, meet 10 | from utils import OVERFLOW_LIMIT, UNDERFLOW_LIMIT 11 | 12 | if __name__ == "__main__": 13 | sys.setrecursionlimit(100000) 14 | try: 15 | assert len(sys.argv) >= 2 and len(sys.argv) <= 3 16 | pbtxt = sys.argv[1] 17 | assert pbtxt[-6:] == ".pbtxt" 18 | if len(sys.argv) == 3: 19 | assert sys.argv[2] in ["unbounded_weight", "unbounded_input"] 20 | if sys.argv[2] == "unbounded_weight": 21 | parse.parse_format_text.unbounded_weight = True 22 | else: 23 | parse.parse_format_text.unbounded_input = True 24 | 25 | except: 26 | print( 27 | "Please run 'python analysis_main.py PBTXT_FILENAME'.\nAborted...") 28 | exit(1) 29 | 30 | rule = ["Log", "Exp", "RealDiv", "Sqrt", "Rsqrt", "Expm1", "Log1p", "Reciprocal"] 31 | 32 | network_name = os.path.basename(pbtxt)[:-6] 33 | if network_name in SpecifiedRanges.specified_ranges: 34 | SpecifiedRanges.ranges_looking_up = SpecifiedRanges.specified_ranges[network_name] 35 | 36 | graph = Graph(pbtxt, "verbose.txt") 37 | suspected_nodes = [] 38 | for node in graph.graph_def.node: 39 | if node.op in rule and graph.f.find(node.name) == graph.main_clique: 40 | suspected_nodes.append(node) 41 | print(graph.get_info()) 42 | 43 | cnt_all = 0 44 | cnt_sat = 0 45 | cnt_unknown = 0 46 | cnt_unsat = 0 47 | for suspected_node in suspected_nodes: 48 | # calculate the range of input of the unsafe operations 49 | if suspected_node.op in ["RealDiv", "Floormod"]: 50 | # special treatment for div because we only care about the denominator 51 | ret = graph.forward_analysis(graph.node_by_name[graph.graph_backward[0][suspected_node.name][1]], 52 | suspected_node) 53 | else: 54 | ret = graph.forward_analysis(suspected_node) 55 | if ret is None: 56 | continue 57 | 58 | if suspected_node.op in ["Exp", "Expm1"]: 59 | suspected_node_input = Range(left=math.log(OVERFLOW_LIMIT), right=None, const_type=0) 60 | backward_analysis_const_start = graph.graph_backward[0][suspected_node.name][0] 61 | index = graph.edge_index[suspected_node.name][0] 62 | elif suspected_node.op in ["RealDiv", "Floormod"]: 63 | suspected_node_input = Range(left=-UNDERFLOW_LIMIT, right=UNDERFLOW_LIMIT, const_type=0) 64 | backward_analysis_const_start = graph.graph_backward[0][suspected_node.name][1] 65 | index = graph.edge_index[suspected_node.name][1] 66 | elif suspected_node.op == "Log": 67 | suspected_node_input = Range(left=None, right=UNDERFLOW_LIMIT, const_type=0) 68 | backward_analysis_const_start = graph.graph_backward[0][suspected_node.name][0] 69 | index = graph.edge_index[suspected_node.name][0] 70 | elif suspected_node.op == "Sqrt": 71 | suspected_node_input = Range(left=None, right=-UNDERFLOW_LIMIT, const_type=0) 72 | backward_analysis_const_start = graph.graph_backward[0][suspected_node.name][0] 73 | index = graph.edge_index[suspected_node.name][0] 74 | elif suspected_node.op == "Rsqrt": 75 | suspected_node_input = Range(left=None, right=UNDERFLOW_LIMIT, const_type=0) 76 | backward_analysis_const_start = graph.graph_backward[0][suspected_node.name][0] 77 | index = graph.edge_index[suspected_node.name][0] 78 | elif suspected_node.op == "Log1p": 79 | suspected_node_input = Range(left=-UNDERFLOW_LIMIT - 1, right=UNDERFLOW_LIMIT - 1, const_type=0) 80 | backward_analysis_const_start = graph.graph_backward[0][suspected_node.name][0] 81 | index = graph.edge_index[suspected_node.name][0] 82 | elif suspected_node.op == "Reciprocal": 83 | suspected_node_input = Range(left=-UNDERFLOW_LIMIT, right=UNDERFLOW_LIMIT, const_type=0) 84 | backward_analysis_const_start = graph.graph_backward[0][suspected_node.name][0] 85 | index = graph.edge_index[suspected_node.name][0] 86 | else: 87 | raise NotImplementedError("No rule for ", suspected_node.op) 88 | 89 | 90 | # check whether the input_range intersects with its danger zone 91 | # return true if dose no intersect; otherwise, return false 92 | def is_valid(input_range): 93 | additional_constraint = meet(input_range, suspected_node_input) 94 | S = z3.Solver() 95 | S.add(additional_constraint) 96 | ans = S.check() 97 | assert ans != z3.unknown 98 | return ans == z3.unsat 99 | 100 | 101 | # check whether the unsafe operation's input is valid 102 | def is_valid_by_split(): 103 | # if it is valid without predicate splitting 104 | if is_valid(graph.node_output[backward_analysis_const_start].index_of(index).value): 105 | return True 106 | else: 107 | # otherwise, try predicate splitting 108 | range_to_split, nodes_interested = ret 109 | range_to_split = list(range_to_split) 110 | for name in range_to_split: 111 | override_dict = {} 112 | # if the name has |, we have to remove it to get the name in the graph 113 | changed = set() 114 | if name.find('|') != -1: 115 | changed.add(name[:name.find('|')]) 116 | else: 117 | changed.add(name) 118 | value = graph.get_value(name) 119 | if value.left < 0 and value.right > 0: 120 | spans = [Range(left=value.left, right=0), Range(left=0, right=value.right)] 121 | is_span_valid = True 122 | for span in spans: 123 | override_dict[name] = span 124 | # incrementally rerun the dataflow analysis on changed node set with the node output 125 | # overridden to override_dict 126 | node_out = graph.reevaluate(nodes_interested, backward_analysis_const_start, changed, 127 | override_dict) 128 | if not is_valid(node_out.index_of(index).value): 129 | is_span_valid = False 130 | break 131 | 132 | if is_span_valid: 133 | return True 134 | 135 | return False 136 | 137 | 138 | if not is_valid_by_split(): 139 | print(suspected_node.op, suspected_node.name) 140 | print("warning") 141 | cnt_sat += 1 142 | else: 143 | cnt_unsat += 1 144 | cnt_all += 1 145 | print(network_name, ", all: ", cnt_all, "\twarnings: ", cnt_sat, "\tsafe: ", cnt_unsat) 146 | -------------------------------------------------------------------------------- /appendix.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ForeverZyh/DEBAR/3a2880697fa67b6b1beb127f1fc773b690f1c67c/appendix.pdf -------------------------------------------------------------------------------- /docs/analysis.md: -------------------------------------------------------------------------------- 1 | # Analysis 2 | 3 | `analysis` folder contains the definition fo abstracted values and abstract interpretations for operations in computation graphs. 4 | 5 | * `abstract_interpretation.py` contains the class type of abstracted values that we use for our tensor abstraction and interval abstraction with affine relations. 6 | The main component of `abstract_interpretation.py` is the `AbstractInterpretation` class, which is the data structure of abstracted values. It contains: 7 | 8 | * `size`: the shape of the tensor extracted from the protocol buffer format. The shape of the tensor may have an unknown dimension marked as $-1$ or ?. All shapes are inferred by TensorFlow. 9 | * `dtype`: the data type of the tensor extracted from the protocol buffer format. All data types are inferred by TensorFlow. 10 | * `value`: the interval abstraction stored in a `Range` object or a `numpy` concrete value. 11 | * `array`: the tensor partition stored in an `Array` object. 12 | * `constraints`: deprecated, used to store the z3 constraints generated alongside dataflow analysis. 13 | 14 | Notice that the value of `size`, `dtype`, `value`, and `array` can be a list because the output value of a node can be a list. 15 | 16 | * `index_of(self, i)` gets the $i$-th index of all the fields and returns a new `AbstractInterpretation` object. It returns `self` if `i` is `None`. 17 | * `has_none(self)` checks whether some of the fields are `None`, which indicates that dataflow analysis cannot infer this abstracted value due to unimplemented TensorFlow APIs. We do not necessarily need to throw an exception or to generate a warning to the unsafe operation under detection, because the input range of the unsafe operations may not depend on the unimplemented TensorFlow API. 18 | 19 | * `inference.py` contains the abstract interpretations for operations in computation graphs. 20 | 21 | * `real_size(a, b)` infers real size of `a` and `b` under the assumption that a = b, even though one of them might be unknown, i.e., equals to ?. 22 | * `dumy()`: returns an unbounded interval abstraction with [-inf, +inf]. 23 | * `safeXXX(...)` are functions that calculate arithmetic function `XXX` in a numerical safe manner. 24 | * `identity(args, node)`: the abstract interpretation of identity. It returns `None` if the input is a zero-size array with no concrete value. If the input is already an interval abstraction, then the input is returned. Otherwise, it converts the concrete `numpy` array (tensor) into its interval abstraction. `identity` will be called by some operations that only change the shape of the tensor, but will not change the values in the tensor or will only remove some values from the tensor so it is sound to abstract the operation by identity. 25 | * `packtorange(args, node)`: the abstract interpretation of joining ($\sqcup$) of a list of interval abstractions. It computes the lower bound of the output as the minimum of all lower bounds of abstracted inputs and concrete inputs. It computes the upper bound of the output as the maximum of all upper bounds of abstracted inputs and concrete inputs. `packtorange ` will be called by some operations like `pack`, `concatv2` that merge multiple tensors into one. 26 | * `InferValue` contains the abstract interpretations of TensorFlow APIs used in interval abstraction + tensor smashing. 27 | * All member methods in `InferValue` are static. 28 | * Their names are the same as the lowercase operation names of the corresponding TensorFlow APIs in the protocol buffer format. This property should be preserved because the methods are being called by their names. 29 | * All methods accept two arguments, the first one is a list of abstracted values describing the inputs to the TensorFlow API, the second one is the node attribute in the protocol buffer format. The node attribute is used to extract additional information for DEBAR to understand the semantics of the API. 30 | * The methods return a `Range` object (or a concrete `numpy` array or a list of them). 31 | * `InferArray` contains the abstract interpretations of TensorFlow APIs used in the tensor partition and the linear affine relation. 32 | * All member methods have the same first three properties as described in `InferValue`. 33 | * The methods return an `Array` object (or a list of them). 34 | * Notice that ideally the `InferArray` shoule be split to `InferArray` and `InferLinear`, but the coupling of the tensor partition and the linear affine relation are so strong that we decide to implement them together in the `InferArray`. 35 | 36 | ## Contributes to DEBAR by Implementing the Abstract Interpretations of TensorFlow APIs 37 | 38 | We encourage the developers to contributes to DEBAR by implementing abstract interpretations of other TensorFlow APIs that are not handled by DEBAR. This Section is a guideline for implementing the abstract interpretations. 39 | 40 | 1. Please read the description of `InferValue` and `InferArray` in the previous Section. If you want to contribute to `InferValue`, please also read the `Range` class in [Overview](./overview.md). If you want to contribute to `InferArray`, please also read the `Array` and `Linear` class in [Overview](./overview.md). 41 | 2. Contribute to `InferValue`: `InferValue` contains the abstract interpretations of TensorFlow APIs used in interval abstraction. Please make sure you add a method that: 42 | 1. The method is static. 43 | 2. The method name is the same as the lowercase operation name of the TensorFlow API in the protocol buffer format. 44 | 3. The method accepts two arguments, the first one is a list of abstracted values describing the inputs to the TensorFlow API, the second one is the node attributes in the protocol buffer format. 45 | 4. Please check the arity of the first argument if possible. 46 | 5. Please make sure the abstract interpretation is sound. If you cannot handle some cases, it is always sound to return `None` or `dumy()` which instructs the dataflow analysis that part of the implementation is not available or the output range is unbounded. 47 | 6. Returns a new `Range` object storing the results of the abstracted TensorFlow API. 48 | 3. Contribute to `InferArray`: `InferArray` contains the abstract interpretations of TensorFlow APIs used in the tensor partition and the linear affine relation. Please make sure you add a method that meets the 6 requirements above. 49 | Notice that it is not necessary to have an abstract interpretation in `InferArray` for any TensorFlow APIs. It is recommended to implement unhandled affine transformations. It is also recommended to implement some shape transformations whose input and output have the same interval abstraction. 50 | 51 | -------------------------------------------------------------------------------- /docs/dynamic_tool.md: -------------------------------------------------------------------------------- 1 | # Dynamic Tool 2 | 3 | `dynamic_tool` is a dynamic fuzzing tool guided by gradients to detect numerical bugs. The fuzzing tool does not require concrete parameters in the neural networks (trained models). The fuzzing tool can significantly reduce the time and training epoch needed to trigger the numerical bugs by injecting fuzzing process inside neural network training. 4 | 5 | NOTICE: The folder `dynamic_tool` does not belong to the ESEC/FSE 2020 paper. For more information, please see [my undergraduate dissertation](./dynamic_tool/Yuhao Zhang-undergraduate dissertation.pdf). Unfortunately, only the Chinese version is available (a English abstract as well), I am glad to translate this paper upon request. 6 | 7 | -------------------------------------------------------------------------------- /docs/imgs/fig0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ForeverZyh/DEBAR/3a2880697fa67b6b1beb127f1fc773b690f1c67c/docs/imgs/fig0.png -------------------------------------------------------------------------------- /docs/imgs/fig1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ForeverZyh/DEBAR/3a2880697fa67b6b1beb127f1fc773b690f1c67c/docs/imgs/fig1.png -------------------------------------------------------------------------------- /docs/imgs/fig2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ForeverZyh/DEBAR/3a2880697fa67b6b1beb127f1fc773b690f1c67c/docs/imgs/fig2.png -------------------------------------------------------------------------------- /docs/overview.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | We describe the functionality of each python source file and each folder in the followings: 4 | 5 | * `analysis` folder contains the definition fo abstracted values and abstract interpretations for operations in computation graphs. 6 | 7 | * `abstract_interpretation.py` contains the class type of abstracted values that we use for our tensor abstraction and interval abstraction with affine relations. 8 | * `inference.py` contains the abstract interpretations for operations in computation graphs. 9 | 10 | For more information please see [Analysis](./analysis.md). 11 | 12 | * `parse` folder contains the parsing process of the Protocol Buffer format to the computation graph, the process of static dataflow analysis, the parsing process of values, and the user-specified weights/inputs ranges. 13 | 14 | * `parse_graph.py` contains the parsing process of the Protocol Buffer format to the computation graph and the process of static dataflow analysis. 15 | * `parse_format_text.py` contains the parsing process of constant values, variables, and placeholders. 16 | * `specified_ranges.py` contains the reusable weights/inputs ranges specified by users. 17 | 18 | For more information please see [Parse](./parse.md). 19 | 20 | * `analysis_main.py` is the entry of DEBAR. It takes one argument that is the target Protocol Buffer file containing the computation graph under verification. It also takes one option argument denoting whether to specify the range of the weights and the range of the inputs. Please see **Running DEBAR** Section in [README](../README.md) for how the usage of DEBAR. 21 | The workflow of `analysis_main.py`: 22 | 23 | * First, it calls `parse_graph.py` to obtain the computation graph. 24 | * Second, it scans the list of unsafe operations and calls the dataflow analysis in `parse_graph.py` to get the range of the input to the unsafe operations. 25 | * Third, it checks whether the range of the input to the unsafe operation intersects with its danger zone. 26 | * If safe, then the unsafe operation is verified to be safe. 27 | * Otherwise, go to the next step. 28 | * Fourth, if the range of input to the unsafe operation cannot be proved as safe, we will further split the ranges of some nodes using constant predicates such as 0. If all the splits of any node can prove the range of input to the unsafe operation does not intersect with its danger zone, then the operation is safe. The motivation and the details of predicate splitting can be found at **Predicate Splitting** Section. 29 | * If safe, then the unsafe operation is verified to be safe. 30 | * Otherwise, DEBAR generates a warning for the unsafe operation. 31 | 32 | * `main.py` is the entry of reproducing the evaluation results reported in the paper. It takes one argument that is the path to the downloaded datasets. 33 | Please see **Reproduce Evaluation in our Paper** Section in [README](../README.md) for how to reproduce the evaluation results in our paper. 34 | 35 | * `solver.py` 36 | 37 | * `Solver` is a legacy class that was used in z3-solver aided dataflow analysis. We are considering removing it because it is never used. 38 | 39 | * `Range` is a data structure for interval abstraction. `left` and `right` denote the lower and upper bound of the abstracted value. 40 | 41 | * `check_range_const(range_const)` checks whether a `Range` object `range_const` has const lower and upper bound. It is also a legacy function. Because we do not use z3-solver any more, the function should always return true, also considering removing it. 42 | 43 | * `meet(range, range_const)` checks whether the interval of `range` intersects with the interval of `range_const`. In other words, `meet` returns true iff `range` $\cap$ `range_const` $= \varnothing$. 44 | `meet` will be called in `analysis_main.py` to check whether the interval bound of unsafe operations computed by static analysis intersects with their danger zone. 45 | 46 | * `Array` is the data structure supporting the tensor partitioning. It mainly contains: 47 | 48 | * `index_slices`: a list stores the partitioning positions of each dimension. If the tensor has dimension $d$ then `index_slices` contains $d$ tuples, each of which is the set of partitioning positions. If the $i$-th dimension is partitioned to $[0,p^{(i)}_0), [p^{(i)}_0,p^{(i)}_1), \dots, [p^{(i)}_{m-1},p^{(i)}_m)$, where $p^{(i)}_m$ is equal to the size of $i$-th dimension, then the $i$-th tuple of `index_slices` will be $(p^{(i)}_0,p^{(i)}_2,\ldots,p^{(i)}_m)$. 49 | Figure0 50 | 51 | For example, a two-dimension tensor (matrix) A has the shape $3\times 4$ and it is partitioned into 6 partitions $[0,1)\times [0,2)$, $[0,1)\times [2,3)$, $[0,1)\times [3,4)$, $[1,3)\times [0,2)$, $[1,3)\times [2,3)$, $[1,3)\times [3,4)$. Then the `index_slices` will be `[(1, 3), (2, 3, 4)]`. 52 | 53 | * `block_to_symbol`: a map maps from each partition to a `Linear` object, which maintains the linear affine relation. Each partition is defined by the Cartesian product of $d$ tuples in `index_slices`, called **partitioning positions**. 54 | 55 | Taking the above example, the keys of the map `block_to_symbol` will be a set of 6 $2$-tuples: $(1,2),(1,3),(1,4),(3,2),(3,3),(3,4)$, each of which denotes the ending points of partitions in all dimensions. 56 | 57 | * `join_index_slices(a, b)` aligns two sets of partitioning positions `a` and `b`. Both `a` and `b` are in the form of `Array.index_slices`. 58 | 59 | For example, suppose `a = [(1,3), (2,3,4)]` and `b=[(3), (1,4)]`, then the aligned partitioning positions `c` will be `[(1,3), (1,2,3,4)]`. It can be seen that `c` has a finer granularity of partitioning than `a` and `b` in this case. 60 | 61 | * `get_corresponding_keys(self, index_slices)` gets the corresponding `Linear` objects according to `index_slices`. Notice that `index_slices` may have a finer granularity of partitioning than `self.index_slices`, so the `Linear` object (as well as the variables stored in the `Linear` object) may need to be further partitioned. 62 | ![Figure1](./imgs/fig1.png) 63 | 64 | For example, suppose `self.index_slices=[(1, 3), (2, 3, 4)]` and `index_slices = [(1, 3), (1, 2, 3, 4)]`, then the partition $[1,3)\times[0,1)$ corresponds to the partition $[0,2)\times[0,1)$ of `self.block_to_symbol[(3,2)]`. 65 | 66 | * `Linear` is the data structure supporting the linear affine relation. For example, considering the following affine relation: 67 | $$ 68 | 3x-relu(x)+4y-z+5=0. 69 | $$ 70 | Each `Linear` object has a main variable, because each `Linear` object is stored in the `block_to_symbol` field in an `Array` object, and the `Array` object is the abstracted value of the main variable. For example, $z$ may be the main variable in the above affine relation, then what is stored in the `Linear` object is the following **affine expression**: 71 | $$ 72 | 3x-relu(x)+4y+5, 73 | $$ 74 | whose semantics is $z=3x-relu(x)+4y+5$, where $z$ is omitted since it can be inferred since $z$ is the main variable. 75 | 76 | In order to store such affine expression, `Linear` uses: 77 | 78 | * `value`: a map maps from variables to their factors. Taking the above example, `value[x] = 3`, `value[relu(x)] = -1`, `value[y] = 4`, and `value[CONST] = 5`. 79 | And each variable is a partition of a tensor which is the output of one operation. The variable is defined as a pair of `(name, indexes)`, where the `name` is the name of the operation, and `indexes` is the partitioning positions. 80 | 81 | * `map_to_index`: a map maintains an index mapping. The purpose of maintaining this index mapping is that additional dimensions may be added after operations like `pack` , the dimensions may be deleted (these dimensions are all equal to 1 and do not change the size of the partition) after operations like `unpack`, the dimensions may be permuted after operations like `transpose`. 82 | Considering the following code: 83 | 84 | ```python 85 | z = transpose(x) 86 | ``` 87 | 88 | , where `z` is a matrix $3\times 4$ and `x` is a matrix $4 \times 3$. Suppose there is only one (whole) partition of `z`, then the tensor partition `Array` of `z` contains `index_slices=[(3,), (4,)]` and `block_to_symbol=[(3,4)]` maps to the linear affine expression `x`, where the partition positions of `x` are `[(4,), (3,)]`. Notice that the 0-th dimension of `z` is the 1-th dimension of `x` and the 1-th dimension of `z` is the 0-th dimension of `x`. 89 | 90 | The semantics of `map_to_index` is that `z[t]` corresponds to `x[map_to_index[t]]`. 91 | 92 | * `__add__(self, other)`, `__sub__(self, other)`: adds/subs between two affine expressions and returns a new `Linear` object. 93 | 94 | * `neg(self)`: calculates the negation of the affine expression. 95 | 96 | * `choose(self, start_ind)`: further partitions the variables inside the `Linear` object and returns a new partitioned `Linear` object. Recall in `Array.get_corresponding_keys`, we may further partition the `Linear` object as well as the variables stored in the `Linear` object. 97 | 98 | * `add_pack_ind(self, pack_ind)`: adds an axis at the `pack_ind`-th dimension and returns a new packed `Linear` object. 99 | 100 | * `remove_unpack_axis(self, axis)`: removes an axis at the `axis`-th dimension and returns a new unpacked `Linear` object. 101 | 102 | * `transpose(self, perm)`: transposes the ``map_to_index`` according to the permutation `perm` and returns a new transposed `Linear` object. 103 | 104 | * `relu(self)`: calculates the $relu$ of the affine expression and returns a new `Linear` object. It only supports calculating the relu of a singleton affine expression that only contains one variable or one constant value, e.g., `x`, `-x`, `relu(x)`, `-relu(x)`, and constant value `c`. 105 | The following axioms are used to calculate $relu$: 106 | $$ 107 | relu(x)=relu(x)\\ 108 | relu(-x)=-x+relu(x)\\ 109 | relu(relu(x))=relu(x)\\ 110 | relu(-relu(x))=0\\ 111 | relu(c)=max(c,0). 112 | $$ 113 | 114 | * `meet_relation_variable(rv, range_const)` is never used, also considering removing it. 115 | 116 | * `utils.py` 117 | 118 | * `OVERFLOW_LIMIT`, `UNDERFLOW_LIMIT`, `OVERFLOW_D`, and `UNDERFLOW_D` specify the overflow and underflow limit in `tf.float32`. 119 | * `resolve_type(y)` converts `y` from data types in `numpy` to python primitive data types. 120 | * `shape_from_proto(shape)` parses the tensor shape from protocol buffer format `shape` into a python list. 121 | 122 | ## Predicate Splitting 123 | 124 | The workflow of `analysis_main.py` has been described previously. We further describe the motivation and the details of predicate splitting. 125 | 126 | ### Motivation 127 | 128 | Considering the following expression: 129 | $$ 130 | y = e^{-relu(x)} + e^{x-relu(x)} 131 | $$ 132 | 133 | 134 | ### fig2 135 | 136 | The range of $y$ is $(1,2]$ if the range of $x$ is $[-50,40]$. Using the interval abstraction with affine relation, we are able to calculate the range of $-relu(x)$ is $[-40,0]$ and the range of $x-relu(x)$ is $[-50,0]$. Then the range of $e^{-relu(x)}$ is $[0,1]$ and the range of $e^{x-relu(x)}$ is $[0,1]$, leading the range of $y$ to be $[0,2]$. 137 | 138 | The range of $y$ is an over-approximation because $e^{-relu(x)}$ is decreasing and $e^{x-relu(x)}$ is increasing. Besides, due to the nonlinearity of the exponential function, affine relation alone cannot eliminate the over-approximation of $y$. However, we can infer that $y$ depends on $x$ nonlinearly. 139 | 140 | When we find out a variable $y$ is depends on another variable $x$ nonlinearly, we apply the **predicate splitting** technique, which further splits the range of $x$ according to some constant predicates like 0, and then merge the ranges of $y$ to get a preciser result. 141 | 142 | In this case, if we split the range of $x$ to $[-50, 0]\cup [0, 40]$. 143 | 144 | * $x=[-50,0]$: we are able to calculate the range of $-relu(x)$ is $[0,0]$ and the range of $x-relu(x)$ is $[-50,0]$. Then the range of $e^{-relu(x)}$ is $[1,1]$ and the range of $e^{x-relu(x)}$ is $[0,1]$, leading the range of $y$ to be $[1,2]$. 145 | * $x=[0,40]$: we are able to calculate the range of $-relu(x)$ is $[-40,0]$ and the range of $x-relu(x)$ is $[0,0]$. Then the range of $e^{-relu(x)}$ is $[0,1]$ and the range of $e^{x-relu(x)}$ is $[1,1]$, leading the range of $y$ to be $[1,2]$. 146 | 147 | After merge the two ranges of $y$, we get the precise range $[1,2]$. 148 | 149 | ### Details 150 | 151 | For each unsafe operation under verification, `analysis_main.py` first calls `graph.forward_analysis` in `parse_grahp.py` to get the dataflow analysis results of the computation graph. The analysis results are stored in `graph.node_output`, and the return values of `graph.forward_analysis` contain the ranges of node needed to be split `range_to_split`. 152 | 153 | Function `is_valid_by_split` checks whether the unsafe operation's input is valid. It first checks whether the input ranges of the unsafe operation is valid without without predicate splitting, if false, it tries to split each node in `range_to_split` and reevaluate the dataflow analysis in an incremental manner by calling `graph.reevaluate`. If the merged result of any split node is valid, then the input ranges of the unsafe operation is proved to be valid. 154 | 155 | Theoretically, the more splits we try the preciser results we will get. In the implementation, we only try to split the range into two splits and the constant predicate is always 0. 156 | 157 | -------------------------------------------------------------------------------- /docs/parse.md: -------------------------------------------------------------------------------- 1 | # Parse 2 | 3 | * `parse_graph.py` contains the parsing process of the Protocol Buffer format to the computation graph and the process of static dataflow analysis. 4 | 5 | * `UnionSet` implements the [disjoint-set data structure](https://en.wikipedia.org/wiki/Disjoint-set_data_structure) for identifying the largest connected component in the parsed computation graph. 6 | 7 | * `Graph` mainly implements the parsing process of the Protocol Buffer format to the computation graph and the process of static dataflow analysis, as well as other functionalities that are related to the computation graph. We describe the main components of `Graph`. 8 | 9 | * `graph_backward` is a field storing the reversed edges of the computation graph. `graph_backward[0]` stores the dataflow edges and `graph_backward[1]` stores the control flow edges. The `graph_backward[0]` is seldomly used since we only care about the data flow. 10 | 11 | * `graph_forward` is a field storing the edges of the computation graph. `graph_forward[0]` stores the dataflow edges and `graph_forward[1]` stores the control flow edges. Similarly, the `graph_forward[0]` is seldomly used since we only care about the data flow. 12 | 13 | * `node_by_name` is a map mapping from the name of an operation (string) to the node attribute in protocol buffer format. 14 | 15 | * `node_output` is a map mapping from the name of an operation to an `AbstractInterpretation` object (or a list of `AbstractInterpretation` objects) denoting the abstracted output of the node computed by dataflow analysis. 16 | One thing to mention is that if the output of a node "x" is a list of tensors, we will use an instrumented string "x|i" to denote the i-th element in the list. 17 | 18 | * `edge_index` is a map mapping from the name of an operation to a list. The list indicates which value is passed to the next node if the output of the current node is a list of `AbstractInterpretation` objects. 19 | 20 | For example, node `x` has three edges `x -> y0`, `x -> y1`,`x -> y2` in order and the output of `x` is a list of `AbstractInterpretation` objects `[a0, a1, a2, a3]`. Suppose that `x` passes `a0` to `y0`, `a3` to `y2`, and `a2` to `y3`, then `self.edge_index[x] = [0, 3, 2]`. 21 | 22 | * `node_visited` is a set storing which nodes have been visited by dataflow analysis and it is used for incremental dataflow analysis. 23 | 24 | * `tensor_to_op` is a map mapping from tensor name to the name of the operation (node) that generates this tensor. However, a small number of node inputs are named by the tensor names but not by the operation names. The purpose of this map is to unify the naming rule. 25 | 26 | * `nodes_in_main_clique_topology` is a map mapping from an operation name to its topological order, instructing the order of dataflow analysis. We first identify the DAG part of the graph using the topological traverse of the graph. Then we identify the loops in the graph and mark the loop entries. At last, we specify the order of traversing the loop to get the topological order of loops as well. 27 | 28 | * `build(self)` parses the Protocol Buffer format, builds the computation graph, and the topological order of the nodes. The `size`, `dtype` of `AbstractInterpretation` in `node_output` will be extracted from protocol buffer format in `build` method. 29 | 30 | * `backward_slice(self, node, visited, non_control_only)` returns a list of nodes in the backward slice starting at `node`. `visited` is a set recording which nodes have already been visited to avoid potential loops. `non_control_only` is a flag instructing the method whether to visit control flow edges. 31 | 32 | * `summary_node(self, son, u, override_dict)` calculates the abstracted output of node `son` with its attribute `u` in protocol buffer format according to the abstractions of its inputs while the abstracted outputs of some nodes have been overridden in `override_dict`. `override_dict` is a map mapping the names to their overridden abstractions. It will only be used in **predicate splitting** (see [Overview](./overview.md)) and **handling element-wise `Select` operation **(see next section). 33 | This method mainly contains two parts: 34 | 35 | 1. The logic of computing `value` and `array` of `AbstractInterpretation` in `node_output`. It first computes `value` and `array` using the abstract interpretations in `analysis/inference.py`. Then it further improves the precision of `value` (interval abstraction + tensor smashing) by the information in `array` computed by the tensor partition and the linear affine relation. Notice that the results of the tensor partition and the linear affine relation will be provably more precise than or equal to the results of interval abstraction + tensor smashing. Thus, as long as the result of `array` is available, we will use the results of the tensor partition and the linear affine relation as `value`. `get_left_right` method computes the results of the tensor partition and the linear affine relation. 36 | 2. Abstract interpretation of the element-wise `Select` operation. Ideally, this part should be located in `analysis/`. However, the coupling between `Select` operation and dataflow analysis is so strong that we decide to leave it in `parse_graph.py`. Considering to refactor it into `analysis/`. The detail of this part can be found in the next section. 37 | 38 | * `forward_analysis(self, node_interested, appended)` is the body of dataflow analysis. It computes the abstracted output of `node_interested`, and returns the ranges for **predicate splitting** (see [Overview](./overview.md)). `appended` is the node of the unsafe operation. `node_interested` is one input of `appended`. In most of the cases, `node_interested` is the only input of `appended`. For operations like `RealDiv`, we only care about the denominator so `node_interested` will be the second input of `appended` (denoting the denominator). 39 | 40 | 1. First, `forward_analysis` computes the backward slice from `node_interested` by calling `backward_slice`, and sorts the nodes in the backward slice in the topological order `nodes_in_main_clique_topology`. 41 | 2. Second, `forward_analysis` calls `summary_node` for every node in the backward slice in the topological order iteratively to get the abstracted output. If the node has already been visited by dataflow analysis, we can skip this node because the abstracted output has been computed when verifying other unsafe operations. 42 | 3. Third, `forward_analysis` collects and returns the ranges for predicate splitting. 43 | 44 | * `reevaluate(self, nodes_interested, node_interested, changed, override_dict)` reevaluates the dataflow analysis for `nodes_interested` which contains the nodes in the backward slice of `node_interested`. The reevaluation is implemented in an incremental manner, which only reevaluates the nodes which will be affected by nodes in `changed`. The abstracted outputs of nodes in `changed` are overridden in `override_dict`. 45 | 46 | * `get_value(self, name)` gets the corresponding abstracted output in `node_output`. It will also consider the specially instrumented name like "x|i" denoting the i-th element in the abstracted output. 47 | 48 | * `get_left_right(self, groups, node_name, override_dict)` computes the abstracted output of `node_name` using the tensor partition and the linear affine relation with values of some nodes overridden by `override_dict` . `groups` is the `block_to_symbol` field of the `Array` object. 49 | The abstracted output is the joining ($\sqcup$) of all the abstracted outputs in tensor partitions stored in `groups`. The joining ($\sqcup$) of interval abstractions can be easily defined: setting the lower bound as the minimum of all lower bounds and the upper bound as the maximum of all upper bounds. 50 | The key is to compute the abstracted output of every tensor partition from the linear affine relation stored in the `Linear` object. Considering the example in Overview: 51 | $$ 52 | 3x-relu(x)+4y+5. 53 | $$ 54 | This expression depends on the abstracted outputs of $x$ and $y$. Since we compute the abstracted outputs in the topological order, the abstracted outputs of $x$ and $y$ must have been computed previously. Thus, the abstracted value of this expression can be computed in the interval arithmetic. Moreover, the cancellation like $(x+y)+(x-y)=2y$ has been handled in `Linear` class. However, the cancellation of $relu$ is handled in `get_left_right` by the following axiom of $relu$ to get a more precise result: 55 | $$ 56 | x - relu(x) = -relu(-x). 57 | $$ 58 | 59 | 60 | Thus, 61 | $$ 62 | \alpha(x - relu(x)) = -_{\alpha}relu_{\alpha}(-_{\alpha}\alpha(x)), 63 | $$ 64 | where $\alpha(t)$ means the interval abstraction of $t$, and $-_{\alpha}$, $relu_{\alpha}$ are negation and $relu$ functions in interval arithmetic. 65 | 66 | For example, we have an expression $x-relu(x)$, where $\alpha(x)=[-1,2]$. Naive calculation $\alpha(x)-_{\alpha}relu_{\alpha}(\alpha(x))$ leads to interval $[-3,2]$. However, using the above axiom of $relu$ leads to interval $[-1,0]$, which is more precise than $[-3,2]$ computed by naive calculation. 67 | 68 | * `parse_format_text.py` contains the parsing process of constant values, variables, and placeholders. 69 | 70 | * `const(node)` parses the constant values from the `node` attribute. 71 | * `iteratorv2(node)`, `oneshotiterator(node)` parse the inputs obtained by the `iteratorv2` and `oneshotiterator` operations and return a list of Range objects. The `node` attribute is used to get to `size` and `dtype` of the inputs. 72 | * `variablev2(node)` parses the weights obtained by the `variablev2` operation, and returns a Range object. The `node` attribute is used to get to `size` and `dtype` of the weights. 73 | * `placeholder(node, weight)` parses the inputs obtained by the placeholder operation, and returns a Range object. The `node` attribute is used to get to `size` and `dtype` of the inputs. `placeholder` can also be called by `variablev2` when `weight=True`. 74 | 75 | * `specified_ranges.py` contains the reusable weights/inputs ranges specified by users. It mainly contains class `SpecifiedRanges` which has two static fields: 76 | 77 | * `models` is a list containing all the architecture names collected in our datasets. Notice that we shortened some of the architecture names to fit into the table in our paper. 78 | * `specified_ranges` is a map storing the reusable weights/inputs ranges specified by users. The map has keys denoting architecture names and values containing another map mapping from variable names to their ranges. A range is a 2-elements list denoting the lower bound and the upper bound. If the lower bound is `None`, it means `-inf` and if the upper bound is `None`, it means `+inf`. We show how we infer these specified ranges for all architectures in the comments. 79 | 80 | ## Abstract Interpretation of the Element-wise `Select` Operation 81 | 82 | We implement the abstract interpretation of the element-wise `select` operation in `summary_node`. The element-wise `select` operation takes three inputs `cond`, `b1`, `b2`, and the return value `ret = select(cond, b1, b2)`, where `cond` is a bool tensor, `b1` , `b2`, and `ret` are two tensors with the same type. Moreover, `cond`, `b1`, `b2`, and `ret` have the same shape. The semantics of element-wise `select` operation over 1-dimension tensors (vectors) is defined as follow: 83 | $$ 84 | ret[i] = b1[i] \text{ if } cond[i] \text{ else } b2[i]. 85 | $$ 86 | The `ret[i]` is equal to `b1[i]` if `cond[i]` evaluates to true, otherwise, `ret[i]` is equal to `b2[i]`. 87 | 88 | ### Motivation 89 | 90 | We get the abstracted values of `cond`, `b1`, and `b2` before analyzing the abstracted value of `ret`. Considering the results obtained by the tensor smashing with the interval abstraction, the abstracted value `cond` vector can be in the following three cases: 91 | 92 | 1. *All true*, then the abstracted value of `ret` is equal to `b1` 93 | 2. *All false*, then the abstracted value of `ret` is equal to `b2` 94 | 3. *Otherwise*, then the abstracted value of `ret` is equal to the joining ($\sqcup$) of `b1` and `b2`. 95 | 96 | Consider the following concrete example: `cond = x > 0`, `b1 = x`, and `b2 = -x`, where `x` is a numerical vector with interval abstraction $\alpha(b1)=\alpha(x)=[-1,2]$ and $\alpha(b2)=\alpha(-x)=[-2,1]$. Thus, the abstraction of `cond` is the case *otherwise*, leading to $\alpha(ret) = [-1,2] \sqcup [-2,1]= [-2,2]$. 97 | 98 | However, this abstraction of `ret` is an over-approximation. We can get a more precise result by considering the range of the numerical vector in `cond`. If a value in `b1` is chosen, it implies that the corresponding element in `x` is greater than $0$. For the same reason, if a value in `b2` is chosen, it implies that the corresponding value in `x` is less than or equal to $0$. Thus, the interval abstraction $\alpha(b1)$ and $\alpha(b2)$ can be improved to $[0,2]$ and $[0,1]$ respectively, leading to $\alpha(ret) = [0,2] \sqcup [0,1]= [0,2]$. 99 | 100 | ### Details 101 | 102 | Like **predicate splitting**, the abstract interpretation of element-wise `select` operation needs to "split" the numerical tensors used in `cond`, reevaluate the abstracted outputs of two branches, and finally compute the abstracted output of `select` operation by joining ($\sqcup$) abstracted outputs of two branches. 103 | 104 | `cond` has the form of `arg0 cmp arg1`, where `arg0` and `arg1` are numerical tensors, and `cmp` is the compare operation. We require that one of (or both of) `arg0` and `arg1` depends on only one variable in the linear affine relation without $relu$. The one satisfies the above requirement, say node `x`, will be split according to the comparison operation `cmp`. (If both of them satisfy the above requirement, we will choose the first one.) Without losing the generalizability, the `cond` can be written as `x cmp y`, where $\alpha(x)=[l_x,u_x]$ and $\alpha(y)=[l_y,u_y]$. We summarize the splits of `x` for different comparison operations `cmp`: 105 | 106 | | `cmp` | $\alpha(x)$ for `b1` | $\alpha(x)$ for `b2` | 107 | | ------------------------- | -------------------------------- | -------------------------------- | 108 | | `GreaterEqual`, `Greater` | $[\max(l_x,l_y), \max(u_x,l_y)]$ | $[\min(l_x,u_y), \min(u_x,u_y)]$ | 109 | | `LessEqual`, `Less` | $[\min(l_x,u_y), \min(u_x,u_y)]$ | $[\max(l_x,l_y), \max(u_x,l_y)]$ | 110 | | `NotEqual` | $[l_x,u_x]$ | $[\max(l_x,l_y), \min(u_x,u_y)]$ | 111 | | `Equal` | $[\max(l_x,l_y), \min(u_x,u_y)]$ | $[l_x,u_x]$ | 112 | 113 | ## Specify Ranges of Weights and Inputs 114 | 115 | We provide all the specified ranges of weights and inputs in `specified_ranges.py`, not only for the reproduction of the evaluation results but also for users to specify ranges of weights and inputs by taking examples of provided cases. We find that specifying ranges of weights and inputs can eliminate unnecessary false positives. Thus, we hope these examples can help reduce false positives and users' manually inspection time. 116 | 117 | Here is a short guideline for adding specified ranges in your setup: 118 | 119 | 1. Please read the description of `parse_format_text.py` and `specified_ranges.py` in the previous Section. 120 | 2. Add a data entry with the architecture name as the key and the mapping from variable names to their ranges as the value. 121 | 3. Make sure the types of ranges are matched. For `iteratorv2` and `oneshotiterator`, the types are lists of 2-elements lists. For `variablev2` and `placeholder`, the types are 2-elements lists. 122 | 123 | -------------------------------------------------------------------------------- /docs/troubleshoot.md: -------------------------------------------------------------------------------- 1 | # Troubleshoot 2 | 3 | ## Warnings 4 | 5 | * If you spot runtime warnings like `XXX not implemented`, it means that the computation graph contains operations like `XXX` that does not affect the data flow values, so it is safe that we do not implement its abstraction semantics. 6 | * If you spot runtime warnings like `fail to analysis YYY due to NotImplemented`, it means that the computation graph contains operations like `YYY` whose abstraction semantics is not implemented. DEBAR handles `YYY` in a sound manner, it treats the output of `YYY` as unbounded, i.e., in the range `[-inf,+inf]`. For better analysis results, we encourage other developers to contribute more implementations of abstraction semantics. Please see [Analysis](./analysis.md) for how to implement abstraction semantics. 7 | The unhandled operations will be prompt into console before the static analysis. 8 | 9 | ## Runtime Errors 10 | 11 | Please open an issue if you spot any runtime errors. 12 | 13 | -------------------------------------------------------------------------------- /dynamic_tool/TFNBDetector.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import math 3 | import TFNBUtils as utils 4 | import numpy as np 5 | import copy 6 | 7 | class TFNBDetector: 8 | def __init__(self, sess, x, y, x_train, y_train, x_test, y_test, train_op, loss, batch_size, max_epochs, **kwargs): 9 | """ 10 | :param sess: the TF sess 11 | :param x: the input tensor x, e.g. the images 12 | :param y: the input tensor y, e.g. the labels 13 | :param x_train, y_train, x_test, y_test: the training and testing data 14 | :param train_op: the training operator 15 | :param loss: the loss tensor 16 | :param batch_size: the batch size 17 | :param max_epochs: the max training epochs 18 | :param large_batch_size: a large batch size for inference/testing, not used for training. default: 1000 19 | :param clip_min, clip_max: the range of the input data 20 | """ 21 | self.sess = sess 22 | self.x = x 23 | self.y = y 24 | self.x_train = x_train 25 | self.y_train = y_train 26 | self.x_test = x_test 27 | self.y_test = y_test 28 | self.train_op = train_op 29 | self.batch_size = batch_size 30 | self.max_epochs = max_epochs 31 | self.loss = loss 32 | self.weight = None 33 | if "large_batch_size" in kwargs: 34 | self.large_batch_size = kwargs["large_batch_size"] 35 | else: 36 | self.large_batch_size = 1000 37 | if "clip_min" in kwargs: 38 | self.clip_min = kwargs["clip_min"] 39 | else: 40 | self.clip_min = min(np.min(x_train), np.min(x_test)) 41 | if "clip_max" in kwargs: 42 | self.clip_max = kwargs["clip_max"] 43 | else: 44 | self.clip_max = max(np.max(x_train), np.max(x_test)) 45 | if "train_feed_dict" in kwargs: 46 | self.train_feed_dict = kwargs["train_feed_dict"] 47 | else: 48 | self.train_feed_dict = {} 49 | if "test_feed_dict" in kwargs: 50 | self.test_feed_dict = kwargs["test_feed_dict"] 51 | else: 52 | self.test_feed_dict = {} 53 | 54 | def trigger(self, suspect_input, suspect, trigger_type, flag_NNweights, flag_inputs, eps = 0.3, fix_epoch = 1000, rand_pool_size = 20): 55 | """ 56 | :param suspect_input: the input of the suspect node 57 | :param suspect: the suspect node which may lead to NAN or INF 58 | :param trigger_type: trigger method type: "max", "max_difference" 59 | :param flag_NNweights: if set up, the method will iteratively find training batches to guide the triggering process 60 | :param flag_inputs: if set up, the method will modify some training inputs to guide the triggering process 61 | if neither the above two flags are set up, the trigger procedure is a normal training process 62 | :param eps: linf norm of the edit on the input, larger eps makes more edits on the input 63 | :param fix_epoch: If set up flag_NNweights, before the fix epoch is reached, the process will find the best triggering input. Larger fix_epoch makes the process runs slower but reduces the number of total epochs. 64 | :param rand_pool_size: larger rand_pool_size will make the process more effective but reduce the randomness of the training batch 65 | :return: True if NAN of INF is found, False otherwise 66 | """ 67 | self.sess.run(tf.global_variables_initializer()) 68 | if trigger_type == "max": 69 | pass 70 | elif trigger_type == "max_difference": 71 | suspect_input = tf.reduce_max(suspect_input, axis=-1) - tf.reduce_min(suspect_input, axis=-1) 72 | elif trigger_type == "min_abs": 73 | suspect_input = -tf.reduce_min(tf.abs(suspect_input), axis=-1) 74 | else: 75 | raise NotImplemented("Error unknow trigger type: %s" % trigger_type) 76 | 77 | if flag_inputs: 78 | grad_input = tf.gradients(suspect_input, self.x) 79 | if flag_NNweights: 80 | weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) 81 | grad_loss = tf.gradients(self.loss, weights) 82 | grad_weights = tf.gradients(suspect_input, weights) 83 | self.weights = [] 84 | self.grad_loss = [] 85 | self.grad_weights = [] 86 | for (grad_l, grad_w, weight) in zip(grad_loss, grad_weights, weights): 87 | if grad_l is not None and grad_w is not None: 88 | self.weights.append(weight) 89 | self.grad_loss.append(grad_l) 90 | self.grad_weights.append(grad_w) 91 | weights = self.weights 92 | grad_weights = self.grad_weights 93 | 94 | target = None 95 | input_id = [np.random.randint(0, len(self.x_train))] 96 | while target is not None and np.argmax(self.y_train[input_id[0]]) != target: 97 | input_id = [np.random.randint(0, len(self.x_train))] 98 | whole = range(len(self.x_train)) 99 | x_new_train = copy.copy(self.x_train) 100 | for i in range(self.max_epochs): 101 | # generate the trigger input 102 | if flag_inputs: 103 | ret = self.sess.run(grad_input, feed_dict={**self.test_feed_dict, self.x:x_new_train[input_id], self.y:self.y_train[input_id]})[0] 104 | # ord = 2 105 | # delta = np.clip(x_new_train[input_id] + ret , self.clip_min, self.clip_max) - self.x_train[input_id] 106 | # scale_eps = (self.clip_max - self.clip_min) * eps 107 | # scale = np.sqrt(np.sum(delta * delta)) / scale_eps 108 | # x_new_train[input_id]=np.clip(self.x_train[input_id] + delta / scale, self.clip_min, self.clip_max) 109 | 110 | # ord = np.inf 111 | # delta = np.clip(x_new_train[input_id] + ret, self.clip_min, self.clip_max) - self.x_train[input_id] + 1e-8 112 | # scale_eps = (self.clip_max - self.clip_min) * eps 113 | # scale = np.max(np.abs(delta)) / scale_eps 114 | # x_new_train[input_id]=np.clip(self.x_train[input_id] + delta / scale, self.clip_min, self.clip_max) 115 | 116 | # ord = np.inf iterative 117 | scale = (self.clip_max - self.clip_min) * eps / np.max(np.abs(ret) + 1e-8) 118 | x_new_train[input_id]=np.clip(x_new_train[input_id] + ret * scale, self.clip_min, self.clip_max) 119 | trigger_input = x_new_train[input_id], self.y_train[input_id] 120 | if flag_NNweights: 121 | whole = utils.rank_delta(x_new_train, self.y_train, self.large_batch_size, suspect_input, self.sess, [self.x, self.y], whole, self.test_feed_dict) 122 | if i < fix_epoch: 123 | whole = whole[:-(len(x_new_train) // (fix_epoch + 1))] 124 | input_id = [whole[0]] 125 | trigger_input = x_new_train[input_id], self.y_train[input_id] 126 | 127 | # generate the training batch 128 | if flag_NNweights and i >= fix_epoch: 129 | eval_grads = self.sess.run(grad_weights, feed_dict={**self.test_feed_dict, self.x:x_new_train[input_id], self.y:self.y_train[input_id]}) 130 | random_idx = utils.random_choose(len(self.x_train), self.batch_size * rand_pool_size) 131 | x_batch, y_batch, _, _ = utils.choose_max_batch(self.x_train[random_idx], self.y_train[random_idx], self.batch_size, self.grad_loss, eval_grads, self.sess, [self.x, self.y], self.test_feed_dict) 132 | else: 133 | random_idx = utils.random_choose(len(self.x_train), self.batch_size) 134 | x_batch, y_batch = self.x_train[random_idx], self.y_train[random_idx] 135 | 136 | # if none of the flag is set, we use a random batch as the trigger inputs 137 | if flag_inputs or flag_NNweights: 138 | suspect_val = self.sess.run(suspect, feed_dict={**self.test_feed_dict, self.x:trigger_input[0], self.y:trigger_input[1]}) 139 | else: 140 | suspect_val = self.sess.run(suspect, feed_dict={**self.test_feed_dict, self.x:x_batch, self.y:y_batch}) 141 | 142 | if i % 2000 == 0 and (flag_inputs or flag_NNweights): 143 | print(np.max(self.sess.run(suspect_input, feed_dict={**self.test_feed_dict, self.x:trigger_input[0], self.y:trigger_input[1]}))) 144 | 145 | if i % 2000 == 0 and not (flag_inputs or flag_NNweights): 146 | print(np.max(self.sess.run(suspect_input, feed_dict={**self.test_feed_dict, self.x:x_batch, self.y:y_batch}))) 147 | 148 | if len(np.where(np.isnan(suspect_val))[0]) > 0: 149 | if flag_inputs: 150 | utils.to_img(self.x_train[input_id[0]], "./norm.png") 151 | utils.to_img(trigger_input[0][0], "./nan.png") 152 | print("NAN found in %d-th epoch" % i) 153 | return True 154 | elif len(np.where(np.isinf(suspect_val))[0]) > 0: 155 | if flag_inputs: 156 | utils.to_img(self.x_train[input_id[0]], "./norm.png") 157 | utils.to_img(trigger_input[0][0], "./nan.png") 158 | print("INF found in %d-th epoch" % i) 159 | return True 160 | 161 | self.sess.run(self.train_op, feed_dict={**self.train_feed_dict, self.x:x_batch, self.y:y_batch}) 162 | 163 | return False 164 | -------------------------------------------------------------------------------- /dynamic_tool/TFNBUtils.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import math 4 | 5 | def choose_max_batch(x_train, y_train, batch_size, grads, evaled, sess, feed_dict, additional): 6 | ret_x = None 7 | ret_y = None 8 | min_value = 1e20 9 | ix = None 10 | for i in range(0, len(x_train), batch_size): 11 | eval_grads = sess.run(grads, feed_dict={**additional, feed_dict[0]:x_train[i:i+batch_size], feed_dict[1]:y_train[i:i+batch_size]}) 12 | value = 0 13 | for k in range(len(eval_grads)): 14 | value += np.sum(np.array(eval_grads[k]) * evaled[k]) 15 | if value < min_value: 16 | ret_x = x_train[i:i+batch_size] 17 | ret_y = y_train[i:i+batch_size] 18 | min_value = value 19 | ix = i 20 | return ret_x, ret_y, min_value, ix 21 | 22 | def rank_delta(x_train, y_train, batch_size, delta, sess, feed_dict, whole=None, additional={}): 23 | if whole is None: 24 | whole = range(len(x_train)) 25 | good_points = [] 26 | for i in range(0, len(whole), batch_size): 27 | eval_delta = sess.run(delta, feed_dict={**additional, feed_dict[0]:x_train[whole[i:min(i+batch_size, len(x_train))]], feed_dict[1]:y_train[whole[i:min(i+batch_size, len(x_train))]]}) 28 | for j in range(len(eval_delta)): 29 | good_points.append((eval_delta[j], whole[j + i])) 30 | good_points.sort(key=lambda x:-x[0]) 31 | return np.array(list(map(lambda x:x[1], good_points))) 32 | 33 | def random_choose(size, batch_size): 34 | return np.random.choice(np.arange(size), batch_size) 35 | 36 | 37 | def iterative_find(x_new_train, y_new_train, x_train, y_train, chosen_idx, grad_logits_delta, grad_loss, feed_dict, batch_size, sess, weights, grad_image): 38 | # for i in range(len(x_train)): 39 | modify_everytime = 10 40 | for i in range(10): 41 | best = chosen_idx[:1] 42 | 43 | eval_grads = [np.zeros(weight.shape) for weight in weights] 44 | eval_grads_tmp = sess.run(grad_logits_delta, feed_dict={feed_dict[0]:x_new_train[best], feed_dict[1]:y_new_train[best]}) 45 | for j in range(len(eval_grads)): 46 | eval_grads[j] += eval_grads_tmp[j] 47 | 48 | # for times in range(100): 49 | random_idx = random_choose(len(x_train), batch_size * modify_everytime) 50 | x_batch, y_batch, value, ix = choose_max_batch(x_train[random_idx], y_train[random_idx], batch_size, grad_loss, eval_grads, sess, feed_dict) 51 | # if value < 0: 52 | # break 53 | 54 | ret = sess.run(grad_image, feed_dict={feed_dict[0]:x_new_train[best], feed_dict[1]:y_new_train[best]})[0] 55 | x_new_train[best]=np.clip(x_new_train[best] + ret * 1e-2, -1, 1) 56 | 57 | if value < 0 or i == 9: 58 | return x_batch, y_batch 59 | 60 | def to_img(x, des): 61 | if len(x.shape) == 1: 62 | n = int(math.sqrt(x.shape[0])) 63 | x =np.reshape(x, (n, x.shape[0]//n)) 64 | v_min = np.min(x) 65 | v_max = np.max(x) 66 | x = np.uint8(np.squeeze((x-v_min)/(v_max-v_min)) * 255) 67 | from PIL import Image 68 | im = Image.fromarray(np.squeeze(x)) 69 | im.save(des) 70 | -------------------------------------------------------------------------------- /dynamic_tool/Yuhao Zhang-undergraduate dissertation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ForeverZyh/DEBAR/3a2880697fa67b6b1beb127f1fc773b690f1c67c/dynamic_tool/Yuhao Zhang-undergraduate dissertation.pdf -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | # call the analysis_main.py 2 | import sys 3 | import os 4 | import time 5 | 6 | from parse.specified_ranges import SpecifiedRanges 7 | 8 | path = "/tmp" 9 | try: 10 | assert len(sys.argv) == 2 11 | path = sys.argv[1] 12 | 13 | except: 14 | print( 15 | "Please run 'python main.py PATH_TO_DATASETS'.\nAborted...") 16 | exit(1) 17 | 18 | unbounded_weight = False 19 | unbounded_input = False 20 | interpreter_path = sys.executable 21 | print("Running at: ", interpreter_path) 22 | result_filename = "results.txt" 23 | 24 | open(result_filename, 'w').close() 25 | 26 | times = {} 27 | for model in SpecifiedRanges.models: 28 | t0 = time.time() 29 | print("Running %s" % model) 30 | if not unbounded_weight and not unbounded_input: 31 | os.system( 32 | "(%s ./analysis_main.py %s/%s.pbtxt) >> %s 2>&1" % (interpreter_path, path, model, result_filename)) 33 | elif unbounded_weight: 34 | os.system("(%s ./analysis_main.py %s/%s.pbtxt unbounded_weight) >> %s 2>&1" % ( 35 | interpreter_path, path, model, result_filename)) 36 | elif unbounded_input: 37 | os.system("(%s ./analysis_main.py %s/%s.pbtxt unbounded_input) >> %s 2>&1" % ( 38 | interpreter_path, path, model, result_filename)) 39 | times[model] = time.time() - t0 40 | 41 | lines = open(result_filename).readlines() 42 | f = open(result_filename, 'a') 43 | info = {} 44 | for line in lines: 45 | if line.find("warnings") != -1 and len(line) > 10: 46 | splits = line.split() 47 | model_name = splits[0] 48 | info[model_name] = line.strip() 49 | 50 | for model in SpecifiedRanges.models: 51 | if model in info: 52 | print(info[model] + "\t in time: %.2f" % times[model]) 53 | f.write(info[model] + "\t in time: %.2f" % times[model] + "\n") 54 | else: 55 | print("Runtime error when running %s." % model) 56 | f.write("Runtime error when running %s." % model + "\n") 57 | -------------------------------------------------------------------------------- /parse/parse_format_text.py: -------------------------------------------------------------------------------- 1 | '''https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto''' 2 | 3 | from tensorflow.python.framework import tensor_util 4 | import ast 5 | import numpy as np 6 | import z3 7 | 8 | from solver import Range 9 | from parse.specified_ranges import SpecifiedRanges 10 | from utils import * 11 | 12 | placeholder_map = {} 13 | unbounded_weight = False 14 | unbounded_input = False 15 | 16 | 17 | # parses the constant values from the node attribute 18 | def const(node): 19 | attrs = node.attr 20 | tensor = attrs["value"].tensor 21 | value = tensor_util.MakeNdarray(tensor) 22 | return value 23 | 24 | 25 | # parses the inputs obtained by the iteratorv2 operation, and returns a list of Range objects 26 | def iteratorv2(node): 27 | attrs = node.attr 28 | return oneshotiterator(node) 29 | 30 | 31 | # parses the weights obtained by the variablev2 operation, and returns a Range object 32 | def variablev2(node): 33 | if unbounded_weight: 34 | return Range(left=-OVERFLOW_LIMIT, right=OVERFLOW_LIMIT) 35 | attrs = node.attr 36 | dtype = attrs["dtype"].type 37 | shape = attrs["shape"].shape 38 | if node.op.lower() == "conv2dbackpropinput" or node.name.find("BatchNorm") != -1: 39 | return Range(left=-1, right=1) 40 | elif node.name.find("/step") != -1: 41 | return Range(left=1, right=OVERFLOW_LIMIT) 42 | if node.name in SpecifiedRanges.ranges_looking_up: 43 | return placeholder(node, True) # if the weight=True, it will not return dumy() even if unbounded_input = True 44 | elif dtype in [1, 2, 19] and len(shape_from_proto(shape)) > 0: 45 | return Range(left=-1, right=1) 46 | else: 47 | return placeholder(node, True) # if the weight=True, it will not return dumy() even if unbounded_input = True 48 | 49 | 50 | # parses the inputs obtained by the oneshotiterator operation, and returns a list of Range objects 51 | def oneshotiterator(node): 52 | if node.name in placeholder_map: 53 | return placeholder_map[node.name] 54 | attrs = node.attr 55 | shapes = attrs["shapes"].list.shape 56 | output_shapes = attrs["output_shapes"].list.shape 57 | if len(output_shapes) > len(shapes): 58 | shapes = output_shapes 59 | if unbounded_input: 60 | return [Range(left=-OVERFLOW_LIMIT, right=OVERFLOW_LIMIT) for _ in range(len(shapes))] 61 | value = [] 62 | if node.name in SpecifiedRanges.ranges_looking_up: 63 | input_list = SpecifiedRanges.ranges_looking_up[node.name] 64 | else: 65 | print(node) 66 | while True: 67 | x = input("Please specify the range of inputs\n" 68 | "e.g. [[-1, 1], [0, None]] means the first range is [-1, 1] and the second range is [0 ,inf):\n") 69 | try: 70 | input_list = ast.literal_eval(x) 71 | except: 72 | input_list = None 73 | 74 | if not isinstance(input_list, list): 75 | print("Input string is not a list!") 76 | elif np.array(input_list).shape != (len(shapes), 2): 77 | print("Input list's shape not match with %s (received %s)!" % ( 78 | str((len(shapes), 2)), str(np.array(input_list)))) 79 | else: 80 | break 81 | 82 | for (i, rng) in enumerate(input_list): 83 | if None in rng: 84 | value.append(Range(left=rng[0] if rng[0] is not None else -OVERFLOW_LIMIT, 85 | right=rng[1] if rng[1] is not None else OVERFLOW_LIMIT)) 86 | else: 87 | value.append(Range(left=rng[0], right=rng[1])) 88 | 89 | if len(value) == 1: 90 | value = value[0] 91 | placeholder_map[node.name] = value 92 | return placeholder_map[node.name] 93 | 94 | 95 | # parses the inputs obtained by the placeholder operation, and returns a Range object 96 | def placeholder(node, weight=False): 97 | if unbounded_input and not weight: 98 | return Range(left=-OVERFLOW_LIMIT, right=OVERFLOW_LIMIT) 99 | if node.name in placeholder_map: 100 | return placeholder_map[node.name] 101 | attrs = node.attr 102 | dtype = attrs["dtype"].type 103 | 104 | if node.name in SpecifiedRanges.ranges_looking_up: 105 | rng = SpecifiedRanges.ranges_looking_up[node.name] 106 | else: 107 | print(node) 108 | while True: 109 | x = input("Please specify the range of the placeholder \n" 110 | "e.g. [-1, 1] means the range is [-1, 1] \n" 111 | "e,g, [0, None] means the range is [0 ,inf):\n") 112 | try: 113 | rng = ast.literal_eval(x) 114 | except: 115 | rng = None 116 | 117 | if isinstance(rng, list) and len(rng) == 2: 118 | break 119 | 120 | if None in rng: 121 | placeholder_map[node.name] = Range(left=rng[0] if rng[0] is not None else -OVERFLOW_LIMIT, 122 | right=rng[1] if rng[1] is not None else OVERFLOW_LIMIT) 123 | else: 124 | placeholder_map[node.name] = Range(left=rng[0], right=rng[1]) 125 | 126 | return placeholder_map[node.name] 127 | -------------------------------------------------------------------------------- /parse/parse_graph.py: -------------------------------------------------------------------------------- 1 | from google.protobuf import text_format 2 | import tensorflow as tf 3 | from analysis.inference import InferValue, InferArray, identity, dumy 4 | from analysis.abstract_interpretation import AbstractInterpretation 5 | import queue 6 | from graphviz import Digraph 7 | import warnings 8 | import z3 9 | from solver import meet, meet_relation_variable, magic 10 | from solver import Range, Array, Solver 11 | from utils import * 12 | import numpy as np 13 | import copy 14 | 15 | turn_on_array = True 16 | 17 | 18 | # implements the disjoint-set data structure https://en.wikipedia.org/wiki/Disjoint-set_data_structure 19 | # for identifying the largest connected component in the parsed computation graph. 20 | class UnionSet: 21 | def __init__(self, eles): 22 | self.f = {} 23 | self.rank = {} 24 | for ele in eles: 25 | self.f[ele] = ele 26 | self.rank[ele] = 1 27 | 28 | def find(self, x): 29 | if self.f[x] == x: 30 | return x 31 | self.f[x] = self.find(self.f[x]) 32 | self.rank[x] = self.rank[self.f[x]] 33 | return self.f[x] 34 | 35 | def union(self, x, y): 36 | """merge y to x""" 37 | u = self.find(x) 38 | v = self.find(y) 39 | if u != v: 40 | if self.rank[u] < self.rank[v]: 41 | u, v = v, u 42 | self.f[v] = u 43 | self.rank[u] += self.rank[v] 44 | 45 | 46 | # implements the parsing process of the Protocol Buffer file to the computation graph and the process of static 47 | # dataflow analysis, as well as other functionalities that are related to the computation graph. 48 | class Graph: 49 | def __init__(self, filename, verbose_file=None): 50 | with open(filename) as f: 51 | txt = f.read() 52 | self.graph_def = text_format.Parse(txt, tf.GraphDef()) 53 | tf.import_graph_def(self.graph_def, name="") 54 | self.tf_graph = tf.get_default_graph() 55 | # storing the reversed edges of the computation graph 56 | self.graph_backward = [{}, {}] # [0] for non_control; [1] for control 57 | # storing the edges of the computation graph 58 | self.graph_forward = [{}, {}] # [0] for non_control; [1] for control 59 | # is a map mapping from the name of an operation (string) to the node attribute in protocol buffer format 60 | self.node_by_name = {} 61 | self.f = UnionSet([node.name for node in self.graph_def.node]) 62 | # is a map mapping from the name of an operation (string) to an AbstractInterpretation object (or a list of 63 | # AbstractInterpretation objects) denoting the output of the node computed by dataflow analysis. 64 | self.node_output = {} 65 | # is a map mapping from the name of an operation to a list. The list indicates which value is passed to the 66 | # next node if the output of the current node is a list of AbstractInterpretation objects. 67 | self.edge_index = {} 68 | # is a set storing which nodes have been visited by data flow analysis and it is used for incremental 69 | # dataflow analysis. 70 | self.node_visited = set() 71 | self.unique_clique = [] 72 | self.main_clique = None 73 | # is a map mapping from tensor name to the operation (node) name 74 | self.tensor_to_op = {} 75 | # is a map mapping from an operation name to its topological order, instructing the order of dataflow analysis 76 | self.nodes_in_main_clique_topology = {} 77 | self.file = None if verbose_file is None else open(verbose_file, "w") 78 | self.build() 79 | 80 | def write(self, x): 81 | if self.file is None: 82 | print(x) 83 | else: 84 | self.file.write(str(x) + "\n") 85 | 86 | # parses the Protocol Buffer format, builds the computation graph, and the topological order of the nodes. 87 | def build(self): 88 | for node in self.graph_def.node: 89 | self.node_by_name[node.name] = node 90 | self.node_output[node.name] = AbstractInterpretation() 91 | for op in self.tf_graph.get_operations(): 92 | for tensor in op.values(): 93 | self.tensor_to_op[tensor.name] = op.name 94 | 95 | # parse the protocol buffer format and builds the computation graph. 96 | for node in self.graph_def.node: 97 | self.graph_backward[0][node.name] = [] 98 | self.graph_backward[1][node.name] = [] 99 | self.edge_index[node.name] = [] 100 | node_values = self.tf_graph.get_operation_by_name(node.name).values() 101 | 102 | if node_values is None or len(node_values) == 0: 103 | self.node_output[node.name] = AbstractInterpretation() 104 | elif len(node_values) > 1: 105 | self.node_output[node.name] = AbstractInterpretation( 106 | size=[node_value.shape for node_value in node_values], 107 | dtype=[node_value.dtype for node_value in node_values], 108 | array=[Array(node.name + "|" + str(i), node_value.shape) for 109 | (i, node_value) in enumerate(node_values)]) 110 | else: 111 | self.node_output[node.name] = AbstractInterpretation( 112 | size=node_values[0].shape, dtype=node_values[0].dtype, 113 | array=Array(node.name, node_values[0].shape)) 114 | for in_node_raw in node.input: 115 | is_control = False 116 | if in_node_raw[0] == '^': 117 | in_node_raw = in_node_raw[1:] 118 | is_control = True 119 | 120 | if in_node_raw in self.tensor_to_op: # if the input is defined by the tensor's name 121 | in_node = self.tensor_to_op[in_node_raw] 122 | 123 | in_tensor_names = [tensor.name for tensor in self.tf_graph.get_operation_by_name( 124 | self.tensor_to_op[in_node_raw]).values()] 125 | if not is_control: 126 | self.edge_index[node.name].append(None if len( 127 | in_tensor_names) == 1 else in_tensor_names.index(in_node_raw)) 128 | else: # if the input is defined by the operation's name 129 | in_node = in_node_raw 130 | 131 | in_tensor_names = [tensor.name for tensor in self.tf_graph.get_operation_by_name( 132 | in_node_raw).values()] 133 | if not is_control: 134 | self.edge_index[node.name].append(None if len(in_tensor_names) == 1 else 0) 135 | 136 | if in_node not in self.graph_forward[0]: 137 | self.graph_forward[0][in_node] = [] 138 | self.graph_forward[1][in_node] = [] 139 | self.graph_forward[is_control][in_node].append(node.name) 140 | self.graph_backward[is_control][node.name].append(in_node) 141 | self.f.union(in_node, node.name) 142 | 143 | max_rank = 0 144 | for node in self.f.f: 145 | if self.f.find(node) == node: 146 | self.unique_clique.append(node) 147 | max_rank = max(max_rank, self.f.rank[node]) 148 | 149 | for node_name in self.unique_clique: 150 | if max_rank == self.f.rank[node_name]: 151 | self.main_clique = node_name 152 | 153 | node_inds = {} 154 | q = queue.Queue() 155 | nodes_in_main_clique = set() 156 | cnt = 0 157 | for node_name in self.f.f: 158 | node_inds[node_name] = 0 if node_name not in self.graph_backward[0] else len( 159 | self.graph_backward[0][node_name]) # is sufficient to only query in self.graph_backward[0] 160 | if self.f.find(node_name) == self.main_clique: 161 | nodes_in_main_clique.add(node_name) 162 | 163 | for node_name in nodes_in_main_clique: 164 | if node_inds[node_name] == 0: 165 | q.put(node_name) 166 | 167 | # build nodes_in_main_clique_topology instructing the order of dataflow analysis 168 | while True: 169 | while not q.empty(): 170 | son = q.get() 171 | nodes_in_main_clique.remove(son) 172 | self.nodes_in_main_clique_topology[son] = cnt 173 | cnt += 1 174 | if son in self.graph_forward[0]: 175 | for next_node_name in self.graph_forward[0][son]: 176 | node_inds[next_node_name] -= 1 177 | if node_inds[next_node_name] == 0 and next_node_name in nodes_in_main_clique: 178 | q.put(next_node_name) 179 | 180 | if len(nodes_in_main_clique) == 0: 181 | break 182 | 183 | # identify loops 184 | min_ind = None 185 | for node_name in nodes_in_main_clique: 186 | if self.node_by_name[node_name].op == "Merge": 187 | can_add = True 188 | for in_node_name in self.graph_backward[0][node_name]: 189 | if in_node_name in nodes_in_main_clique and self.node_by_name[ 190 | in_node_name].op != "NextIteration": 191 | # if a Merge is not dominated by a NextIteration, then we cannot add it into the queue 192 | can_add = False 193 | break 194 | 195 | if can_add and (min_ind is None or node_inds[node_name] < node_inds[min_ind]): 196 | min_ind = node_name 197 | 198 | assert min_ind is not None 199 | q.put(min_ind) 200 | 201 | # returns a list of nodes in the backward slice starting at node 202 | def backward_slice(self, node, visited, non_control_only=True): # return a list of nodes 203 | visited.add(node) 204 | ret = [node] 205 | for in_node in self.graph_backward[0][node]: 206 | if in_node not in visited: 207 | ret.extend(self.backward_slice(in_node, visited)) 208 | if not non_control_only: 209 | for in_node in self.graph_backward[1][node]: 210 | if in_node not in visited: 211 | ret.extend(self.backward_slice(in_node, visited)) 212 | 213 | return ret 214 | 215 | def draw(self, clique, filename): 216 | dot = Digraph() 217 | clique = set(clique) 218 | for x in clique: 219 | dot.node(x, self.node_by_name[x].op) 220 | 221 | for node_name in clique: 222 | if node_name in self.graph_forward[0]: 223 | for is_contorl in range(2): 224 | for next_node_name in self.graph_forward[is_contorl][node_name]: 225 | if next_node_name in clique: 226 | dot.edge(node_name, next_node_name, color="blue" if is_contorl == 0 else "red") 227 | 228 | dot.render("./%s.gv" % filename, view=False) 229 | 230 | # calculates the abstracted output of node son with its attribute u in protocol buffer format according to the 231 | # abstractions of its inputs while the abstracted outputs of some nodes have been overridden in override_dict. 232 | # override_dict is a map mapping the names to their overridden abstractions. It will only be used in predicate 233 | # splitting and handling element-wise Select operation. 234 | def summary_node(self, son, u, override_dict={}): 235 | self.write(son) 236 | parents_aps = [] 237 | all_none = True 238 | for (i, in_node_name) in enumerate(self.graph_backward[0][son]): # only care about non_control edges 239 | if in_node_name not in self.node_visited: 240 | # there is a loop, and the node is "Merge" 241 | assert self.node_by_name[in_node_name].op == "NextIteration" 242 | self.node_visited.add(in_node_name) 243 | self.node_output[in_node_name].value = dumy() 244 | 245 | parents_aps.append(self.node_output[in_node_name].index_of(self.edge_index[son][i])) 246 | all_none &= parents_aps[-1].has_none() 247 | 248 | temp = None 249 | temp_array = None 250 | if all_none and len(parents_aps) > 0: 251 | warnings.warn("fail to analysis %s due to None" % son, RuntimeWarning) 252 | else: 253 | try: 254 | temp = getattr(InferValue, u.op.lower())(parents_aps, u) 255 | if temp is not None and isinstance(temp, tuple): 256 | raise AssertionError 257 | except AttributeError: 258 | if u.op.lower() in ["assert"]: 259 | pass 260 | else: 261 | temp = None 262 | warnings.warn("fail to analysis %s due to NotImplemented" % son, RuntimeWarning) 263 | except AssertionError: 264 | raise AssertionError 265 | 266 | # TODO refactor the handling of Select operation to analysis/inference.py 267 | if u.op == "Select": # special treatment for Select 268 | compare_node_name = self.graph_backward[0][son][0] 269 | compare_node = self.node_by_name[compare_node_name] 270 | branch_node_name = self.graph_backward[0][son][1:] 271 | branch_value = [self.node_output[branch_node_name[i - 1]].index_of(self.edge_index[son][i]).value for i 272 | in range(1, 3)] 273 | branch_array = [self.node_output[branch_node_name[i - 1]].index_of(self.edge_index[son][i]).array for i 274 | in range(1, 3)] 275 | if compare_node.op in ["GreaterEqual", "Greater", "LessEqual", "Less", "Equal", "NotEqual"]: 276 | args = self.graph_backward[0][compare_node_name] # args --> compare_node_name --> son 277 | range_args = [identity([self.node_output[args[i]].index_of(self.edge_index[compare_node_name][i])]) 278 | for i in range(2)] 279 | 280 | # check whether the cond tensor can be determined to be all true or all false 281 | def can_determine(): 282 | if compare_node.op == "GreaterEqual": 283 | if range_args[0].left >= range_args[1].right: 284 | return branch_value[0] 285 | elif range_args[0].right < range_args[1].left: 286 | return branch_value[1] 287 | elif compare_node.op == "Greater": 288 | if range_args[0].left > range_args[1].right: 289 | return branch_value[0] 290 | elif range_args[0].right <= range_args[1].left: 291 | return branch_value[1] 292 | elif compare_node.op == "LessEqual": 293 | if range_args[0].right <= range_args[1].left: 294 | return branch_value[0] 295 | elif range_args[0].left > range_args[1].right: 296 | return branch_value[1] 297 | elif compare_node.op == "Less": 298 | if range_args[0].right < range_args[1].left: 299 | return branch_value[0] 300 | elif range_args[0].left >= range_args[1].right: 301 | return branch_value[1] 302 | elif compare_node.op == "Equal": 303 | if range_args[0].single() and range_args[1].single() and range_args[1].left == range_args[ 304 | 0].left: 305 | return branch_value[0] 306 | elif range_args[0].left > range_args[1].right or range_args[0].right < range_args[1].left: 307 | return branch_value[1] 308 | elif compare_node.op == "NotEqual": 309 | if range_args[0].single() and range_args[1].single() and range_args[1].left == range_args[ 310 | 0].left: 311 | return branch_value[1] 312 | elif range_args[0].left > range_args[1].right or range_args[0].right < range_args[1].left: 313 | return branch_value[0] 314 | else: 315 | raise NotImplementedError 316 | return None 317 | 318 | temp_ret = can_determine() 319 | if temp_ret is not None: 320 | temp = temp_ret 321 | else: # cannot determine the cond tensor 322 | single_value_array_id = None 323 | array = None 324 | # the cond has the form of: arg0 cmp arg1 325 | # we require one of two args to be single_value_array. If both are single_value_arrays, we 326 | # will choose the first one 327 | # single_value_array: partitions depend on only one variable in linear affine relation without 328 | # relu. 329 | for i in range(2): 330 | array = self.node_output[args[i]].index_of(self.edge_index[compare_node_name][i]).array 331 | single_value_array = True 332 | for key in array.block_to_symbol: 333 | group = array.block_to_symbol[key] 334 | if len(group.value) > 1: 335 | single_value_array = False 336 | break 337 | key = list(group.value.keys())[0] 338 | if key[:len(magic)] == magic: # we don't consider relu 339 | single_value_array = False 340 | break 341 | if single_value_array: 342 | single_value_array_id = i 343 | break 344 | 345 | if single_value_array_id is not None: 346 | # compute the abstracted output of "GreaterEqual", "Greater", "LessEqual", "Less" operations 347 | def compute(op, c): 348 | values = [] 349 | # enumerate the branch id 350 | for branch_id_select in range(2): 351 | override_dict = {} 352 | for key in array.block_to_symbol: 353 | group = array.block_to_symbol[key] 354 | if len(group.value) == 1: 355 | for (name, position) in group.value: 356 | factor = group.value[(name, position)] 357 | if factor == 0: 358 | continue 359 | value = self.get_value(name) 360 | rhs = c * (1 / factor) 361 | if factor < 0: 362 | rhs = Range(left=rhs.right, right=rhs.left) 363 | if (factor > 0) ^ (op in ["GreaterEqual", "Greater"]) ^ ( 364 | branch_id_select == 0): 365 | # value >= rhs 366 | override_dict[(name, position)] = Range( 367 | left=max(value.left, rhs.left), 368 | right=max(value.right, rhs.left)) 369 | else: 370 | # value <= rhs 371 | override_dict[(name, position)] = Range( 372 | left=min(value.left, rhs.right), 373 | right=min(value.right, rhs.right)) 374 | 375 | values.append(self.get_left_right(branch_array[branch_id_select].block_to_symbol, 376 | branch_node_name[branch_id_select], override_dict)) 377 | if values[-1] is None: 378 | return None 379 | return Range(left=min(values[0].left, values[1].left), 380 | right=max(values[0].right, values[1].right)) 381 | 382 | # compute the abstracted output of "Equal", "NotEqual" 383 | def compute_equal(op, c): 384 | values = [] 385 | # enumerate the branch id 386 | for branch_id_select in range(2): 387 | override_dict = {} 388 | for key in array.block_to_symbol: 389 | group = array.block_to_symbol[key] 390 | if len(group.value) == 1: 391 | for (name, position) in group.value: 392 | factor = group.value[(name, position)] 393 | if factor == 0: 394 | continue 395 | value = self.get_value(name) 396 | rhs = c * (1 / factor) 397 | if factor < 0: 398 | rhs = Range(left=rhs.right, right=rhs.left) 399 | if (op == "NotEqual") ^ (branch_id_select == 0): 400 | # value == rhs 401 | override_dict[(name, position)] = Range( 402 | left=max(value.left, rhs.left), 403 | right=min(value.right, rhs.right)) 404 | else: 405 | # value != rhs 406 | pass 407 | 408 | values.append(self.get_left_right(branch_array[branch_id_select].block_to_symbol, 409 | branch_node_name[branch_id_select], override_dict)) 410 | if values[-1] is None: 411 | return None 412 | return Range(left=min(values[0].left, values[1].left), 413 | right=max(values[0].right, values[1].right)) 414 | 415 | if compare_node.op in ["GreaterEqual", "Greater", "LessEqual", "Less"]: 416 | if single_value_array_id == 1: 417 | temp_ret = compute( 418 | "Less" if compare_node.op in ["GreaterEqual", "Greater"] else "Greater", 419 | range_args[0]) 420 | else: 421 | temp_ret = compute(compare_node.op, range_args[1]) 422 | else: 423 | if single_value_array_id == 1: 424 | temp_ret = compute_equal(compare_node.op, range_args[0]) 425 | else: 426 | temp_ret = compute_equal(compare_node.op, range_args[1]) 427 | 428 | if temp_ret is not None: 429 | temp = temp_ret 430 | 431 | if turn_on_array: 432 | try: 433 | for parents_ap in parents_aps: 434 | assert parents_ap.array.index_slices is not None 435 | temp_array = getattr(InferArray, u.op.lower())(parents_aps, u) 436 | flag = True 437 | if isinstance(self.node_output[son].dtype, list): 438 | for x in self.node_output[son].dtype: 439 | if int(x) == 10: 440 | flag = False 441 | break 442 | else: 443 | flag = int(self.node_output[son].dtype) != 10 444 | 445 | if not flag: 446 | temp_array = None 447 | except AttributeError: 448 | pass 449 | except AssertionError: 450 | pass 451 | 452 | self.node_output[son].value = temp 453 | 454 | if temp_array is not None and isinstance(temp, Range): 455 | self.node_output[son].array = temp_array 456 | if isinstance(temp_array, list): 457 | temp = [] 458 | for (i, tmp_array) in enumerate(temp_array): 459 | if temp_array[i].index_slices is None: 460 | temp.append(self.node_output[son].value[i]) 461 | continue 462 | value = self.get_left_right(tmp_array.block_to_symbol, son, override_dict) 463 | if value is None: 464 | temp.append(self.node_output[son].value[i]) 465 | else: 466 | temp.append(value) 467 | elif temp_array.index_slices is not None: 468 | value = self.get_left_right(temp_array.block_to_symbol, son, override_dict) 469 | if value is not None: 470 | temp = value 471 | 472 | self.node_output[son].value = temp 473 | 474 | self.node_output[son].constraints = None 475 | self.write(self.node_output[son]) 476 | 477 | # is the body of dataflow analysis. It computes the abstracted output of node_interested, and returns the ranges 478 | # for predicate splitting. appended is the node of the unsafe operation. 479 | def forward_analysis(self, node_interested, appended=None): 480 | nodes_interested = self.backward_slice(node_interested.name, set(), True) # only care about non_control edges 481 | # we do not consider operations related to gradient descent. 482 | for node in nodes_interested: 483 | if "gradient" in node.lower() and "stopgradient" not in node.lower(): 484 | self.write("----------Gradients are not interested----------") 485 | return None 486 | 487 | nodes_interested.sort(key=lambda x: self.nodes_in_main_clique_topology[x]) 488 | if appended is not None: 489 | if "gradient" in appended.name.lower() and "stopgradient" not in appended.name.lower(): 490 | self.write("----------Gradients are not interested----------") 491 | return None 492 | nodes_interested.append(appended.name) 493 | 494 | pre_check = True 495 | for son in nodes_interested[:-1]: 496 | u = self.node_by_name[son] 497 | try: 498 | getattr(InferValue, u.op.lower())([], u) 499 | except AttributeError: 500 | if u.op.lower() not in ["assert", "nextiteration"]: 501 | print(u.op, " not Implemented!") 502 | except: 503 | pass 504 | 505 | for son in nodes_interested[:-1]: 506 | u = self.node_by_name[son] 507 | if son in self.node_visited: 508 | continue 509 | 510 | self.node_visited.add(son) 511 | self.summary_node(son, u) 512 | 513 | range_to_split = set() 514 | for son in nodes_interested[:-1]: 515 | u = self.node_by_name[son] 516 | if u.op in ["Exp"]: # if it is a non-linear function 517 | in_node_name = self.graph_backward[0][son][0] 518 | in_node_output = self.node_output[in_node_name].index_of(self.edge_index[son][0]) 519 | non_self = True 520 | groups = in_node_output.array.block_to_symbol 521 | range_to_split_local = set() 522 | for key in groups: 523 | group = groups[key] 524 | for (name, position) in group.value: 525 | if name == in_node_name: 526 | non_self = False 527 | break 528 | factor = group.value[(name, position)] 529 | if factor != 0: 530 | if name[:len(magic)] == magic: 531 | range_to_split_local.add(name[len(magic):]) 532 | 533 | if non_self: 534 | range_to_split.update(range_to_split_local) 535 | 536 | return range_to_split, nodes_interested[:-1] 537 | 538 | # reevaluates the dataflow analysis for nodes_interested which contains the nodes in the backward slice of 539 | # node_interested. The reevaluation is implemented in an incremental manner, which only reevaluates the nodes which 540 | # will be affected by nodes in changed. The abstracted outputs of nodes in changed are overridden in override_dict. 541 | def reevaluate(self, nodes_interested, node_interested, changed, override_dict): 542 | back_up = {} 543 | for son in nodes_interested: 544 | u = self.node_by_name[son] 545 | has_changed = False 546 | for in_node_name in self.graph_backward[0][son]: # only care about non_control edges 547 | if in_node_name in changed: 548 | has_changed = True 549 | break 550 | 551 | if has_changed: 552 | back_up[son] = copy.deepcopy(self.node_output[son]) 553 | self.summary_node(son, u, override_dict) 554 | changed.add(son) 555 | 556 | ret = copy.deepcopy(self.node_output[node_interested]) 557 | # restore the back up 558 | for key in back_up: 559 | self.node_output[key] = back_up[key] 560 | return ret 561 | 562 | # gets the corresponding abstracted output in node_output. It will also consider the specially instrumented name 563 | # like "x|i" denoting the i-th element in the abstracted output. 564 | def get_value(self, name): 565 | if name.find("|") != -1: 566 | pos = name.find('|') 567 | index = int(name[pos + 1:]) 568 | return identity([self.node_output[name[:pos]].index_of(index)]) 569 | else: 570 | return identity([self.node_output[name].index_of(None)]) 571 | 572 | # computes the abstracted output of node_name using the tensor partition and the linear affine relation with values 573 | # of some nodes overridden by override_dict . groups is the block_to_symbol field of the Array object. 574 | def get_left_right(self, groups: dict, node_name, override_dict): 575 | left = [] 576 | right = [] 577 | for key in groups: 578 | left_ele = 0 579 | right_ele = 0 580 | group = groups[key] 581 | new_relu = {} 582 | 583 | def get_value(name, position): 584 | if name in override_dict: 585 | return override_dict[name] 586 | if (name, position) in override_dict: 587 | return override_dict[(name, position)] 588 | return self.get_value(name) 589 | 590 | def update_ele(factor, value, is_relu): 591 | if is_relu: 592 | value = Range(left=max(0, value.left), right=max(0, value.right)) 593 | 594 | value = value * factor 595 | if factor < 0: 596 | value.left, value.right = value.right, value.left 597 | return value 598 | 599 | for (name, position) in group.value: 600 | if name[:5] == magic: # We first store relu_value 601 | new_relu[(name, position)] = group.value[(name, position)] 602 | 603 | for (name, position) in group.value: 604 | if name[:5] == magic: # We then skip relu_value 605 | continue 606 | 607 | if name == node_name: 608 | if name in override_dict or (name, position) in override_dict: 609 | return get_value(name, position) 610 | return None 611 | 612 | value = get_value(name, position) 613 | 614 | if value is None: # only happens when self.node_output[name].index_of(index) is a zero-size array. 615 | continue 616 | 617 | value = Range(left=value.left, right=value.right) 618 | 619 | non_relu_factor = group.value[(name, position)] 620 | relu_name = magic + name 621 | # we first del the key,value pair in the dict 622 | if (relu_name, position) in group.value: 623 | relu_factor = group.value[(relu_name, position)] 624 | else: 625 | relu_factor = 0 626 | 627 | # this will be encountered secondly 628 | # axiom: x - relu(x) = -relu(-x). 629 | if relu_factor < 0 and non_relu_factor > 0: 630 | t = min(-relu_factor, non_relu_factor) 631 | non_relu_factor -= t # sub back the non_relu_factor 632 | relu_factor += t # add back the relu_factor 633 | left_ele += min(0, value.left) * t 634 | right_ele += min(0, value.right) * t 635 | 636 | if relu_factor > 0 and non_relu_factor < 0: 637 | t = min(relu_factor, -non_relu_factor) 638 | non_relu_factor += t # add back the non_relu_factor 639 | relu_factor -= t # sub back the relu_factor 640 | left_ele += max(0, -value.right) * t 641 | right_ele += max(0, -value.left) * t 642 | 643 | # we add back non-zero relu_factor 644 | new_relu[(relu_name, position)] = relu_factor 645 | 646 | value = update_ele(non_relu_factor, value, False) 647 | left_ele = left_ele + value.left 648 | right_ele = right_ele + value.right 649 | 650 | for (name, position) in new_relu: 651 | non_relu = name[len(magic):] 652 | value = get_value(non_relu, position) 653 | 654 | if value is None: # only happens when self.node_output[name].index_of(index) is a zero-size array. 655 | continue 656 | 657 | value = Range(left=value.left, right=value.right) 658 | value = update_ele(new_relu[(name, position)], value, True) 659 | left_ele = left_ele + value.left 660 | right_ele = right_ele + value.right 661 | 662 | left.append(left_ele) 663 | right.append(right_ele) 664 | 665 | if len(left) == 0 or len(right) == 0: 666 | return None 667 | return Range(left=min(left), right=max(right)) 668 | 669 | def get_info(self): 670 | variable_cnt = 0 671 | for op in self.node_by_name: 672 | if self.node_by_name[op].op.lower() in ["variablev2", "variable", "varhandleop"]: 673 | u = self.node_output[op].size 674 | if self.node_by_name[op].op.lower() == "varhandleop": 675 | u = shape_from_proto(self.node_by_name[op].attr["shape"].shape) 676 | 677 | tmp = 1 678 | if str(u) == '': 679 | continue 680 | for x in u: 681 | tmp *= int(x) 682 | variable_cnt += tmp 683 | 684 | return len(self.nodes_in_main_clique_topology), variable_cnt 685 | 686 | 687 | def main(): 688 | graph = Graph("./test.pbtxt") 689 | graph.backward_slice("Log", set()) 690 | -------------------------------------------------------------------------------- /parse/specified_ranges.py: -------------------------------------------------------------------------------- 1 | class SpecifiedRanges: 2 | models = ["Github-IPS-1", 3 | "Github-IPS-6", 4 | "Github-IPS-9", 5 | "StackOverflow-IPS-1", 6 | "StackOverflow-IPS-2", 7 | "StackOverflow-IPS-6", 8 | "StackOverflow-IPS-7", 9 | "StackOverflow-IPS-14", 10 | "TensorFuzz", 11 | "ssd_mobile_net_v1", 12 | "ssd_inception_v2", 13 | "ssd_mobile_net_v2", 14 | "faster_rcnn_resnet_50", 15 | "deep_speech", 16 | "deeplab", 17 | "autoencoder_mnae", 18 | "autoencoder_vae", 19 | "attention_ocr", 20 | "textsum", 21 | "shake_shake_32", 22 | "shake_shake_96", 23 | "shake_shake_112", 24 | "pyramid_net", 25 | "sbn", 26 | "sbnrebar", 27 | "sbndynamicrebar", 28 | "sbngumbel", 29 | "audioset", 30 | "learning_to_remember_rare_events", 31 | "neural_gpu1", 32 | "neural_gpu2", 33 | "ptn", 34 | "namignizer", 35 | "feelvos", 36 | "fivo_srnn", 37 | "fivo_vrnn", 38 | "fivo_ghmm", 39 | "deep_contextual_bandits_var_bnn", 40 | "deep_contextual_bandits_neural_ban", 41 | "deep_contextual_bandits_bb_alpha_nn", 42 | "deep_contextual_bandits_rms_bnn", 43 | "adversarial_crypto", 44 | "sentiment_analysis", 45 | "next_frame_prediction", 46 | "minigo", 47 | "compression_entropy_coder", 48 | "lfads", 49 | "lm_1b", 50 | "swivel", 51 | "skip_thought", 52 | "video_prediction", 53 | "gan_mnist", 54 | "gan_cifar", 55 | "gan_image_compression", 56 | "vid2depth", 57 | "domain_adaptation", 58 | "delf", ] 59 | 60 | # a dictionary with key = "filename", and value with another dictionary {"variable_name" -> ranges} 61 | specified_ranges = { 62 | "Github-IPS-1": {"Placeholder_2": [0.5, 1], "Placeholder": [-1, 1]}, 63 | # keep prob; mnist image pixel 64 | "Github-IPS-6": {"x-input": [-1, 1]}, 65 | # mnist image pixel 66 | "Github-IPS-9": {"Placeholder": [-1, 1]}, 67 | # mnist image pixel 68 | "StackOverflow-IPS-1": {"Placeholder_2": [0.5, 1], "Placeholder": [-1, 1]}, 69 | # keep prob; mnist image pixel 70 | "StackOverflow-IPS-2": {"Placeholder_2": [0.5, 1], "Placeholder": [-1, 1]}, 71 | # keep prob; mnist image pixel 72 | "StackOverflow-IPS-6": {"Placeholder_2": [0.5, 1], "Placeholder": [-1, 1]}, 73 | # keep prob; mnist image pixel 74 | "StackOverflow-IPS-7": {"Placeholder": [0.5, 1]}, 75 | # mnist image pixel 76 | "StackOverflow-IPS-14": {"Placeholder": [0.5, 1]}, 77 | # mnist image pixel 78 | "TensorFuzz": {"OneShotIterator": [[-1, 1], [0, 9]]}, 79 | # mnist image pixel; labels 80 | "ssd_mobile_net_v1": { 81 | "IteratorV2": [[0, None], [-1, 1], [None, None], [1, None], [None, None], [0, 299], [0, 1], [0, 1], 82 | [None, None], [False, True], [0, 1], [1, 100]]}, 83 | # HASH_KEY ; image pixels from [-1, 1]; unknown; real shape of image; unknown; the corner of boxes; 84 | # one hot values; one hot values; unknown; boolean value; weights; number of boxes 85 | "ssd_inception_v2": { 86 | "IteratorV2": [[0, None], [-1, 1], [None, None], [1, None], [None, None], [0, 299], [0, 1], [0, 1], 87 | [None, None], [False, True], [0, 1], [1, 100]]}, 88 | # the same reason above 89 | "ssd_mobile_net_v2": { 90 | "IteratorV2": [[0, None], [-1, 1], [None, None], [1, None], [None, None], [0, 299], [0, 1], [0, 1], 91 | [None, None], [False, True], [0, 1], [1, 100]]}, 92 | # the same reason above 93 | "faster_rcnn_resnet_50": { 94 | "IteratorV2": [[0, None], [-1, 1], [None, None], [1, None], [None, None], [0, 299], [0, 1], [0, 1], 95 | [None, None], [False, True], [0, 1], [1, 100]]}, 96 | # the same reason above 97 | "deep_speech": {"IteratorV2": [[-1, 1], [0, None], [0, None], [0, None]]}, 98 | # spectrogram features; input length; label length; number of classes 99 | "deeplab": {}, 100 | # no input related 101 | "autoencoder_vae": {"Placeholder": [-1, 1]}, 102 | # mnist image pixel 103 | "autoencoder_mnae": {"Placeholder": [0.5, 1]}, 104 | # keep prob 105 | "attention_ocr": {"CharAccuracy/mean/count": [1, None], "SequenceAccuracy/mean/count": [1, None]}, 106 | # count; count 107 | "textsum": {"targets": [0, None], "loss_weights": [1e-10, 1]}, 108 | # token_id; loss_weights 109 | "shake_shake_32": {"model/accuracy/count": [1, None], "model_1/accuracy/count": [1, None]}, 110 | # count; count 111 | "shake_shake_96": {"model/accuracy/count": [1, None], "model_1/accuracy/count": [1, None]}, 112 | # the same reason above 113 | "shake_shake_112": {"model/accuracy/count": [1, None], "model_1/accuracy/count": [1, None]}, 114 | # the same reason above 115 | "pyramid_net": {"model/accuracy/count": [1, None], "model_1/accuracy/count": [1, None]}, 116 | # the same reason above 117 | "sbn": {"Variable": [-1, 1], "Placeholder": [1, None], "Placeholder_1": [-1, 1]}, 118 | # weights; number of examples ; image pixels 119 | "sbnrebar": {"Variable": [-1, 1], "Placeholder": [1, None], "Placeholder_1": [-1, 1]}, 120 | # the same reason above 121 | "sbndynamicrebar": {"Variable": [-1, 1], "Placeholder": [1, None], "Placeholder_1": [-1, 1]}, 122 | # the same reason above 123 | "sbngumbel": {"Variable": [-1, 1], "Placeholder": [1, None], "Placeholder_1": [-1, 1]}, 124 | # the same reason above 125 | "audioset": {"vggish/input_features": [-1, 1]}, 126 | # vggish input_features 127 | "learning_to_remember_rare_events": {"Placeholder": [-1, 1], "recent_idx": [0, None], "Placeholder_1": [0, 9], 128 | "memvals": [None, None]}, 129 | # mnist pixel, index, mnist label; unknown 130 | "neural_gpu1": {"global_step": [1, None], "inp": [None, None], "length": [1, None], "tgt": [None, None], 131 | "do_training": [0.1, 1]}, 132 | # global training step; unknown inp, length, unknown tgt, dropout rate 133 | "neural_gpu2": {"global_step": [1, None], "inp": [None, None], "length": [1, None], "tgt": [None, None], 134 | "do_training": [0.1, 1]}, 135 | # the same reason above 136 | "ptn": {}, 137 | # no input related 138 | "namignizer": {"model/Placeholder_2": [1e-10, 1]}, 139 | # weights 140 | "feelvos": {}, 141 | # no input related 142 | "fivo_srnn": {"OneShotIterator": [[0, 1], [0, 1], [1, None]]}, 143 | # one hot values, one hot values, len 144 | "fivo_vrnn": {"OneShotIterator": [[0, 1], [0, 1], [1, None]]}, 145 | # the same reason above 146 | "fivo_ghmm": {"OneShotIterator": [[-1, 1], [-1, 1]]}, 147 | # inputs range 148 | "deep_contextual_bandits_var_bnn": {"Placeholder_1": [None, None], "Placeholder": [1, None]}, 149 | # rewards, size of data 150 | "deep_contextual_bandits_neural_ban": {"global_step": [1, None]}, 151 | # global training step 152 | "deep_contextual_bandits_bb_alpha_nn": {"data_size": [1, None], "w": [0, 1], "y": [None, None], 153 | "x": [None, None]}, 154 | # data size; weights for actions, rewards, rewards 155 | "deep_contextual_bandits_rms_bnn": {"global_step": [1, None]}, 156 | # global training step 157 | "adversarial_crypto": {}, 158 | # no input related 159 | "sentiment_analysis": {"input_1": [0, None], "batch_normalization_v1/keras_learning_phase": [False, True], 160 | "batch_normalization_v1/moving_variance": [0, None]}, 161 | # token_id; train or test; variance 162 | "next_frame_prediction": {"shuffle_batch/random_shuffle_queue": [[-1, 1]]}, 163 | # video feature 164 | "minigo": {"pos_tensor": [-1, 1]}, 165 | # pos feature 166 | "compression_entropy_coder": {"padding_fifo_queue": [[-1, 1]]}, 167 | # image pixel 168 | "lfads": {"LFADS/keep_prob": [0.95, 1], "LFADS/data": [-1, 1], 169 | "LFADS/z/ic_enc_2_post_g0/logvar/b": [-0.0625, 0.0625], 170 | "LFADS/z/ic_enc_2_post_g0/logvar/W": [-0.0625, 0.0625]}, 171 | # dropout prob; embedding; ranged initialize; ranged initialize 172 | "lm_1b": {}, 173 | # no related input 174 | "swivel": {"input_producer": [[None, None]]}, 175 | # unknown 176 | "skip_thought": { 177 | "random_input_queue": [[0, None], [0, None], [0, None], [0, None], [0, None], [0, None], [0, None], 178 | [0, None], [0, None], [0, None], [0, None]], "beta2_power": [0.1, 0.9], 179 | "beta1_power": [0.1, 0.9]}, 180 | # token_id & one hot values; optimizer beta powers; optimizer beta powers 181 | "video_prediction": {"model/Placeholder": [1, 10000], 182 | "model/batch/fifo_queue": [[-1, 1], [None, None], [None, None]], 183 | "val_model/Placeholder": [1, 10000], 184 | "val_model/batch/fifo_queue": [[-1, 1], [None, None], [None, None]]}, 185 | # iter num; inputs video, unknown, unknown; iter num; inputs video, unknown, unknown 186 | "gan_mnist": {"inputs/batch/fifo_queue": [[-1, 1], [None, None]]}, 187 | # mnist image pixel; unknown 188 | "gan_cifar": {}, 189 | # no related input 190 | "gan_image_compression": {}, 191 | # no related input 192 | "vid2depth": {"data_loading/batching/shuffle_batch/random_shuffle_queue": [[0, 1], [1, 100], [1, 100]]}, 193 | # inputs video, unknown, unknown; 194 | "domain_adaptation": {"batch_1/fifo_queue": [[-1, 1], [None, None]], 195 | "batch/fifo_queue": [[-1, 1], [None, None]]}, 196 | # mnist pixel, unknown; cifar pixel, unknown; 197 | "real_nvp": {"model/shuffle_batch/random_shuffle_queue": [[0, 1]]}, 198 | # 199 | "delf": {"input_scales": [0.1, 1], "input_image": [0, 255], "input_abs_thres": [0, None], 200 | "input_max_feature_num": [0, None]} 201 | # scale; input pixel in 0-255; abs value; feature num 202 | } 203 | 204 | # the following dict is called by the parse_format_text.py 205 | ranges_looking_up = {} 206 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | z3-solver 3 | protobuf 4 | graphviz 5 | 6 | -------------------------------------------------------------------------------- /solver.py: -------------------------------------------------------------------------------- 1 | import z3 2 | import math 3 | import numpy as np 4 | from itertools import product 5 | import copy 6 | import bisect 7 | 8 | from utils import resolve_type 9 | 10 | magic = "$relu" 11 | 12 | 13 | # legacy class of z3-solver 14 | class Solver: 15 | solver = z3.Solver() 16 | index = {} 17 | variable_by_name = {} 18 | 19 | @staticmethod 20 | def add_variable(name, dtype): 21 | if name not in Solver.index: 22 | Solver.index[name] = 0 23 | variable_name = name + "_" + str(Solver.index[name]) 24 | Solver.index[name] += 1 25 | if dtype in [3]: 26 | real_name = variable_name + "_Int" 27 | Solver.variable_by_name[real_name] = z3.Int(real_name) 28 | elif dtype in [1]: 29 | real_name = variable_name + "_Real" 30 | Solver.variable_by_name[real_name] = z3.Real(real_name) 31 | elif dtype in [10]: 32 | real_name = variable_name + "_Bool" 33 | Solver.variable_by_name[real_name] = z3.Bool(real_name) 34 | else: 35 | raise NotImplementedError("Cannot Recognize: ", dtype) 36 | return Solver.variable_by_name[real_name] 37 | 38 | @staticmethod 39 | def max(x, ys_): 40 | ys1 = [y for y in list(map(resolve_type, ys_)) if str(y) != 'inf'] 41 | ys = [y for y in ys1 if str(y) != '-inf'] 42 | if len(ys1) != len(ys_): 43 | z3.And([x >= y for y in ys]) 44 | 45 | try: 46 | return x == max(ys) 47 | except: 48 | pass 49 | if len(ys) == 1: 50 | return x == ys[0] 51 | if len(ys) == 2: 52 | return x == z3.If(ys[0] > ys[1], ys[0], ys[1]) 53 | return z3.And(z3.Or([x == y for y in ys]), z3.And([x >= y for y in ys])) 54 | 55 | @staticmethod 56 | def min(x, ys_): 57 | ys1 = [y for y in list(map(resolve_type, ys_)) if str(y) != '-inf'] 58 | ys = [y for y in ys1 if str(y) != 'inf'] 59 | if len(ys1) != len(ys_): 60 | z3.And([x <= y for y in ys]) 61 | 62 | try: 63 | return x == min(ys) 64 | except: 65 | pass 66 | if len(ys) == 1: 67 | return x == ys[0] 68 | if len(ys) == 2: 69 | return x == z3.If(ys[0] < ys[1], ys[0], ys[1]) 70 | return z3.And(z3.Or([x == y for y in ys]), z3.And([x <= y for y in ys])) 71 | 72 | @staticmethod 73 | def in_interval(x, interval): 74 | if isinstance(interval, tuple): 75 | if interval[0] > 0 or interval[1] > 0: 76 | # (a, b] 77 | if math.isinf(interval[1]): 78 | return z3.And(interval[0] < x) 79 | else: 80 | return z3.And(interval[0] <= x, x <= interval[1]) 81 | else: 82 | # [a, b) 83 | if math.isinf(interval[0]): 84 | return z3.And(x < interval[1]) 85 | else: 86 | return z3.And(interval[0] <= x, x <= interval[1]) 87 | else: 88 | return x == interval 89 | 90 | 91 | # data structure for interval abstraction 92 | class Range: 93 | def __init__(self, *args, **kwargs): 94 | """Two ways of construction: 95 | left, right 96 | name, dtype 97 | One optional parameter for range_const 98 | 99 | The int and float tensor representation --> interval 100 | The bool tensor representation --> [True, False] for all False, 101 | [False, True] for all True, 102 | [True, True] for both True and False 103 | """ 104 | if "const_type" in kwargs: 105 | self.const_type = kwargs["const_type"] 106 | else: 107 | self.const_type = None 108 | if "name" in kwargs and "dtype" in kwargs: 109 | name = kwargs["name"] 110 | dtype = kwargs["dtype"] 111 | self.left = Solver.add_variable(name + "L", dtype) 112 | self.right = Solver.add_variable(name + "R", dtype) 113 | elif "left" in kwargs and "right" in kwargs: 114 | self.left = resolve_type(kwargs["left"]) 115 | self.right = resolve_type(kwargs["right"]) 116 | else: 117 | raise NotImplementedError(args, kwargs, " setting not implemented") 118 | 119 | def __str__(self): 120 | return "[%s, %s]\n[%s, %s]" % (self.left, self.right, str(type(self.left)), str(type(self.right))) 121 | 122 | def __repr__(self): 123 | return "[%s, %s]\n[%s, %s]" % (self.left, self.right, str(type(self.left)), str(type(self.right))) 124 | 125 | def __mul__(self, other): 126 | return Range(left=None if self.left is None else self.left * other, 127 | right=None if self.right is None else self.right * other, 128 | const_type=self.const_type) 129 | 130 | def __add__(self, other): 131 | return Range(left=None if self.left is None else self.left + other, 132 | right=None if self.right is None else self.right + other, 133 | const_type=self.const_type) 134 | 135 | def single(self): 136 | return self.left == self.right 137 | 138 | 139 | class Linear: 140 | def __init__(self, e): 141 | # a map maps from variables to the their factors 142 | self.value = {e: 1} 143 | self.map_to_index = {e: list(range(len(e[1])))} 144 | # map_to_index is the mapping from e = (name, position) to the Array's partition 145 | # i-th index of Array's partition is mapped to map_to_index[i]-th index of name's position. 146 | # The purpose of maintaining this index mapping is that after operations like unpack, additional dimensions may 147 | # be added, after operation like pack, the dimensions may be deleted (these dimensions are all equal to 1 and do 148 | # not change the size of the partition), after operation like transpose, the dimensions may be permuted. 149 | 150 | def __str__(self): 151 | return "\t\tvalue: %s\n\t\tmap_to_index: %s" % (str(self.value), str(self.map_to_index)) 152 | 153 | def __repr__(self): 154 | return "\t\tvalue: %s\n\t\tmap_to_index: %s" % (str(self.value), str(self.map_to_index)) 155 | 156 | def __add__(self, other): 157 | # adds between two affine expressions and returns a new Linear object. 158 | ret = copy.deepcopy(self) 159 | for x in other.value: 160 | if x in ret.value: 161 | ret.value[x] += other.value[x] 162 | else: 163 | ret.value[x] = other.value[x] 164 | ret.map_to_index[x] = other.map_to_index[x] 165 | return ret 166 | 167 | def __sub__(self, other): 168 | # subs between two affine expressions and returns a new Linear object. 169 | ret = copy.deepcopy(self) 170 | for x in other.value: 171 | if x in ret.value: 172 | ret.value[x] -= other.value[x] 173 | else: 174 | ret.value[x] = -other.value[x] 175 | ret.map_to_index[x] = other.map_to_index[x] 176 | return ret 177 | 178 | def choose(self, start_ind): 179 | # futher partitions the variables inside the Linear object and returns a new partitioned Linear object. 180 | # len(start_ind) = len(x[1]) = len(map) 181 | ret = copy.deepcopy(self) 182 | ret.value = {} 183 | ret.map_to_index = {} 184 | for x in self.value: 185 | name, position = x 186 | new_tp = list(position) # if not mapped, then remain 187 | map = self.map_to_index[x] 188 | for t in range(len(start_ind)): 189 | if map[t] is not None: 190 | i = map[t] 191 | if start_ind[t] is not None: 192 | new_tp[i] = (new_tp[i][0] + start_ind[t][0], new_tp[i][0] + start_ind[t][1]) 193 | 194 | ret.value[(name, tuple(new_tp))] = self.value[x] 195 | ret.map_to_index[(name, tuple(new_tp))] = copy.deepcopy(map) 196 | 197 | return ret 198 | 199 | def transpose(self, perm): 200 | # transposes the map_to_index according to the permutation perm and returns a new transposed Linear object. 201 | # len(perm) = len(x[1]) = len(map) 202 | ret = copy.deepcopy(self) 203 | for x in self.value: 204 | map = self.map_to_index[x] 205 | new_map = [None] * len(map) 206 | for t in range(len(perm)): 207 | new_map[t] = map[perm[t]] 208 | ret.map_to_index[x] = new_map 209 | 210 | return ret 211 | 212 | def add_pack_ind(self, pack_ind): 213 | # adds an axis at the pack_ind-th dimension and returns a new packed Linear object. 214 | ret = copy.deepcopy(self) 215 | for x in self.value: 216 | map = self.map_to_index[x] 217 | new_map = map[:pack_ind] + [None] + map[pack_ind:] 218 | ret.map_to_index[x] = new_map 219 | 220 | return ret 221 | 222 | def remove_unpack_axis(self, axis): 223 | # removes an axis at the axis-th dimension and returns a new unpacked Linear object. 224 | ret = copy.deepcopy(self) 225 | for x in self.value: 226 | map = self.map_to_index[x] 227 | new_map = map[:axis] + map[axis:] 228 | ret.map_to_index[x] = new_map 229 | 230 | return ret 231 | 232 | def neg(self): 233 | # calculates the negation of the affine expression. 234 | for x in self.value: 235 | self.value[x] *= -1 236 | 237 | def relu(self): 238 | # calculates the relu of the affine expression and returns a new Linear object. 239 | # only supports calculating the relu of a singleton affine expression that only contains one variable or one 240 | # constant value. 241 | # The following axioms are used to calculate relu: 242 | # relu(x)=relu(x) 243 | # relu(-x)=-x+relu(x) 244 | # relu(relu(x))=relu(x) 245 | # relu(-relu(x))=0 246 | 247 | assert len(self.value) <= 1 248 | ret = Linear(("dumy", (0, 1))) 249 | ret.value = {} 250 | ret.map_to_index = {} 251 | for x in self.value: 252 | name, position = x 253 | if name[:len(magic)] != magic: # relu(name) 254 | if self.value[x] >= 0: 255 | ret.value[(magic + name, position)] = self.value[x] 256 | ret.map_to_index[(magic + name, position)] = self.map_to_index[x] 257 | else: 258 | ret.value[(magic + name, position)] = -self.value[x] 259 | ret.map_to_index[(magic + name, position)] = self.map_to_index[x] 260 | ret.value[(name, position)] = self.value[x] 261 | ret.map_to_index[(name, position)] = self.map_to_index[x] 262 | else: 263 | if self.value[x] >= 0: 264 | ret.value[(name, position)] = self.value[x] 265 | ret.map_to_index[(name, position)] = self.map_to_index[x] 266 | else: 267 | ret.value[(name, position)] = 0 268 | ret.map_to_index[(name, position)] = self.map_to_index[x] 269 | 270 | return ret 271 | 272 | 273 | class Array: 274 | 275 | def __init__(self, name, size): 276 | # a list stores the partitioning positions of each dimension 277 | self.index_slices = [] 278 | # a map maps from each partition to a Linear object, which maintains the linear affine relation. 279 | # Each partition is defined by the Cartesian product of d tuples in index_slices . 280 | self.block_to_symbol = {} 281 | try: 282 | len(size) 283 | except: 284 | self.index_slices = None 285 | return 286 | 287 | for i in range(len(size)): 288 | try: 289 | self.index_slices.append([int(size[i])]) 290 | except: 291 | self.index_slices.append([None]) 292 | self.block_to_symbol = { 293 | tuple([x[0] for x in self.index_slices]): Linear((name, tuple([(0, x[0]) for x in self.index_slices])))} 294 | 295 | @staticmethod 296 | def join_index_slices(a, b): 297 | # aligns two sets of partitioning positions a and b. 298 | ret = [] 299 | for i in range(len(a)): 300 | if a[i][0] is None and b[i][0] is None: # if one of the dimension is unknown 301 | ret.append([None]) 302 | else: 303 | assert a[i][0] is not None and b[i][0] is not None 304 | c = np.unique(a[i] + b[i]) # join the current dimension of a and b 305 | ret.append(list(c)) 306 | 307 | return ret 308 | 309 | def get_corresponding_keys(self, index_slices): 310 | # gets the corresponding Linear objects according to index_slices. 311 | # Notice that index_slices may have a finer granularity than self.index_slices, so the Linear object may need 312 | # to be further partitioned. 313 | ret = [] 314 | for indexes in product(*index_slices): # enumerate the Cartesian product of index_slices 315 | key = () 316 | start_ind = [] 317 | for i in range(len(indexes)): 318 | if indexes[i] is not None: # if the dimension is not unknown 319 | t = bisect.bisect_left(index_slices[i], indexes[i]) 320 | start_ind.append([0 if t == 0 else index_slices[i][t - 1], indexes[i]]) 321 | iargs = bisect.bisect_left(self.index_slices[i], indexes[i]) 322 | # calculate the partitioning positions inside Linear object 323 | if iargs > 0: 324 | start_ind[-1][0] -= self.index_slices[i][iargs - 1] 325 | start_ind[-1][1] -= self.index_slices[i][iargs - 1] 326 | 327 | key += (self.index_slices[i][iargs],) 328 | else: 329 | key += (None,) 330 | start_ind.append(None) 331 | 332 | # further partition the Linear object 333 | ret.append(self.block_to_symbol[key].choose(start_ind)) 334 | 335 | return ret 336 | 337 | def __str__(self): 338 | ret_str = "" 339 | for x in self.block_to_symbol: 340 | ret_str += str(x) + "\t" + str(self.block_to_symbol[x]) + "\n" 341 | ret_str += str(self.index_slices) + "\n" 342 | return ret_str 343 | 344 | def __repr__(self): 345 | ret_str = "" 346 | for x in self.block_to_symbol: 347 | ret_str += str(x) + "\t" + str(self.block_to_symbol[x]) + "\n" 348 | ret_str += str(self.index_slices) + "\n" 349 | return ret_str 350 | 351 | 352 | # checks whether a Range object has a const lower and upper bound 353 | def check_range_const(range_const: Range): 354 | if z3.is_arith(range_const.left) or z3.is_arith(range_const.right): 355 | return True 356 | return not (range_const.left is not None and range_const.right is not None and range_const.left > range_const.right) 357 | 358 | 359 | # checks whether the interval of `range` intersects with the interval of `range_const` 360 | def meet(range, range_const: Range): 361 | if not check_range_const(range_const): 362 | return False 363 | assert range_const.const_type is not None 364 | 365 | if range_const.const_type == 0: 366 | if isinstance(range, Range): 367 | if range_const.left is not None and range_const.right is not None: 368 | return z3.Not(z3.Or(range_const.right < range.left, range.right < range_const.left)) 369 | if range_const.right is not None: 370 | return z3.Or(range.left <= range_const.right, range.right <= range_const.right) 371 | if range_const.left is not None: 372 | return z3.Or(range_const.left <= range.left, range_const.left <= range.right) 373 | else: 374 | return True 375 | else: 376 | if range_const.left is not None and range_const.right is not None: 377 | return bool(np.all(range_const.left <= range) and np.all(range <= range_const.right)) 378 | if range_const.right is not None: 379 | return bool(np.all(range <= range_const.right)) 380 | if range_const.left is not None: 381 | return bool(np.all(range_const.left <= range)) 382 | else: 383 | return True 384 | else: 385 | raise NotImplementedError 386 | 387 | 388 | def meet_relation_variable(rv, range_const: Range): 389 | if not check_range_const(range_const): 390 | return False 391 | assert range_const.const_type is not None 392 | 393 | if range_const.const_type == 0: 394 | if range_const.left is not None and range_const.right is not None: 395 | return z3.And(range_const.left <= rv, rv <= range_const.right) 396 | if range_const.right is not None: 397 | return rv <= range_const.right 398 | if range_const.left is not None: 399 | return range_const.left <= rv 400 | else: 401 | return True 402 | else: 403 | raise NotImplementedError 404 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # the overflow and underflow limit in tf.float32. 4 | OVERFLOW_LIMIT = 1e38 5 | UNDERFLOW_LIMIT = 1e-37 6 | OVERFLOW_D = 38 7 | UNDERFLOW_D = -37 8 | 9 | 10 | # onverts data types in numpy to python primitive data types. 11 | def resolve_type(y): 12 | if isinstance(y, np.int32) or isinstance(y, np.int64): 13 | return int(y) 14 | elif isinstance(y, np.float32) or isinstance(y, np.float64): 15 | return float(y) 16 | elif isinstance(y, np.bool): 17 | return bool(y) 18 | else: 19 | return y 20 | 21 | # parses the tensor shape from protocol buffer file into a python list 22 | def shape_from_proto(shape): 23 | s = str(shape) 24 | x = 0 25 | u = [] 26 | for i in range(len(s)): 27 | if s[i] >= '0' and s[i] <= '9': 28 | x = x * 10 + ord(s[i]) - 48 29 | elif x != 0: 30 | u.append(x) 31 | x = 0 32 | 33 | return u 34 | --------------------------------------------------------------------------------