├── .gitignore ├── LICENSE ├── README.md ├── example └── lite_tracer_example.py ├── lite_tracer ├── __init__.py ├── exceptions.py ├── lite_trace.py └── tracker.py ├── logos ├── Litetracer-Logo-Full-DB.png └── Litetracer-Logo-Full-LB.png ├── pytest.ini ├── setup.py └── tests ├── helper.py ├── test_dest.py ├── test_flags.py ├── test_lite_tracer.py └── test_search.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | .pytest_cache/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | db.sqlite3 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # IPython 78 | profile_default/ 79 | ipython_config.py 80 | 81 | # pyenv 82 | .python-version 83 | 84 | # celery beat schedule file 85 | celerybeat-schedule 86 | 87 | # SageMath parsed files 88 | *.sage.py 89 | 90 | # Environments 91 | .env 92 | .venv 93 | env/ 94 | venv/ 95 | ENV/ 96 | env.bak/ 97 | venv.bak/ 98 | 99 | # Spyder project settings 100 | .spyderproject 101 | .spyproject 102 | 103 | # Rope project settings 104 | .ropeproject 105 | 106 | # mkdocs documentation 107 | /site 108 | 109 | # mypy 110 | .mypy_cache/ 111 | .dmypy.json 112 | dmypy.json 113 | 114 | # Pyre type checker 115 | .pyre/ 116 | 117 | # Swap 118 | [._]*.s[a-v][a-z] 119 | [._]*.sw[a-p] 120 | [._]s[a-rt-v][a-z] 121 | [._]ss[a-gi-z] 122 | [._]sw[a-p] 123 | 124 | # Session 125 | Session.vim 126 | 127 | # Temporary 128 | .netrwhist 129 | *~ 130 | # Auto-generated tag files 131 | tags 132 | # Persistent undo 133 | [._]*.un~ 134 | 135 | lt_records 136 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Attribution-NonCommercial-ShareAlike 4.0 International 2 | 3 | ======================================================================= 4 | 5 | Creative Commons Corporation ("Creative Commons") is not a law firm and 6 | does not provide legal services or legal advice. Distribution of 7 | Creative Commons public licenses does not create a lawyer-client or 8 | other relationship. Creative Commons makes its licenses and related 9 | information available on an "as-is" basis. Creative Commons gives no 10 | warranties regarding its licenses, any material licensed under their 11 | terms and conditions, or any related information. Creative Commons 12 | disclaims all liability for damages resulting from their use to the 13 | fullest extent possible. 14 | 15 | Using Creative Commons Public Licenses 16 | 17 | Creative Commons public licenses provide a standard set of terms and 18 | conditions that creators and other rights holders may use to share 19 | original works of authorship and other material subject to copyright 20 | and certain other rights specified in the public license below. The 21 | following considerations are for informational purposes only, are not 22 | exhaustive, and do not form part of our licenses. 23 | 24 | Considerations for licensors: Our public licenses are 25 | intended for use by those authorized to give the public 26 | permission to use material in ways otherwise restricted by 27 | copyright and certain other rights. Our licenses are 28 | irrevocable. Licensors should read and understand the terms 29 | and conditions of the license they choose before applying it. 30 | Licensors should also secure all rights necessary before 31 | applying our licenses so that the public can reuse the 32 | material as expected. Licensors should clearly mark any 33 | material not subject to the license. This includes other CC- 34 | licensed material, or material used under an exception or 35 | limitation to copyright. More considerations for licensors: 36 | wiki.creativecommons.org/Considerations_for_licensors 37 | 38 | Considerations for the public: By using one of our public 39 | licenses, a licensor grants the public permission to use the 40 | licensed material under specified terms and conditions. If 41 | the licensor's permission is not necessary for any reason--for 42 | example, because of any applicable exception or limitation to 43 | copyright--then that use is not regulated by the license. Our 44 | licenses grant only permissions under copyright and certain 45 | other rights that a licensor has authority to grant. Use of 46 | the licensed material may still be restricted for other 47 | reasons, including because others have copyright or other 48 | rights in the material. A licensor may make special requests, 49 | such as asking that all changes be marked or described. 50 | Although not required by our licenses, you are encouraged to 51 | respect those requests where reasonable. More considerations 52 | for the public: 53 | wiki.creativecommons.org/Considerations_for_licensees 54 | 55 | ======================================================================= 56 | 57 | Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International 58 | Public License 59 | 60 | By exercising the Licensed Rights (defined below), You accept and agree 61 | to be bound by the terms and conditions of this Creative Commons 62 | Attribution-NonCommercial-ShareAlike 4.0 International Public License 63 | ("Public License"). To the extent this Public License may be 64 | interpreted as a contract, You are granted the Licensed Rights in 65 | consideration of Your acceptance of these terms and conditions, and the 66 | Licensor grants You such rights in consideration of benefits the 67 | Licensor receives from making the Licensed Material available under 68 | these terms and conditions. 69 | 70 | 71 | Section 1 -- Definitions. 72 | 73 | a. Adapted Material means material subject to Copyright and Similar 74 | Rights that is derived from or based upon the Licensed Material 75 | and in which the Licensed Material is translated, altered, 76 | arranged, transformed, or otherwise modified in a manner requiring 77 | permission under the Copyright and Similar Rights held by the 78 | Licensor. For purposes of this Public License, where the Licensed 79 | Material is a musical work, performance, or sound recording, 80 | Adapted Material is always produced where the Licensed Material is 81 | synched in timed relation with a moving image. 82 | 83 | b. Adapter's License means the license You apply to Your Copyright 84 | and Similar Rights in Your contributions to Adapted Material in 85 | accordance with the terms and conditions of this Public License. 86 | 87 | c. BY-NC-SA Compatible License means a license listed at 88 | creativecommons.org/compatiblelicenses, approved by Creative 89 | Commons as essentially the equivalent of this Public License. 90 | 91 | d. Copyright and Similar Rights means copyright and/or similar rights 92 | closely related to copyright including, without limitation, 93 | performance, broadcast, sound recording, and Sui Generis Database 94 | Rights, without regard to how the rights are labeled or 95 | categorized. For purposes of this Public License, the rights 96 | specified in Section 2(b)(1)-(2) are not Copyright and Similar 97 | Rights. 98 | 99 | e. Effective Technological Measures means those measures that, in the 100 | absence of proper authority, may not be circumvented under laws 101 | fulfilling obligations under Article 11 of the WIPO Copyright 102 | Treaty adopted on December 20, 1996, and/or similar international 103 | agreements. 104 | 105 | f. Exceptions and Limitations means fair use, fair dealing, and/or 106 | any other exception or limitation to Copyright and Similar Rights 107 | that applies to Your use of the Licensed Material. 108 | 109 | g. License Elements means the license attributes listed in the name 110 | of a Creative Commons Public License. The License Elements of this 111 | Public License are Attribution, NonCommercial, and ShareAlike. 112 | 113 | h. Licensed Material means the artistic or literary work, database, 114 | or other material to which the Licensor applied this Public 115 | License. 116 | 117 | i. Licensed Rights means the rights granted to You subject to the 118 | terms and conditions of this Public License, which are limited to 119 | all Copyright and Similar Rights that apply to Your use of the 120 | Licensed Material and that the Licensor has authority to license. 121 | 122 | j. Licensor means the individual(s) or entity(ies) granting rights 123 | under this Public License. 124 | 125 | k. NonCommercial means not primarily intended for or directed towards 126 | commercial advantage or monetary compensation. For purposes of 127 | this Public License, the exchange of the Licensed Material for 128 | other material subject to Copyright and Similar Rights by digital 129 | file-sharing or similar means is NonCommercial provided there is 130 | no payment of monetary compensation in connection with the 131 | exchange. 132 | 133 | l. Share means to provide material to the public by any means or 134 | process that requires permission under the Licensed Rights, such 135 | as reproduction, public display, public performance, distribution, 136 | dissemination, communication, or importation, and to make material 137 | available to the public including in ways that members of the 138 | public may access the material from a place and at a time 139 | individually chosen by them. 140 | 141 | m. Sui Generis Database Rights means rights other than copyright 142 | resulting from Directive 96/9/EC of the European Parliament and of 143 | the Council of 11 March 1996 on the legal protection of databases, 144 | as amended and/or succeeded, as well as other essentially 145 | equivalent rights anywhere in the world. 146 | 147 | n. You means the individual or entity exercising the Licensed Rights 148 | under this Public License. Your has a corresponding meaning. 149 | 150 | 151 | Section 2 -- Scope. 152 | 153 | a. License grant. 154 | 155 | 1. Subject to the terms and conditions of this Public License, 156 | the Licensor hereby grants You a worldwide, royalty-free, 157 | non-sublicensable, non-exclusive, irrevocable license to 158 | exercise the Licensed Rights in the Licensed Material to: 159 | 160 | a. reproduce and Share the Licensed Material, in whole or 161 | in part, for NonCommercial purposes only; and 162 | 163 | b. produce, reproduce, and Share Adapted Material for 164 | NonCommercial purposes only. 165 | 166 | 2. Exceptions and Limitations. For the avoidance of doubt, where 167 | Exceptions and Limitations apply to Your use, this Public 168 | License does not apply, and You do not need to comply with 169 | its terms and conditions. 170 | 171 | 3. Term. The term of this Public License is specified in Section 172 | 6(a). 173 | 174 | 4. Media and formats; technical modifications allowed. The 175 | Licensor authorizes You to exercise the Licensed Rights in 176 | all media and formats whether now known or hereafter created, 177 | and to make technical modifications necessary to do so. The 178 | Licensor waives and/or agrees not to assert any right or 179 | authority to forbid You from making technical modifications 180 | necessary to exercise the Licensed Rights, including 181 | technical modifications necessary to circumvent Effective 182 | Technological Measures. For purposes of this Public License, 183 | simply making modifications authorized by this Section 2(a) 184 | (4) never produces Adapted Material. 185 | 186 | 5. Downstream recipients. 187 | 188 | a. Offer from the Licensor -- Licensed Material. Every 189 | recipient of the Licensed Material automatically 190 | receives an offer from the Licensor to exercise the 191 | Licensed Rights under the terms and conditions of this 192 | Public License. 193 | 194 | b. Additional offer from the Licensor -- Adapted Material. 195 | Every recipient of Adapted Material from You 196 | automatically receives an offer from the Licensor to 197 | exercise the Licensed Rights in the Adapted Material 198 | under the conditions of the Adapter's License You apply. 199 | 200 | c. No downstream restrictions. You may not offer or impose 201 | any additional or different terms or conditions on, or 202 | apply any Effective Technological Measures to, the 203 | Licensed Material if doing so restricts exercise of the 204 | Licensed Rights by any recipient of the Licensed 205 | Material. 206 | 207 | 6. No endorsement. Nothing in this Public License constitutes or 208 | may be construed as permission to assert or imply that You 209 | are, or that Your use of the Licensed Material is, connected 210 | with, or sponsored, endorsed, or granted official status by, 211 | the Licensor or others designated to receive attribution as 212 | provided in Section 3(a)(1)(A)(i). 213 | 214 | b. Other rights. 215 | 216 | 1. Moral rights, such as the right of integrity, are not 217 | licensed under this Public License, nor are publicity, 218 | privacy, and/or other similar personality rights; however, to 219 | the extent possible, the Licensor waives and/or agrees not to 220 | assert any such rights held by the Licensor to the limited 221 | extent necessary to allow You to exercise the Licensed 222 | Rights, but not otherwise. 223 | 224 | 2. Patent and trademark rights are not licensed under this 225 | Public License. 226 | 227 | 3. To the extent possible, the Licensor waives any right to 228 | collect royalties from You for the exercise of the Licensed 229 | Rights, whether directly or through a collecting society 230 | under any voluntary or waivable statutory or compulsory 231 | licensing scheme. In all other cases the Licensor expressly 232 | reserves any right to collect such royalties, including when 233 | the Licensed Material is used other than for NonCommercial 234 | purposes. 235 | 236 | 237 | Section 3 -- License Conditions. 238 | 239 | Your exercise of the Licensed Rights is expressly made subject to the 240 | following conditions. 241 | 242 | a. Attribution. 243 | 244 | 1. If You Share the Licensed Material (including in modified 245 | form), You must: 246 | 247 | a. retain the following if it is supplied by the Licensor 248 | with the Licensed Material: 249 | 250 | i. identification of the creator(s) of the Licensed 251 | Material and any others designated to receive 252 | attribution, in any reasonable manner requested by 253 | the Licensor (including by pseudonym if 254 | designated); 255 | 256 | ii. a copyright notice; 257 | 258 | iii. a notice that refers to this Public License; 259 | 260 | iv. a notice that refers to the disclaimer of 261 | warranties; 262 | 263 | v. a URI or hyperlink to the Licensed Material to the 264 | extent reasonably practicable; 265 | 266 | b. indicate if You modified the Licensed Material and 267 | retain an indication of any previous modifications; and 268 | 269 | c. indicate the Licensed Material is licensed under this 270 | Public License, and include the text of, or the URI or 271 | hyperlink to, this Public License. 272 | 273 | 2. You may satisfy the conditions in Section 3(a)(1) in any 274 | reasonable manner based on the medium, means, and context in 275 | which You Share the Licensed Material. For example, it may be 276 | reasonable to satisfy the conditions by providing a URI or 277 | hyperlink to a resource that includes the required 278 | information. 279 | 3. If requested by the Licensor, You must remove any of the 280 | information required by Section 3(a)(1)(A) to the extent 281 | reasonably practicable. 282 | 283 | b. ShareAlike. 284 | 285 | In addition to the conditions in Section 3(a), if You Share 286 | Adapted Material You produce, the following conditions also apply. 287 | 288 | 1. The Adapter's License You apply must be a Creative Commons 289 | license with the same License Elements, this version or 290 | later, or a BY-NC-SA Compatible License. 291 | 292 | 2. You must include the text of, or the URI or hyperlink to, the 293 | Adapter's License You apply. You may satisfy this condition 294 | in any reasonable manner based on the medium, means, and 295 | context in which You Share Adapted Material. 296 | 297 | 3. You may not offer or impose any additional or different terms 298 | or conditions on, or apply any Effective Technological 299 | Measures to, Adapted Material that restrict exercise of the 300 | rights granted under the Adapter's License You apply. 301 | 302 | 303 | Section 4 -- Sui Generis Database Rights. 304 | 305 | Where the Licensed Rights include Sui Generis Database Rights that 306 | apply to Your use of the Licensed Material: 307 | 308 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right 309 | to extract, reuse, reproduce, and Share all or a substantial 310 | portion of the contents of the database for NonCommercial purposes 311 | only; 312 | 313 | b. if You include all or a substantial portion of the database 314 | contents in a database in which You have Sui Generis Database 315 | Rights, then the database in which You have Sui Generis Database 316 | Rights (but not its individual contents) is Adapted Material, 317 | including for purposes of Section 3(b); and 318 | 319 | c. You must comply with the conditions in Section 3(a) if You Share 320 | all or a substantial portion of the contents of the database. 321 | 322 | For the avoidance of doubt, this Section 4 supplements and does not 323 | replace Your obligations under this Public License where the Licensed 324 | Rights include other Copyright and Similar Rights. 325 | 326 | 327 | Section 5 -- Disclaimer of Warranties and Limitation of Liability. 328 | 329 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE 330 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS 331 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF 332 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, 333 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, 334 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR 335 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, 336 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT 337 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT 338 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. 339 | 340 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE 341 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, 342 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, 343 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, 344 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR 345 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN 346 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR 347 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR 348 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. 349 | 350 | c. The disclaimer of warranties and limitation of liability provided 351 | above shall be interpreted in a manner that, to the extent 352 | possible, most closely approximates an absolute disclaimer and 353 | waiver of all liability. 354 | 355 | 356 | Section 6 -- Term and Termination. 357 | 358 | a. This Public License applies for the term of the Copyright and 359 | Similar Rights licensed here. However, if You fail to comply with 360 | this Public License, then Your rights under this Public License 361 | terminate automatically. 362 | 363 | b. Where Your right to use the Licensed Material has terminated under 364 | Section 6(a), it reinstates: 365 | 366 | 1. automatically as of the date the violation is cured, provided 367 | it is cured within 30 days of Your discovery of the 368 | violation; or 369 | 370 | 2. upon express reinstatement by the Licensor. 371 | 372 | For the avoidance of doubt, this Section 6(b) does not affect any 373 | right the Licensor may have to seek remedies for Your violations 374 | of this Public License. 375 | 376 | c. For the avoidance of doubt, the Licensor may also offer the 377 | Licensed Material under separate terms or conditions or stop 378 | distributing the Licensed Material at any time; however, doing so 379 | will not terminate this Public License. 380 | 381 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public 382 | License. 383 | 384 | 385 | Section 7 -- Other Terms and Conditions. 386 | 387 | a. The Licensor shall not be bound by any additional or different 388 | terms or conditions communicated by You unless expressly agreed. 389 | 390 | b. Any arrangements, understandings, or agreements regarding the 391 | Licensed Material not stated herein are separate from and 392 | independent of the terms and conditions of this Public License. 393 | 394 | 395 | Section 8 -- Interpretation. 396 | 397 | a. For the avoidance of doubt, this Public License does not, and 398 | shall not be interpreted to, reduce, limit, restrict, or impose 399 | conditions on any use of the Licensed Material that could lawfully 400 | be made without permission under this Public License. 401 | 402 | b. To the extent possible, if any provision of this Public License is 403 | deemed unenforceable, it shall be automatically reformed to the 404 | minimum extent necessary to make it enforceable. If the provision 405 | cannot be reformed, it shall be severed from this Public License 406 | without affecting the enforceability of the remaining terms and 407 | conditions. 408 | 409 | c. No term or condition of this Public License will be waived and no 410 | failure to comply consented to unless expressly agreed to by the 411 | Licensor. 412 | 413 | d. Nothing in this Public License constitutes or may be interpreted 414 | as a limitation upon, or waiver of, any privileges and immunities 415 | that apply to the Licensor or You, including from the legal 416 | processes of any jurisdiction or authority. 417 | 418 | ======================================================================= 419 | 420 | Creative Commons is not a party to its public 421 | licenses. Notwithstanding, Creative Commons may elect to apply one of 422 | its public licenses to material it publishes and in those instances 423 | will be considered the “Licensor.” The text of the Creative Commons 424 | public licenses is dedicated to the public domain under the CC0 Public 425 | Domain Dedication. Except for the limited purpose of indicating that 426 | material is shared under a Creative Commons public license or as 427 | otherwise permitted by the Creative Commons policies published at 428 | creativecommons.org/policies, Creative Commons does not authorize the 429 | use of the trademark "Creative Commons" or any other trademark or logo 430 | of Creative Commons without its prior written consent including, 431 | without limitation, in connection with any unauthorized modifications 432 | to any of its public licenses or any other arrangements, 433 | understandings, or agreements concerning use of licensed material. For 434 | the avoidance of doubt, this paragraph does not form part of the 435 | public licenses. 436 | 437 | Creative Commons may be contacted at creativecommons.org. 438 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![lite_tracer Logo](https://github.com/BorealisAI/lite_tracer/blob/master/logos/Litetracer-Logo-Full-LB.png) 2 | 3 | -------------------------------------------------------------------------------- 4 | 5 | # LiteTracer: a light weight experiment reproducibility toolset 6 | 7 | LiteTracer acts as a drop-in replacement for argparse, and it can generate unique identifiers for experiments in addition to what argparse already does. 8 | Along with a reverse lookup tool, LiteTracer can trace-back the state of a project that generated any result tagged by the identifier. 9 | The identifiers are unique based on the combination of four factors: 10 | 1) code version; 11 | 2) un-committed code changes 12 | 3) untracked files in the project; 13 | 3) any command line arguments supplied at runtime. 14 | 15 | As the name suggests, LiteTracer is designed to be as lightweight as possible. It is a minimalistic toolset and convention to enable reproducible experimental research, rather than a framework that one has to learn about. 16 | 17 | ## To track: 18 | 19 | 1) Instead of using argparse `from argparse import ArgumentParser`, use `LTParser`, e.g.: 20 | 21 | ``` 22 | from lite_tracer import LTParser 23 | parser = LTParser("...") 24 | parser.add_argument(...) 25 | args = parser.parse_args() 26 | ``` 27 | 28 | 2) Then in any result file you save (tensorboard results included), include as part of filename: `args.hash_code`, for example: 29 | ``` 30 | result_path = './results/{}/{}'.format(args.data_name, args.hash_code) 31 | ``` 32 | 33 | NEVER manually change output filenames (e.g. use generated filenames directly in your latex source code) 34 | 35 | ## Given hash code, to trace back to the exact configuration that produced a result: 36 | By default, LTParser saves tracking information to ./lt_records/, which has three things: 37 | 38 | `settings_.txt` which has all arguments used for the experiments (command line supplied merged with defaults), 39 | as well as some dynamically collected information such as git version information 40 | 41 | `diff.patch` any source code change from last committed version 42 | 43 | `untracked/` any untracked and not ingored files/folders in the project dir 44 | 45 | ## To find all results with certain param settings: 46 | `lite_trace.py --include [[[PARAM1:VAL1] PARAM2:VAL2] ...] --exclude [[[PARAM1:VAL1] PARAM2:VAL2] ...]` 47 | 48 | ## See a complete example in example/lite_tracer_example.py 49 | Example search: 50 | `lite_trace.py --exclude bsz:12 git_label:f6afeb8 --include sgd` 51 | -------------------------------------------------------------------------------- /example/lite_tracer_example.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-present, Royal Bank of Canada. 2 | # All rights reserved. 3 | # This source code is licensed under the license found in the 4 | # LICENSE file in the root directory of this source tree.# 5 | # Author: Yanshuai Cao 6 | 7 | from lite_tracer import LTParser 8 | import os 9 | pjoin = os.path.join 10 | 11 | if __name__ == '__main__': 12 | parser = LTParser(description="A reproducible experiment") 13 | 14 | parser.add_argument('--data_name', type=str, default='penn', 15 | help='data name') 16 | 17 | parser.add_argument('--optimizer', type=str, default='sgd', 18 | help='optimizer') 19 | 20 | parser.add_argument('--bsz', type=int, default=512, 21 | help='batch size') 22 | 23 | parser.add_argument('--note', type=str, default='', 24 | help='additional_note_str') 25 | 26 | parser.add_argument('--result_folder', type=str, default='./results/', 27 | help='additional_note_str') 28 | 29 | args = parser.parse_args() 30 | 31 | result_path = pjoin(args.result_folder, 32 | '{}_{}'.format(args.data_name, args.hash_code)) 33 | 34 | if not os.path.exists(args.result_folder): 35 | os.makedirs(args.result_folder) 36 | 37 | # simulate experiment and save results 38 | 39 | with open(result_path + '.txt', 'w') as wr: 40 | wr.write('Some amazing results!') 41 | -------------------------------------------------------------------------------- /lite_tracer/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-present, Royal Bank of Canada. 2 | # All rights reserved. 3 | # This source code is licensed under the license found in the 4 | # LICENSE file in the root directory of this source tree.# 5 | # Author: Yanshuai Cao 6 | 7 | from .tracker import LTParser 8 | -------------------------------------------------------------------------------- /lite_tracer/exceptions.py: -------------------------------------------------------------------------------- 1 | class NoHistory(RuntimeError): 2 | """There are no lite tracer run history""" 3 | 4 | 5 | class ShellError(RuntimeError): 6 | """Shell output had errors""" 7 | 8 | 9 | class GitError(RuntimeError): 10 | """Might not be a git repository or git is not installed""" 11 | def __init__(self, message=None, errors=None): 12 | self.message = "git may not be configured properly" 13 | super(GitError, self).__init__(message) 14 | self.errors = errors 15 | 16 | 17 | class NoMatchError(RuntimeError): 18 | """There are no match for the given parameters""" 19 | 20 | 21 | class NoParameterError(RuntimeError): 22 | """There are no match for the given parameters""" 23 | 24 | 25 | class DestArgumentNotSuppported(RuntimeError): 26 | """There are no match for the given parameters""" 27 | def __init__(self, message=None, errors=None): 28 | self.message = "dest argument for argparser is not supported" 29 | super(DestArgumentNotSuppported, self).__init__(message) 30 | self.errors = errors 31 | 32 | 33 | class ArgumentNotParsable(RuntimeError): 34 | """There are no match for the given parameters""" 35 | -------------------------------------------------------------------------------- /lite_tracer/lite_trace.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright (c) 2018-present, Royal Bank of Canada. 3 | # All rights reserved. 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree.# 6 | # Author: Yanshuai Cao 7 | 8 | from __future__ import print_function 9 | import sys 10 | import glob 11 | import os 12 | import argparse 13 | import time 14 | import re 15 | 16 | from collections import defaultdict 17 | 18 | import lite_tracer.exceptions as exception 19 | 20 | 21 | class Parsed(object): 22 | def __init__(self, file_path, line): 23 | self.line = line 24 | 25 | self.file_name = file_path 26 | self.ctime = os.path.getctime(self.file_name) 27 | self.end_str = ".txt" 28 | self.start_str = "settings_" 29 | 30 | hash_str_regex = re.compile('(?<=settings_)LT.*LT(?=.txt)') 31 | self.hash_str = re.search(hash_str_regex, file_path).group(0) 32 | self.line = line 33 | 34 | tmp = [self._param_extraction(x) 35 | for x in self._param_split(self.line)] 36 | self.kwargs = dict([tuple(kv) for kv in tmp]) 37 | 38 | def _param_extraction(self, split_param_str): 39 | split_param_str = self._clean_params(split_param_str) 40 | split = split_param_str.split(' ') 41 | key = split[0].replace('--', '') 42 | values = split[1:] 43 | 44 | return key, values 45 | 46 | # Needed for backward compatibility, fixed version does not need it 47 | # May cause problems with adding text notes 48 | def _clean_params(self, params): 49 | bad_chars = '[],\'' 50 | for char in bad_chars: 51 | params = params.replace(char, '') 52 | 53 | return params 54 | 55 | def _param_split(self, raw_param_str): 56 | param_split = re.compile('[-]{1,2}[a-zA-Z].*?(?= [-]{1,2}[a-zA-Z]|$)') 57 | split_param_strs = re.findall(param_split, raw_param_str) 58 | return split_param_strs 59 | 60 | 61 | class FindDefault(object): 62 | def __init__(self): 63 | self.non_defaults = set() 64 | self.values = dict() 65 | 66 | def add(self, params): 67 | changed = [k for k, v in params.items() 68 | if self.values.setdefault(k, v) != v] 69 | 70 | if changed: 71 | self.non_defaults |= set(changed) 72 | 73 | 74 | def main(): 75 | parser = argparse.ArgumentParser( 76 | description="explore saved experiment results by matching hyperparameter flags") 77 | 78 | parser.add_argument('-d', '--lt_dir', type=str, 79 | default='./lt_records', help="folder containing LT records") 80 | 81 | parser.add_argument('-i', '--include', type=str, nargs='+') 82 | parser.add_argument('-e', '--exclude', type=str, nargs='+') 83 | 84 | args = parser.parse_args() 85 | 86 | setting_files_regex_path = os.path.expanduser(os.path.join(args.lt_dir, "LT*LT/settings*.txt")) 87 | setting_file_list = [f for f in glob.glob(setting_files_regex_path) 88 | if 'searchable' not in f] 89 | 90 | if not setting_file_list: 91 | raise exception.NoHistory() 92 | 93 | include_params = get_param_operator_value(args.include) if args.include else defaultdict(list) 94 | exclude_params = get_param_operator_value(args.exclude) if args.exclude else defaultdict(list) 95 | 96 | if include_params is None and exclude_params is None: 97 | raise exception.NoParameterError() 98 | 99 | search_results = list() 100 | param_default_checker = FindDefault() 101 | 102 | for file_path in setting_file_list: 103 | with open(file_path, 'r') as setting_file: 104 | line = setting_file.readline() 105 | 106 | parsed = Parsed(file_path, line) 107 | parsed_params = parsed.kwargs 108 | param_default_checker.add(parsed_params) 109 | 110 | include_search_result = compare_match(parsed_params, include_params) 111 | include_found = all(include_search_result) and len(include_search_result) > 0 112 | 113 | exclude_search_result = compare_match(parsed_params, exclude_params, partial=True) 114 | exclude_found = any(exclude_search_result) 115 | 116 | # Include results when parameters are not part of the exclusion list AND 117 | # parameters in -i option is found OR -i option is not set 118 | if not exclude_found and (include_found or not include_params): 119 | search_results.append(parsed) 120 | 121 | if not search_results: 122 | raise exception.NoMatchError() 123 | 124 | search_results.sort(key=lambda x: x.ctime) 125 | for result in search_results: 126 | print(format_output(result, param_default_checker.non_defaults)) 127 | 128 | 129 | def format_output(result, non_defaults): 130 | output_format = "{}\t{}\t{}" 131 | kv_format = '{}:{}' 132 | 133 | result_params = result.kwargs 134 | ctime = time.ctime(result.ctime) 135 | key_values = [kv_format.format(k, ','.join(result_params[k])) 136 | for k in sorted(result_params.keys()) 137 | if k in non_defaults] 138 | key_value_str = ' '.join(key_values) 139 | 140 | return output_format.format(result.hash_str, ctime, key_value_str) 141 | 142 | 143 | def get_param_operator_value(input_param_values): 144 | """Returns dictionary{param:[(operator, value), ...], ...}""" 145 | param_operator_value = defaultdict(list) 146 | split_regex = re.compile(r'(^[a-zA-Z]{1}[\w-]*)(<=|>=|==|<|>|:)?(.*)?') 147 | 148 | for p_v in input_param_values: 149 | p_v_matches = re.match(split_regex, p_v) 150 | if p_v_matches is None: 151 | raise exception.ArgumentNotParsable() 152 | 153 | par_op_val = [m for m in p_v_matches.groups() if m] 154 | 155 | param = par_op_val[0] 156 | if len(par_op_val) == 3: 157 | operator_value = (par_op_val[1], par_op_val[2]) 158 | elif len(par_op_val) == 1: 159 | operator_value = ('', '') 160 | else: 161 | raise exception.ArgumentNotParsable() 162 | 163 | param_operator_value[param].append(operator_value) 164 | 165 | return param_operator_value 166 | 167 | 168 | def compare_match(stored_params, search_params, partial=False): 169 | """Compares stored_param against search_params""" 170 | stored_keys = set(stored_params.keys()) 171 | search_keys = set(search_params.keys()) 172 | possible_search_keys = search_keys.intersection(stored_keys) 173 | results_list = list() 174 | 175 | if not partial and possible_search_keys != search_keys: 176 | return results_list 177 | 178 | for search_key in possible_search_keys: 179 | search_pattern = search_params[search_key] 180 | stored_values = stored_params[search_key] 181 | matches = (get_match_results(stored_values, pattern) 182 | for pattern in search_pattern) 183 | if not partial: 184 | results_list.append(all(matches)) 185 | else: 186 | results_list.append(any(matches)) 187 | 188 | return results_list 189 | 190 | 191 | def get_match_results(stored_values, pattern): 192 | """ Return if any of the the stored params is matching the key """ 193 | operator = pattern[0] 194 | raw_value = pattern[1] 195 | numeric = is_numeric(raw_value) 196 | cast = float if numeric else str 197 | 198 | value = cast(raw_value) 199 | 200 | if raw_value and not numeric: 201 | if operator == ':' or operator == '==': 202 | return any(value == v for v in stored_values) 203 | else: 204 | return False 205 | 206 | casted_values = (cast(v) for v in stored_values 207 | if is_numeric(v)) 208 | 209 | if operator == ':' or operator == '==': 210 | return any(value == v for v in casted_values) 211 | elif operator == '<=': 212 | return any(value <= v for v in casted_values) 213 | elif operator == '>=': 214 | return any(value >= v for v in casted_values) 215 | elif operator == '<': 216 | return any(value < v for v in casted_values) 217 | elif operator == '>': 218 | return any(value > v for v in casted_values) 219 | else: 220 | return True 221 | 222 | 223 | def is_numeric(value): 224 | try: 225 | float(value) 226 | return True 227 | except ValueError: 228 | return False 229 | 230 | 231 | if __name__ == '__main__': 232 | try: 233 | main() 234 | except (exception.NoHistory, exception.NoMatchError, exception.NoParameterError) as e: 235 | if isinstance(e, exception.NoHistory): 236 | print("Error: There are no previous runs of this experiment") 237 | elif isinstance(e, exception.NoParameterError): 238 | print("Error: Parameters were not provided properly") 239 | elif isinstance(e, exception.NoMatchError): 240 | print("There were no match for the given Parameters") 241 | 242 | sys.exit(1) 243 | -------------------------------------------------------------------------------- /lite_tracer/tracker.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-present, Royal Bank of Canada. 2 | # All rights reserved. 3 | # This source code is licensed under the license found in the 4 | # LICENSE file in the root directory of this source tree.# 5 | # Author: Yanshuai Cao 6 | 7 | from __future__ import print_function 8 | import hashlib 9 | from argparse import ArgumentParser 10 | import subprocess 11 | import shutil 12 | import os 13 | from os.path import join as pjoin 14 | import re 15 | 16 | import lite_tracer.exceptions as exception 17 | 18 | 19 | class LTParser(ArgumentParser): 20 | """Lite tracer parses arugments and saves them for future tracking 21 | 22 | Attributes: 23 | record_path(str): Directory in which the record will be saved 24 | args_file (str): File path for the record text path 25 | """ 26 | _BASE_HASH_FIELD = 'base_hash_code' 27 | _HASH_FIELD = 'hash_code' 28 | _GIT_FIELD = 'git_label' 29 | _HASH_FORMAT = 'LT_delta-{}_base-{}_LT' 30 | 31 | def __init__(self, **kwargs): 32 | self._lt_record_dir = kwargs.pop('record_dir', 'lt_records') 33 | self._on_suspicion = kwargs.pop('on_suspicion', 'warn') 34 | self._short_hash = kwargs.pop('short_hash', True) 35 | 36 | self.record_path = '' 37 | self.args_file = '' 38 | 39 | self._flag_params = list() 40 | self._single_params = list() 41 | 42 | if not os.path.exists(self._lt_record_dir): 43 | os.makedirs(self._lt_record_dir) 44 | 45 | super(LTParser, self).__init__(**kwargs) 46 | 47 | def parse_args(self, args=None, namespace=None): 48 | args = super(LTParser, self).parse_args(args, namespace) 49 | 50 | try: 51 | git_label = self._shell_output(["git", "describe", "--always"]) 52 | except RuntimeError: 53 | raise exception.GitError() 54 | 55 | hash_code = self._args_to_hash(args, short=self._short_hash) 56 | setattr(args, self._GIT_FIELD, git_label) 57 | setattr(args, self._BASE_HASH_FIELD, hash_code) 58 | 59 | args = self._handle_unclean(args) 60 | 61 | setting_fname = pjoin(self.record_path, 62 | 'settings_{}'.format(args.hash_code)) 63 | self.args_file = setting_fname + '.txt' 64 | 65 | with open(self.args_file, 'w') as write_file: 66 | write_file.write(self._args_to_str(args)) 67 | 68 | return args 69 | 70 | def add_argument(self, *args, **kwargs): 71 | if len(args) == 1 and args[0].count('-') == 1: 72 | arguments = [s.strip('-') for s in args] 73 | self._single_params.extend(arguments) 74 | if 'dest' in kwargs: 75 | raise exception.DestArgumentNotSuppported() 76 | if 'action' in kwargs: 77 | action = kwargs['action'] 78 | if action == 'store_true' or action == 'store_false': 79 | arguments = [s.strip('-') for s in args] 80 | self._flag_params.extend(arguments) 81 | 82 | super(LTParser, self).add_argument(*args, **kwargs) 83 | 84 | def _handle_unclean(self, args): 85 | unclean_hash = hashlib.md5() 86 | base_hash = getattr(args, self._BASE_HASH_FIELD) 87 | 88 | # Update the hash 89 | git_diff = self._update_diff_hash(unclean_hash) 90 | untracked_files = self._update_untracked_hash(unclean_hash) 91 | 92 | unclean_hash_str = self._hash_to_str(unclean_hash) 93 | hash_text = self._HASH_FORMAT.format(unclean_hash_str, base_hash) 94 | setattr(args, self._HASH_FIELD, hash_text) 95 | 96 | # Check if Directories exist and error according to preference 97 | self.record_path = pjoin(self._lt_record_dir, hash_text) 98 | 99 | # TODO: Default is to create another directory with timestamp 100 | if os.path.exists(self.record_path): 101 | msg = "Experiment {} already exists.".format(hash_text) 102 | if self._on_suspicion == 'warn': 103 | import warnings 104 | warnings.warn(msg + " Overwriting previous record now.") 105 | elif self._on_suspicion == 'error': 106 | raise ValueError(msg) 107 | elif self._on_suspicion == 'ignore': 108 | pass 109 | else: 110 | raise ValueError('on_suspicion needs to be [warn/error/ignore]') 111 | else: 112 | os.makedirs(self.record_path) 113 | 114 | # Save the diff and the untracked_files in to a directory 115 | with open(pjoin(self.record_path, 'diff.patch'), 'w') as git_diff_file: 116 | git_diff_file.write(git_diff) 117 | 118 | self._save_untracked(untracked_files) 119 | 120 | return args 121 | 122 | def _args_to_str(self, args_parse_obj, filter_keys=None): 123 | if filter_keys is None: 124 | filter_keys = [self._HASH_FIELD, self._BASE_HASH_FIELD, 'record_path'] 125 | 126 | cmd_items = [(k, v) for k, v in vars(args_parse_obj).items() 127 | if k not in filter_keys] 128 | 129 | sorted_cmd_items = sorted(cmd_items, key=lambda x: x[0]) 130 | 131 | return ' '.join(self._cmd_to_str(sorted_cmd_items)) 132 | 133 | def _cmd_to_str(self, cmd_items): 134 | cmd_str = list() 135 | 136 | def format_cmd_str(key, values, str_format='--{} {}'): 137 | if isinstance(values, list): 138 | arg_list = ' '.join([str(s) for s in values]) 139 | return str_format.format(key, arg_list) 140 | else: 141 | return str_format.format(key, values) 142 | 143 | for key, values in cmd_items: 144 | is_flag_param = key in self._flag_params 145 | is_single_param = key in self._single_params 146 | 147 | if is_flag_param: 148 | prefix = '-' if is_single_param else '--' 149 | cmd_str.append(prefix + key) 150 | elif is_single_param: 151 | cmd_str.append(format_cmd_str(key, values, str_format='-{} {}')) 152 | else: 153 | cmd_str.append(format_cmd_str(key, values)) 154 | 155 | return cmd_str 156 | 157 | def _update_diff_hash(self, md5_hash): 158 | try: 159 | git_diff = self._shell_output(['git', 'diff']) 160 | except RuntimeError: 161 | raise exception.GitError() 162 | 163 | md5_hash.update(git_diff.encode('utf-8')) 164 | 165 | return git_diff 166 | 167 | def _update_untracked_hash(self, md5_hash): 168 | untracked_files = self._find_untracked() 169 | files, folders = self._sort_files_folders(untracked_files) 170 | 171 | if folders: 172 | self._folder_error_msg(folders) 173 | 174 | untracked_content = self._read_untracked_files(files) 175 | md5_hash.update(''.join(untracked_content).encode('utf-8')) 176 | 177 | # find self._lt_record_dir and remove any content underneath 178 | return untracked_files 179 | 180 | def _find_untracked(self): 181 | try: 182 | git_untracked = self._shell_output(["git", "status", "-s"]) 183 | except RuntimeError: 184 | raise exception.GitError() 185 | 186 | escaped_dir = re.escape(self._lt_record_dir + '/') 187 | regex_string = '(?<=\?\? )(?!\.|{}).*'.format(escaped_dir) 188 | untracked_regex = re.compile(regex_string) 189 | untracked_files = re.findall(untracked_regex, git_untracked) 190 | 191 | return untracked_files 192 | 193 | @staticmethod 194 | def _sort_files_folders(paths): 195 | folders = {p for p in paths 196 | if os.path.isdir(p)} 197 | files = set(paths) - folders 198 | 199 | return files, folders 200 | 201 | @staticmethod 202 | def _read_untracked_files(files): 203 | content = list() 204 | for path in files: 205 | with open(path, 'r') as file_handler: 206 | content.append(file_handler.read()) 207 | 208 | return content 209 | 210 | def _folder_error_msg(self, folders): 211 | folder_str = ', '.join(folders) 212 | msg = ("{} are folders not checked in. " 213 | "Consider adding it to .gitignore or git add".format(folder_str)) 214 | 215 | if self._on_suspicion == 'warn': 216 | import warnings 217 | warnings.warn(msg + " Will backup the folder for now.") 218 | elif self._on_suspicion == 'error': 219 | raise ValueError(msg) 220 | elif self._on_suspicion == 'ignore': 221 | pass 222 | else: 223 | raise ValueError('on_suspicion needs to be [warn/error/ignore]') 224 | 225 | def _save_untracked(self, untracked_files, save_folder='untracked'): 226 | untracked_save_path = pjoin(self.record_path, save_folder) 227 | 228 | if not os.path.exists(untracked_save_path): 229 | os.makedirs(untracked_save_path) 230 | 231 | for u_file in untracked_files: 232 | if os.path.isdir(u_file): 233 | dst_path = pjoin(untracked_save_path, 234 | os.path.basename(u_file.rstrip(os.path.sep))) 235 | if os.path.exists(dst_path): 236 | shutil.rmtree(dst_path) 237 | 238 | shutil.copytree(u_file, dst_path) 239 | else: 240 | shutil.copy(u_file, untracked_save_path) 241 | 242 | def _args_to_hash(self, args_parse_obj, short=True): 243 | md5_hash = hashlib.md5() 244 | args_str = self._args_to_str(args_parse_obj) 245 | md5_hash.update(args_str.encode('utf-8')) 246 | 247 | return self._hash_to_str(md5_hash, short) 248 | 249 | @staticmethod 250 | def _hash_to_str(md5_hash, short=True): 251 | if not short: 252 | hash_code = md5_hash.hexdigest() 253 | else: 254 | from zlib import adler32 255 | hash_code = hex(adler32(md5_hash.digest())) 256 | 257 | return hash_code 258 | 259 | @staticmethod 260 | def _shell_output(cmd): 261 | try: 262 | newline_regex = re.compile("[\n\r]$") 263 | output = subprocess.check_output(cmd).decode('utf-8') 264 | return re.sub(newline_regex, '', output) 265 | 266 | except subprocess.CalledProcessError: 267 | raise RuntimeError("Error in the process that was called") 268 | -------------------------------------------------------------------------------- /logos/Litetracer-Logo-Full-DB.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BorealisAI/lite_tracer/f2c13756dad981fb130cb4fe895284b5e52faddc/logos/Litetracer-Logo-Full-DB.png -------------------------------------------------------------------------------- /logos/Litetracer-Logo-Full-LB.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BorealisAI/lite_tracer/f2c13756dad981fb130cb4fe895284b5e52faddc/logos/Litetracer-Logo-Full-LB.png -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | norecursedirs = lt_records 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018-present, Royal Bank of Canada. 2 | # All rights reserved. 3 | # This source code is licensed under the license found in the 4 | # LICENSE file in the root directory of this source tree.# 5 | # Author: Yanshuai Cao 6 | 7 | from setuptools import setup 8 | 9 | setup(name='lite_tracer', 10 | version='0.1', 11 | description='A lightweight experiment reproducibility toolset', 12 | url='', 13 | author='Yanshuai Cao', 14 | author_email='yanshuai.cao@borealisai.com', 15 | license='GNU LGPL', 16 | packages=['lite_tracer'], 17 | scripts=['lite_tracer/lite_trace.py'], 18 | zip_safe=False) 19 | -------------------------------------------------------------------------------- /tests/helper.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import random 3 | import subprocess 4 | 5 | import pytest 6 | 7 | from lite_tracer import LTParser 8 | 9 | 10 | @pytest.fixture 11 | def cleandir(): 12 | resboundts_path = './lt_records' 13 | shutil.rmtree(resboundts_path) 14 | 15 | 16 | def get_tracer(): 17 | parser = LTParser(description="A reproducible experiment") 18 | 19 | parser.add_argument('--optimizer', type=str, default='a,b,c,d', 20 | help='optimizer') 21 | 22 | parser.add_argument('--device', type=str, default='cuda:0', 23 | help='device') 24 | parser.add_argument('--integer', type=int, default=1, 25 | help='integer testing') 26 | parser.add_argument('--float', type=float, default=1.0, 27 | help='float testing') 28 | parser.add_argument('-d', '--double', default='1.0', type=str, 29 | help='double option testing') 30 | 31 | return parser 32 | 33 | 34 | def add_lists_option(parser): 35 | parser.add_argument('-l', '--list', nargs='+', default='', 36 | help='list testing') 37 | parser.add_argument('-n', '--nlist', nargs='+', default='', type=int, 38 | help='list testing') 39 | parser.add_argument('-f', '--flist', nargs='+', default='', type=float, 40 | help='list testing') 41 | 42 | 43 | def add_notes_option(parser): 44 | parser.add_argument('--notes', default=" Ground Breaking Research ", 45 | help='single option testing') 46 | 47 | 48 | def add_single_option(parser): 49 | parser.add_argument('-a', default=1.0, 50 | help='single option testing') 51 | 52 | 53 | def add_single_double_option(parser): 54 | parser.add_argument('-s', '--single', default=1.0, 55 | help='single option testing') 56 | parser.add_argument('-t', '--triple', default=1.0, 57 | help='single option testing') 58 | 59 | 60 | def add_boolean_option(parser): 61 | parser.add_argument('-b', default=False, action='store_true', 62 | help='testing boolean') 63 | parser.add_argument('--boolean', default=False, action='store_true', 64 | help='testing boolean') 65 | 66 | 67 | def add_different_destination(parser): 68 | parser.add_argument('--diff_var', dest='dvar', type=float, default=1.0, 69 | help='diff destination testing') 70 | parser.add_argument('-z', dest='dvar_short', type=float, default=1.0, 71 | help='diff destination testing') 72 | parser.add_argument('-r', '--diff_var_both', dest='dvar_both', type=float, default=1.0, 73 | help='diff destination testing') 74 | parser.add_argument('-q', '--diff_var_long', dest='dvar_long', type=float, default=1.0, 75 | help='diff destination testing') 76 | 77 | 78 | def generate_sysv(start_number, list_params=True): 79 | def stringify(rlist): 80 | return [str(s) for s in rlist] 81 | 82 | def generate_list_elements(start_number, num_elements=2): 83 | return [num_elements * start_number * i 84 | for i in range(1, 1 + num_elements)] 85 | 86 | integer_value = start_number 87 | float_value = start_number + 0.01 88 | cuda_value = start_number 89 | 90 | arg_str = ['--optimizer', 'a,b,c,d'] 91 | arg_str += ['--integer', str(integer_value), '--float', str(float_value), 92 | '--device', 'cuda:' + str(cuda_value)] 93 | 94 | if list_params: 95 | slist = stringify(generate_list_elements(start_number)) 96 | nlist = stringify(generate_list_elements(start_number)) 97 | flist = [str(i + 0.01) for i in generate_list_elements(start_number)] 98 | 99 | arg_str += ['--list'] 100 | arg_str += slist 101 | arg_str += ['--nlist'] 102 | arg_str += nlist 103 | arg_str += ['--flist'] 104 | arg_str += flist 105 | 106 | return arg_str 107 | 108 | 109 | def generate_default_sysv(): 110 | return ['--optimizer', 'a,b,c,d', 111 | '--list', '-1', '-2', '3', '4', 112 | '--nlist', '-1', '-2', '3', '4', 113 | '--flist', '-0.1', '0.1', '-3.0', '4.0'] 114 | 115 | 116 | def assert_arguments(args): 117 | assert args.optimizer == 'a,b,c,d' 118 | 119 | assert args.integer == 1 120 | assert args.float == 1.0 121 | assert args.double == '1.0' 122 | assert args.device == 'cuda:0' 123 | 124 | 125 | def assert_lists(args): 126 | assert args.list == ['-1', '-2', '3', '4'] 127 | assert args.nlist == [-1, -2, 3, 4] 128 | assert args.flist == [-0.1, 0.1, -3.0, 4.0] 129 | 130 | 131 | def get_cmd_output(cmd): 132 | try: 133 | stdout = subprocess.check_output(cmd, shell=True) 134 | output = stdout.decode('utf-8').replace('\t', ' ') 135 | return output.split('\n')[:-1] 136 | 137 | except subprocess.CalledProcessError: 138 | raise RuntimeError("Error in the process that was called") 139 | 140 | 141 | def read_argument(args_file): 142 | with open(args_file, 'r') as f: 143 | argument_str = f.readline() 144 | return argument_str.split(' ') 145 | 146 | 147 | def pick_values(sysv_dict, param_name, n_values): 148 | values = list() 149 | for path, param in sysv_dict.items(): 150 | search_dict = vars(param) 151 | if param_name in search_dict: 152 | if isinstance(search_dict[param_name], list): 153 | values.extend(search_dict[param_name]) 154 | else: 155 | values.append(search_dict[param_name]) 156 | 157 | random.shuffle(values) 158 | n = n_values if n_values <= len(values) else len(values) 159 | 160 | return values[:n] 161 | 162 | 163 | def create_args_list(args_dict): 164 | return {(str(k), str(v)) 165 | for k, l in args_dict.items() 166 | for v in l} 167 | 168 | 169 | def generate_dne_args(dne_args, cast, upper_limit=100): 170 | if upper_limit: 171 | return {(str(dne), str(cast(random.random() * upper_limit))) 172 | for dne in dne_args} 173 | else: 174 | return {(str(dne), None) 175 | for dne in dne_args} 176 | 177 | 178 | def search(sysv_dict, param_name, value): 179 | found = list() 180 | 181 | for path, param in sysv_dict.items(): 182 | search_dict = vars(param) 183 | if param_name in search_dict: 184 | if isinstance(search_dict[param_name], list): 185 | stored = search_dict[param_name] 186 | else: 187 | stored = [search_dict[param_name]] 188 | 189 | results = [(path, param, value) for s in stored 190 | if value == str(s) or value is None] 191 | found.extend(results) 192 | 193 | return found 194 | -------------------------------------------------------------------------------- /tests/test_dest.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import argparse 3 | import random 4 | import subprocess 5 | import itertools 6 | import pdb 7 | 8 | import pytest 9 | 10 | from lite_tracer import LTParser 11 | 12 | import helper 13 | from helper import cleandir 14 | 15 | 16 | def tracer(): 17 | tracer = helper.get_tracer() 18 | helper.add_lists_option(tracer) 19 | helper.add_boolean_option(tracer) 20 | 21 | return tracer 22 | 23 | 24 | def generate_different_destination(): 25 | return ['--diff_var', '1.0', 26 | '-z', '1.0', 27 | '-r', '1.0', 28 | '--diff_var_long', '1.0'] 29 | 30 | 31 | def assert_different_destination(args): 32 | assert args.dvar == 1.0 33 | assert args.dvar_short == 1.0 34 | assert args.dvar_both == 1.0 35 | assert args.dvar_long == 1.0 36 | 37 | 38 | @pytest.mark.usefixtures("cleandir") 39 | def test_dest(): 40 | parser = tracer() 41 | with pytest.raises(RuntimeError): 42 | helper.add_different_destination(parser) 43 | -------------------------------------------------------------------------------- /tests/test_flags.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import stat 3 | import shutil 4 | import subprocess 5 | import random 6 | import pdb 7 | 8 | from lite_tracer import LTParser 9 | import helper 10 | from helper import cleandir 11 | 12 | 13 | @pytest.fixture 14 | def script_setup(tmpdir): 15 | script = tmpdir.join('script.py') 16 | script.open('w').write( 17 | """#!/usr/bin/env python 18 | 19 | from lite_tracer import LTParser 20 | 21 | parser = LTParser(description="A reproducible experiment") 22 | 23 | parser.add_argument('--device', type=str, default='cuda:0', 24 | help='device') 25 | parser.add_argument('--integer', type=int, default=1, 26 | help='integer testing') 27 | parser.add_argument('--float', type=float, default=1.0, 28 | help='float testing') 29 | parser.add_argument('-d', '--double', default='1.0', type=str, 30 | help='double option testing') 31 | 32 | parser.add_argument('-b', default=False, action='store_true', 33 | help='testing boolean') 34 | parser.add_argument('--boolean', default=False, action='store_true', 35 | help='testing boolean') 36 | 37 | parser.add_argument('-a', default=1.0, 38 | help='single option testing') 39 | 40 | args = parser.parse_args() 41 | print(args) 42 | """) 43 | script.chmod(stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) 44 | 45 | return script 46 | 47 | 48 | def generate_boolean(): 49 | return ['-b', '--boolean'] 50 | 51 | 52 | def generate_single(): 53 | return ['-a', '1.0'] 54 | 55 | 56 | def generate_single_double(): 57 | return ['-s', '1.0', '-t', '1000'] 58 | 59 | 60 | def assert_boolean(args): 61 | assert args.b 62 | assert args.boolean 63 | 64 | 65 | def tracer(): 66 | tracer = helper.get_tracer() 67 | helper.add_lists_option(tracer) 68 | helper.add_boolean_option(tracer) 69 | 70 | return tracer 71 | 72 | 73 | def generate_base_args(): 74 | sysv = helper.generate_default_sysv() 75 | sysv += generate_boolean() 76 | 77 | return sysv 78 | 79 | 80 | @pytest.mark.usefixtures("cleandir") 81 | # Save the arguments to the tracer_file 82 | # Import the saved tracer_file and parse it again 83 | def test_boolean(): 84 | parser = tracer() 85 | sysv = generate_base_args() 86 | parser.parse_args(sysv) 87 | read_args = helper.read_argument(parser.args_file) 88 | 89 | # create new tracer for read 90 | new_tracer = tracer() 91 | args, unknown = new_tracer.parse_known_args(read_args) 92 | 93 | assert len(unknown) == 2 and unknown[0] == '--git_label' 94 | 95 | helper.assert_arguments(args) 96 | helper.assert_lists(args) 97 | assert_boolean(args) 98 | 99 | 100 | @pytest.mark.usefixtures("cleandir") 101 | # Save the arguments to the tracer_file 102 | # Import the saved tracer_file and parse it again 103 | def test_single_argument(): 104 | parser = tracer() 105 | helper.add_single_option(parser) 106 | sysv = generate_base_args() 107 | sysv += generate_single() 108 | parser.parse_args(sysv) 109 | read_args = helper.read_argument(parser.args_file) 110 | 111 | # create new tracer for read 112 | new_tracer = tracer() 113 | helper.add_single_option(new_tracer) 114 | 115 | args, unknown = new_tracer.parse_known_args(read_args) 116 | 117 | assert len(unknown) == 2 and unknown[0] == '--git_label' 118 | 119 | helper.assert_arguments(args) 120 | helper.assert_lists(args) 121 | assert_boolean(args) 122 | 123 | 124 | @pytest.mark.usefixtures("cleandir") 125 | def test_single_double_argument(): 126 | parser = tracer() 127 | helper.add_single_double_option(parser) 128 | sysv = generate_base_args() 129 | sysv += generate_single_double() 130 | parser.parse_args(sysv) 131 | read_args = helper.read_argument(parser.args_file) 132 | 133 | # create new tracer for read 134 | new_tracer = tracer() 135 | helper.add_single_double_option(new_tracer) 136 | 137 | args, unknown = new_tracer.parse_known_args(read_args) 138 | 139 | assert len(unknown) == 2 and unknown[0] == '--git_label' 140 | 141 | helper.assert_arguments(args) 142 | helper.assert_lists(args) 143 | assert_boolean(args) 144 | -------------------------------------------------------------------------------- /tests/test_lite_tracer.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from lite_tracer import LTParser 4 | 5 | import pdb 6 | import helper 7 | from helper import cleandir 8 | 9 | 10 | @pytest.fixture 11 | def tracer(): 12 | parser = LTParser(description="A reproducible experiment") 13 | 14 | parser.add_argument('--data_name', type=str, default='penn', 15 | help='data name') 16 | 17 | parser.add_argument('--optimizer', type=str, default='sgd', 18 | help='optimizer') 19 | 20 | parser.add_argument('--bsz', type=int, default=512, 21 | help='batch size') 22 | 23 | parser.add_argument('--note', type=str, default='', 24 | help='additional_note_str') 25 | 26 | parser.add_argument('--result_folder', type=str, default='./results/', 27 | help='additional_note_str') 28 | 29 | return parser 30 | 31 | 32 | def test_simple_test(tracer): 33 | args, _ = tracer.parse_known_args(None) 34 | 35 | assert args.data_name == 'penn' 36 | assert args.optimizer == 'sgd' 37 | assert args.bsz == int(512) 38 | assert args.note == '' 39 | assert args.result_folder == './results/' 40 | 41 | 42 | @pytest.mark.usefixtures("cleandir") 43 | def test_arg_list(): 44 | tracer = helper.get_tracer() 45 | args, left_over = tracer.parse_known_args() 46 | assert left_over 47 | 48 | helper.assert_arguments(args) 49 | 50 | 51 | @pytest.mark.usefixtures("cleandir") 52 | def test_arguments_file(): 53 | # construct tracer and save results 54 | tracer = helper.get_tracer() 55 | helper.add_lists_option(tracer) 56 | 57 | sysv = helper.generate_default_sysv() 58 | tracer.parse_args(sysv) 59 | arguments = helper.read_argument(tracer.args_file) 60 | 61 | # Start a new tracer and read the arguments from file 62 | new_tracer = helper.get_tracer() 63 | helper.add_lists_option(new_tracer) 64 | args, unknown = new_tracer.parse_known_args(arguments) 65 | 66 | helper.assert_arguments(args) 67 | helper.assert_lists(args) 68 | assert len(unknown) == 2 and unknown[0] == '--git_label' 69 | -------------------------------------------------------------------------------- /tests/test_search.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import pytest 3 | 4 | import helper 5 | from helper import cleandir 6 | 7 | INCLUDE_CMD = "lite_trace.py -i {}" 8 | EXCLUDE_CMD = "lite_trace.py -e {}" 9 | IN_EX_CMD = "lite_trace.py -i {} -e {}" 10 | 11 | 12 | @pytest.mark.usefixtures("cleandir") 13 | # Save the arguments to the tracer settings file 14 | # Search the files 15 | def test_search_parameter(): 16 | n_history = 20 17 | 18 | # Generate a set of results with half with list options 19 | for i in range(n_history): 20 | tracer = helper.get_tracer() 21 | if i % 2 == 0: 22 | helper.add_lists_option(tracer) 23 | sysv = helper.generate_sysv(i) 24 | else: 25 | sysv = helper.generate_sysv(i, False) 26 | tracer.parse_args(sysv) 27 | 28 | list_args = ['list', 'flist'] 29 | single_args = ['integer', 'float', 'device'] 30 | dne_args = ['lists'] 31 | 32 | all_args = list_args + single_args + dne_args 33 | 34 | # Include single arguments 35 | for arg in all_args: 36 | cmd = INCLUDE_CMD.format(arg) 37 | if arg in list_args: 38 | output = helper.get_cmd_output(cmd) 39 | assert len(output) == int(n_history / 2) 40 | elif arg in single_args: 41 | output = helper.get_cmd_output(cmd) 42 | assert len(output) == n_history 43 | else: 44 | with pytest.raises(RuntimeError): 45 | output = helper.get_cmd_output(cmd) 46 | 47 | # Exclude single arguments 48 | for arg in all_args: 49 | cmd = EXCLUDE_CMD.format(arg) 50 | if arg in list_args: 51 | output = helper.get_cmd_output(cmd) 52 | assert len(output) == int(n_history / 2) 53 | elif arg in single_args: 54 | with pytest.raises(RuntimeError): 55 | output = helper.get_cmd_output(cmd) 56 | else: 57 | output = helper.get_cmd_output(cmd) 58 | assert len(output) == n_history 59 | 60 | # Include multiple arguments 61 | for arg1, arg2 in itertools.product(all_args, repeat=2): 62 | arg_str = '{} {}'.format(arg1, arg2) 63 | cmd = INCLUDE_CMD.format(arg_str) 64 | 65 | if arg1 in dne_args or arg2 in dne_args: 66 | with pytest.raises(RuntimeError): 67 | output = helper.get_cmd_output(cmd) 68 | elif arg1 in list_args or arg2 in list_args: 69 | output = helper.get_cmd_output(cmd) 70 | assert len(output) == int(n_history / 2) 71 | else: 72 | output = helper.get_cmd_output(cmd) 73 | assert len(output) == n_history 74 | 75 | # Exclude multiple arguments 76 | for arg1, arg2 in itertools.product(all_args, repeat=2): 77 | arg_str = '{} {}'.format(arg1, arg2) 78 | cmd = EXCLUDE_CMD.format(arg_str) 79 | 80 | if arg1 in single_args or arg2 in single_args: 81 | with pytest.raises(RuntimeError): 82 | output = helper.get_cmd_output(cmd) 83 | elif arg1 in list_args or arg2 in list_args: 84 | output = helper.get_cmd_output(cmd) 85 | assert len(output) == int(n_history / 2) 86 | else: 87 | output = helper.get_cmd_output(cmd) 88 | assert len(output) == n_history 89 | 90 | # Include and Exclude tests 91 | for arg1, arg2 in itertools.product(all_args, repeat=2): 92 | cmd = IN_EX_CMD.format(arg1, arg2) 93 | 94 | if arg1 in dne_args: 95 | with pytest.raises(RuntimeError): 96 | output = helper.get_cmd_output(cmd) 97 | elif arg2 in single_args: 98 | with pytest.raises(RuntimeError): 99 | output = helper.get_cmd_output(cmd) 100 | elif arg1 in single_args and arg2 in list_args: 101 | output = helper.get_cmd_output(cmd) 102 | assert len(output) == (n_history / 2) 103 | elif arg1 in single_args: 104 | output = helper.get_cmd_output(cmd) 105 | assert len(output) == n_history 106 | elif arg1 in list_args and arg2 in dne_args: 107 | output = helper.get_cmd_output(cmd) 108 | assert len(output) == (n_history / 2) 109 | elif arg1 in list_args and arg2 in dne_args: 110 | output = helper.get_cmd_output(cmd) 111 | assert len(output) == int(n_history / 2) 112 | else: 113 | with pytest.raises(RuntimeError): 114 | output = helper.get_cmd_output(cmd) 115 | 116 | 117 | @pytest.mark.usefixtures("cleandir") 118 | def test_search_parameter_value(): 119 | n_history = 20 120 | 121 | sysv_dict = dict() 122 | for i in range(0, n_history * 2, 2): 123 | tracer = helper.get_tracer() 124 | 125 | # half the experiments have list options 126 | if i % 2 == 0: 127 | helper.add_lists_option(tracer) 128 | sysv = helper.generate_sysv(i) 129 | else: 130 | sysv = helper.generate_sysv(i, False) 131 | 132 | args = tracer.parse_args(sysv) 133 | args_file = tracer.args_file 134 | sysv_dict[args_file] = args 135 | 136 | list_args = ['list', 'flist'] 137 | single_args = ['integer', 'float', 'device'] 138 | dne_list_args = ['lists'] 139 | 140 | include_cmd = "lite_trace.py -i {}" 141 | exclude_cmd = "lite_trace.py -e {}" 142 | in_ex_cmd = "lite_trace.py -i {} -e {}" 143 | 144 | num_test = 2 145 | args_dict = {arg: helper.pick_values(sysv_dict, arg, num_test) 146 | for arg in list_args + single_args} 147 | exist_search = helper.create_args_list(args_dict) 148 | 149 | dne_search = helper.generate_dne_args(dne_list_args, cast=float) 150 | dne_search |= helper.generate_dne_args(list_args, cast=float) 151 | 152 | all_search = exist_search | dne_search 153 | 154 | # Include single arguments 155 | for param, value in all_search: 156 | cmd = include_cmd.format(param + ':' + value) 157 | n_results = len(helper.search(sysv_dict, param, value)) 158 | 159 | if n_results > 0: 160 | output = helper.get_cmd_output(cmd) 161 | assert len(output) == n_results 162 | else: 163 | if (param, value) in exist_search: 164 | raise RuntimeError 165 | with pytest.raises(RuntimeError): 166 | output = helper.get_cmd_output(cmd) 167 | 168 | # Exclude single arguments 169 | for param, value in all_search: 170 | cmd = exclude_cmd.format(param + ':' + value) 171 | n_results = len(helper.search(sysv_dict, param, value)) 172 | 173 | if n_results < n_history: 174 | output = helper.get_cmd_output(cmd) 175 | assert n_history - len(output) == n_results 176 | else: 177 | with pytest.raises(RuntimeError): 178 | output = helper.get_cmd_output(cmd) 179 | 180 | # Include multiple arguments 181 | 182 | dual_search = list(itertools.product(all_search, repeat=2)) 183 | 184 | for arg1, arg2 in dual_search: 185 | arg_str = '{} {}'.format(':'.join(arg1), ':'.join(arg2)) 186 | cmd = include_cmd.format(arg_str) 187 | 188 | results_1 = {s[0] for s in helper.search(sysv_dict, arg1[0], arg1[1])} 189 | results_2 = {s[0] for s in helper.search(sysv_dict, arg2[0], arg2[1])} 190 | 191 | n_results = len(results_1 & results_2) 192 | 193 | if arg1 in dne_search or arg2 in dne_search: 194 | with pytest.raises(RuntimeError): 195 | output = helper.get_cmd_output(cmd) 196 | elif n_results > 0: 197 | output = helper.get_cmd_output(cmd) 198 | assert len(output) == n_results 199 | else: 200 | with pytest.raises(RuntimeError): 201 | output = helper.get_cmd_output(cmd) 202 | 203 | # Exclude multiple arguments 204 | for arg1, arg2 in dual_search: 205 | arg_str = '{} {}'.format(':'.join(arg1), ':'.join(arg2)) 206 | cmd = exclude_cmd.format(arg_str) 207 | 208 | results_1 = {s[0] for s in helper.search(sysv_dict, arg1[0], arg1[1])} 209 | results_2 = {s[0] for s in helper.search(sysv_dict, arg2[0], arg2[1])} 210 | 211 | n_results = len(results_1 | results_2) 212 | 213 | if n_results < n_history: 214 | output = helper.get_cmd_output(cmd) 215 | assert n_history - len(output) == n_results 216 | else: 217 | with pytest.raises(RuntimeError): 218 | output = helper.get_cmd_output(cmd) 219 | 220 | # Include and Exclude tests 221 | for arg1, arg2 in dual_search: 222 | cmd = in_ex_cmd.format(':'.join(arg1), ':'.join(arg2)) 223 | 224 | full_files = set(sysv_dict.keys()) 225 | results_1 = {s[0] for s in helper.search(sysv_dict, arg1[0], arg1[1])} 226 | results_2 = {s[0] for s in helper.search(sysv_dict, arg2[0], arg2[1])} 227 | 228 | include_result = results_1 229 | exclude_result = full_files - results_2 230 | 231 | full_result = include_result & exclude_result 232 | n_full_result = len(full_result) 233 | 234 | if arg1 in dne_search: 235 | with pytest.raises(RuntimeError): 236 | output = helper.get_cmd_output(cmd) 237 | elif n_full_result > 0: 238 | output = helper.get_cmd_output(cmd) 239 | assert len(output) == n_full_result 240 | else: 241 | with pytest.raises(RuntimeError): 242 | output = helper.get_cmd_output(cmd) 243 | 244 | 245 | @pytest.mark.usefixtures("cleandir") 246 | def test_search_parameter_and_value(): 247 | n_history = 20 248 | 249 | sysv_dict = dict() 250 | for i in range(0, n_history * 2, 2): 251 | tracer = helper.get_tracer() 252 | 253 | # half the experiments have list options 254 | if i % 2 == 0: 255 | helper.add_lists_option(tracer) 256 | sysv = helper.generate_sysv(i) 257 | else: 258 | sysv = helper.generate_sysv(i, False) 259 | 260 | args = tracer.parse_args(sysv) 261 | args_file = tracer.args_file 262 | sysv_dict[args_file] = args 263 | 264 | list_args = ['list', 'flist'] 265 | single_args = ['integer', 'float', 'device'] 266 | dne_list_args = ['lists'] 267 | arg_wo_value = ['list', 'integer'] 268 | 269 | include_cmd = "lite_trace.py -i {}" 270 | exclude_cmd = "lite_trace.py -e {}" 271 | in_ex_cmd = "lite_trace.py -i {} -e {}" 272 | 273 | num_test = 1 274 | args_dict = {arg: helper.pick_values(sysv_dict, arg, num_test) 275 | for arg in list_args + single_args} 276 | exist_search = helper.create_args_list(args_dict) 277 | 278 | dne_search = helper.generate_dne_args(dne_list_args, cast=float) 279 | dne_search |= helper.generate_dne_args(list_args, cast=float) 280 | 281 | arg_wo_search = helper.generate_dne_args(arg_wo_value, int, upper_limit=None) 282 | 283 | all_search = exist_search | dne_search | arg_wo_search 284 | 285 | dual_search = list(itertools.product(all_search, repeat=2)) 286 | 287 | def process_arg(argument): 288 | if argument[1] is not None: 289 | return ':'.join(argument) 290 | return argument[0] 291 | 292 | for arg1, arg2 in dual_search: 293 | arg_str = '{} {}'.format(process_arg(arg1), process_arg(arg2)) 294 | cmd = include_cmd.format(arg_str) 295 | 296 | results_1 = {s[0] for s in helper.search(sysv_dict, arg1[0], arg1[1])} 297 | results_2 = {s[0] for s in helper.search(sysv_dict, arg2[0], arg2[1])} 298 | 299 | n_results = len(results_1 & results_2) 300 | 301 | if arg1 in dne_search or arg2 in dne_search: 302 | with pytest.raises(RuntimeError): 303 | output = helper.get_cmd_output(cmd) 304 | elif n_results > 0: 305 | output = helper.get_cmd_output(cmd) 306 | assert len(output) == n_results 307 | else: 308 | with pytest.raises(RuntimeError): 309 | output = helper.get_cmd_output(cmd) 310 | 311 | # Exclude multiple arguments 312 | for arg1, arg2 in dual_search: 313 | arg_str = '{} {}'.format(process_arg(arg1), process_arg(arg2)) 314 | cmd = exclude_cmd.format(arg_str) 315 | 316 | results_1 = {s[0] for s in helper.search(sysv_dict, arg1[0], arg1[1])} 317 | results_2 = {s[0] for s in helper.search(sysv_dict, arg2[0], arg2[1])} 318 | 319 | n_results = len(results_1 | results_2) 320 | 321 | if n_results < n_history: 322 | output = helper.get_cmd_output(cmd) 323 | assert n_history - len(output) == n_results 324 | else: 325 | with pytest.raises(RuntimeError): 326 | output = helper.get_cmd_output(cmd) 327 | 328 | # Include and Exclude tests 329 | for arg1, arg2 in dual_search: 330 | cmd = in_ex_cmd.format(process_arg(arg1), process_arg(arg2)) 331 | 332 | full_files = set(sysv_dict.keys()) 333 | results_1 = {s[0] for s in helper.search(sysv_dict, arg1[0], arg1[1])} 334 | results_2 = {s[0] for s in helper.search(sysv_dict, arg2[0], arg2[1])} 335 | 336 | include_result = results_1 337 | exclude_result = full_files - results_2 338 | 339 | full_result = include_result & exclude_result 340 | n_full_result = len(full_result) 341 | 342 | if arg1 in dne_search: 343 | with pytest.raises(RuntimeError): 344 | output = helper.get_cmd_output(cmd) 345 | elif n_full_result > 0: 346 | output = helper.get_cmd_output(cmd) 347 | assert len(output) == n_full_result 348 | else: 349 | with pytest.raises(RuntimeError): 350 | output = helper.get_cmd_output(cmd) 351 | 352 | 353 | # @pytest.mark.usefixtures("cleandir") 354 | # def test_comparison_search(): 355 | # n_history = 20 356 | 357 | # sysv_dict = dict() 358 | 359 | # end = n_history // 2 360 | # start = -1 * end 361 | 362 | # for i in range(start, end): 363 | # tracer = helper.get_tracer() 364 | 365 | # # half the experiments have list options 366 | # if i % 2 == 0: 367 | # helper.add_lists_option(tracer) 368 | # sysv = helper.generate_sysv(i) 369 | # else: 370 | # sysv = helper.generate_sysv(i, False) 371 | 372 | # args = tracer.parse_args(sysv) 373 | # args_file = tracer.args_file 374 | # sysv_dict[args_file] = args 375 | 376 | # list_args = ['list', 'flist'] 377 | # single_args = ['integer', 'float', 'device'] 378 | # arg_wo_value = ['integer', 'list'] 379 | # dne_args = ['lists'] 380 | # operator = ['<', '>', '<=', '>=', '==', ':'] 381 | 382 | # include_cmd = "lite_trace.py -i {}" 383 | # exclude_cmd = "lite_trace.py -e {}" 384 | # in_ex_cmd = "lite_trace.py -i {} -e {}" 385 | 386 | # num_test = 1 387 | # args_dict = {arg: helper.pick_values(sysv_dict, arg, num_test) 388 | # for arg in list_args + single_args} 389 | # exist_search = helper.create_args_list(args_dict) 390 | 391 | # dne_search = helper.generate_dne_args(dne_args, cast=float) 392 | # dne_search |= helper.generate_dne_args(list_args, cast=float) 393 | 394 | # arg_wo_search = helper.generate_dne_args(arg_wo_value, int, upper_limit=None) 395 | 396 | # all_search = exist_search | dne_search | arg_wo_search 397 | 398 | # dual_search = list(itertools.product(all_search, repeat=2)) 399 | 400 | # def process_arg(argument): 401 | # if argument[1] is not None: 402 | # return ':'.join(argument) 403 | # return argument[0] 404 | 405 | # for arg1, arg2 in dual_search: 406 | # arg_str = '{} {}'.format(process_arg(arg1), process_arg(arg2)) 407 | # cmd = include_cmd.format(arg_str) 408 | 409 | # results_1 = {s[0] for s in helper.search(sysv_dict, arg1[0], arg1[1])} 410 | # results_2 = {s[0] for s in helper.search(sysv_dict, arg2[0], arg2[1])} 411 | 412 | # n_results = len(results_1 & results_2) 413 | 414 | # if arg1 in dne_search or arg2 in dne_search: 415 | # with pytest.raises(RuntimeError): 416 | # output = helper.get_cmd_output(cmd) 417 | # elif n_results > 0: 418 | # output = helper.get_cmd_output(cmd) 419 | # assert len(output) == n_results 420 | # else: 421 | # with pytest.raises(RuntimeError): 422 | # output = helper.get_cmd_output(cmd) 423 | 424 | # # Exclude multiple arguments 425 | # for arg1, arg2 in dual_search: 426 | # arg_str = '{} {}'.format(process_arg(arg1), process_arg(arg2)) 427 | # cmd = exclude_cmd.format(arg_str) 428 | 429 | # results_1 = {s[0] for s in helper.search(sysv_dict, arg1[0], arg1[1])} 430 | # results_2 = {s[0] for s in helper.search(sysv_dict, arg2[0], arg2[1])} 431 | 432 | # n_results = len(results_1 | results_2) 433 | 434 | # if n_results < n_history: 435 | # output = helper.get_cmd_output(cmd) 436 | # assert n_history - len(output) == n_results 437 | # else: 438 | # with pytest.raises(RuntimeError): 439 | # output = helper.get_cmd_output(cmd) 440 | 441 | # # Include and Exclude tests 442 | # for arg1, arg2 in dual_search: 443 | # cmd = in_ex_cmd.format(process_arg(arg1), process_arg(arg2)) 444 | 445 | # full_files = set(sysv_dict.keys()) 446 | # results_1 = {s[0] for s in helper.search(sysv_dict, arg1[0], arg1[1])} 447 | # results_2 = {s[0] for s in helper.search(sysv_dict, arg2[0], arg2[1])} 448 | 449 | # include_result = results_1 450 | # exclude_result = full_files - results_2 451 | 452 | # full_result = include_result & exclude_result 453 | # n_full_result = len(full_result) 454 | 455 | # if arg1 in dne_search: 456 | # with pytest.raises(RuntimeError): 457 | # output = helper.get_cmd_output(cmd) 458 | # elif n_full_result > 0: 459 | # output = helper.get_cmd_output(cmd) 460 | # assert len(output) == n_full_result 461 | # else: 462 | # with pytest.raises(RuntimeError): 463 | # output = helper.get_cmd_output(cmd) 464 | --------------------------------------------------------------------------------