├── .coveragerc ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── MANIFEST.in ├── README.rst ├── conff ├── __init__.py ├── data │ ├── malformed_foreach_01.yml │ ├── malformed_foreach_02.yml │ ├── sample_config_01.yml │ ├── sample_config_02.yml │ ├── sample_config_03.yml │ ├── test_config_01.json │ ├── test_config_01.yml │ ├── test_config_02.yml │ ├── test_config_03.yml │ ├── test_config_04.yml │ ├── test_config_05.yml │ └── test_tpl_01.tpl ├── ee.py ├── parser.py ├── test_parser.py └── utils.py ├── requirements-test.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── test.py ├── tox.ini └── unittest.cfg /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = setup.py 3 | .tox/* 4 | /home/travis/virtualenv/* 5 | 6 | [report] 7 | show_missing = True 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | .idea 106 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | python: 4 | - 3.5 5 | - 3.6 6 | install: 7 | - pip install nose 8 | - pip install coveralls 9 | - pip install tox-travis 10 | script: 11 | - tox 12 | after_success: 13 | - coveralls 14 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Logs 2 | 3 | ## 0.5.0 4 | - Add Parser class 5 | - Implement T3: simpleeval options 6 | - Add options to extend structure in template 7 | - Integrate with Jinja2, template in JSON/YAML as string or file then covert back to object 8 | - Add more tests 9 | - Update readme to latest protocol 10 | 11 | ## 0.4.2 12 | - Add support 3.5 13 | - Add more examples and tests 14 | - Expose more functions for encryption 15 | 16 | ## 0.3.3 17 | - Drop 2.8, 3.0, 3.1, 3.2, 3.3, 3.4 due to Python EOL versions 18 | 19 | ## 0.3.2 20 | - Update documentation 21 | 22 | ## 0.3.0 23 | - Change readme to rst format 24 | - Add tox 25 | - fix error on the setup.py 26 | - Add pyyaml requirement 27 | - Drop python 2.7, 3.5 28 | - Update documentation 29 | - Add travis 30 | 31 | ## 0.1.0 32 | - Added ``conff`` 33 | - Initial state 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Robertus Johansyah 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.txt 2 | recursive-include conff *.py *.yml 3 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | conff 2 | ===== 3 | 4 | Simple config parser with evaluator library. 5 | 6 | .. image:: https://badge.fury.io/py/conff.svg 7 | :target: https://badge.fury.io/py/conff 8 | 9 | .. image:: https://travis-ci.com/kororo/conff.svg?branch=master 10 | :target: https://travis-ci.com/kororo/conff 11 | 12 | .. image:: https://coveralls.io/repos/github/kororo/conff/badge.svg?branch=master 13 | :target: https://coveralls.io/github/kororo/conff?branch=master 14 | 15 | .. image:: https://api.codeclimate.com/v1/badges/c476e9c6bfe505bc4b4d/maintainability 16 | :target: https://codeclimate.com/github/kororo/conff/maintainability 17 | :alt: Maintainability 18 | 19 | .. image:: https://badges.gitter.im/kororo-conff.png 20 | :target: https://gitter.im/kororo-conff 21 | :alt: Gitter 22 | 23 | 24 | Why Another Config Parser Module? 25 | --------------------------------- 26 | 27 | This project inspired of the necessity complex config in a project. By means complex: 28 | 29 | - Reusability 30 | 31 | - Import values from file 32 | - Reference values from other object 33 | 34 | - Secure 35 | 36 | - Encrypt/decrypt sensitive values 37 | 38 | - Flexible 39 | 40 | - Make logical expression to derive values 41 | - Combine with `jinja2 `_ template based 42 | 43 | - Powerful 44 | 45 | - Add custom functions in Python 46 | - Link name data from Python 47 | 48 | 49 | Real World Examples 50 | ------------------- 51 | 52 | All the example below located in `data directory `_. 53 | Imagine you start an important project, your code need to analyse image/videos which involves workflow 54 | with set of tasks with AWS Rekognition. The steps will be more/less like this: 55 | 56 | 1. Read images/videos from a specific folder, if images goes to (2), if videos goes to (3). 57 | 58 | 2. Analyse the images with AWS API, then goes (4) 59 | 60 | 3. Analyse the videos with AWS API, then goes (4) 61 | 62 | 4. Write the result back to JSON file, finished 63 | 64 | The configuration required: 65 | 66 | 1. Read images/videos (where is the folder) 67 | 68 | 2. Analyse images (AWS API credential and max resolution for image) 69 | 70 | 3. Analyse videos (AWS API credential and max resolution for video) 71 | 72 | 4. Write results (where is the result should be written) 73 | 74 | 1. Without conff library 75 | ^^^^^^^^^^^^^^^^^^^^^^^^ 76 | 77 | File: `data/sample_config_01.yml `_ 78 | 79 | Where it is all started, if we require to store the configuration as per normally, it should be like this. 80 | 81 | .. code:: yaml 82 | 83 | job: 84 | read_image: 85 | # R01 86 | root_path: /data/project/images_and_videos/ 87 | analyse_image: 88 | # R02 89 | api_cred: 90 | region_name: ap-southeast-2 91 | aws_access_key_id: ACCESSKEY1234 92 | # R03 93 | aws_secret_access_key: ACCESSSECRETPLAIN1234 94 | max_res: [1024, 768] 95 | analyse_video: 96 | # R04 97 | api_cred: 98 | region_name: ap-southeast-2 99 | aws_access_key_id: ACCESSKEY1234 100 | aws_secret_access_key: ACCESSSECRETPLAIN1234 101 | max_res: [800, 600] 102 | write_result: 103 | # R05 104 | output_path: /data/project/result.json 105 | 106 | .. code:: python 107 | 108 | import yaml 109 | with open('data/sample_config_01.yml') as stream: 110 | r1 = yaml.safe_load(stream) 111 | 112 | Notes: 113 | 114 | - R01: The subpath of "/data/project" is repeated between R01 and R05 115 | - R02: api_cred is repeatedly defined with R04 116 | - R03: the secret is plain visible, if this stored in GIT, it is pure disaster 117 | 118 | 2. Fix the repeat 119 | ^^^^^^^^^^^^^^^^^ 120 | 121 | File: `data/sample_config_02.yml `_ 122 | 123 | Repeating values/configuration is bad, this could potentially cause human mistake if changes made is not 124 | consistently applied in all occurences. 125 | 126 | .. code:: yaml 127 | 128 | # this can be any name, as long as not reserved in Python 129 | shared: 130 | project_path: /data/project 131 | aws_cred: 132 | region_name: ap-southeast-2 133 | aws_access_key_id: ACCESSKEY1234 134 | # F03 135 | aws_secret_access_key: F.decrypt('gAAAAABbBBhOJDMoQSbF9jfNgt97FwyflQEZRxv2L2buv6YD_Jiq8XNrxv8VqFis__J7YlpZQA07nDvzYwMU562Mlm978uP9BQf6M9Priy3btidL6Pm406w=') 136 | 137 | job: 138 | read_image: 139 | # F01 140 | root_path: R.shared.project_path + '/images_and_videos/' 141 | analyse_image: 142 | # F02 143 | api_cred: R.shared.aws_cred 144 | max_res: [1024, 768] 145 | analyse_video: 146 | # F04 147 | api_cred: R.shared.aws_cred 148 | max_res: [800, 600] 149 | write_result: 150 | # F05 151 | output_path: R.shared.project_path + '/result.json' 152 | 153 | .. code:: python 154 | 155 | import conff 156 | # ekey is the secured encryption key 157 | # WARNING: this is just demonstration purposes 158 | ekey = 'FOb7DBRftamqsyRFIaP01q57ZLZZV6MVB2xg1Cg_E7g=' 159 | r2 = conff.load(fs_path='data/sample_config_02.yml', params={'ekey': ekey}) 160 | 161 | Notes: 162 | 163 | - F01: it is safe if the prefix '/data/project' need to be changed, it will automatically changed for F05 164 | - F02: no more duplicated config with F04 165 | - F03: it is secured to save this to GIT, as long as the encryption key is stored securely somewhere in server such 166 | as ~/.secret 167 | 168 | 3. Optimise to the extreme 169 | ^^^^^^^^^^^^^^^^^^^^^^^^^^ 170 | 171 | File: `data/sample_config_03.yml `_ 172 | 173 | This is just demonstration purposes to see the full capabilities of this library. 174 | 175 | .. code:: yaml 176 | 177 | # this can be any name, as long as not reserved in Python 178 | shared: 179 | project_path: /data/project 180 | analyse_image_video: 181 | api_cred: 182 | region_name: ap-southeast-2 183 | aws_access_key_id: ACCESSKEY1234 184 | aws_secret_access_key: F.decrypt('gAAAAABbBBhOJDMoQSbF9jfNgt97FwyflQEZRxv2L2buv6YD_Jiq8XNrxv8VqFis__J7YlpZQA07nDvzYwMU562Mlm978uP9BQf6M9Priy3btidL6Pm406w=') 185 | max_res: [1024, 768] 186 | job: 187 | read_image: 188 | root_path: R.shared.project_path + '/images_and_videos/' 189 | analyse_image: R.shared.analyse_image_video 190 | analyse_video: 191 | F.extend: R.shared.analyse_image_video 192 | F.update: 193 | max_res: [800, 600] 194 | write_result: 195 | output_path: R.shared.project_path + '/result.json' 196 | 197 | For completeness, ensuring data is consistent and correct between sample_config_01.yml, sample_config_02.yml 198 | and sample_config_03.yml. 199 | 200 | .. code:: python 201 | 202 | # nose2 conff.test.ConffTestCase.test_sample 203 | fs_path = 'data/sample_config_01.yml' 204 | with open(fs_path) as stream: 205 | r1 = yaml.safe_load(stream) 206 | fs_path = 'data/sample_config_02.yml' 207 | ekey = 'FOb7DBRftamqsyRFIaP01q57ZLZZV6MVB2xg1Cg_E7g=' 208 | r2 = conff.load(fs_path=fs_path, params={'ekey': ekey}) 209 | fs_path = 'data/sample_config_03.yml' 210 | r3 = conff.load(fs_path=fs_path, params={'ekey': ekey}) 211 | self.assertDictEqual(r1['job'], r2['job'], 'Mismatch value') 212 | self.assertDictEqual(r2['job'], r3['job'], 'Mismatch value') 213 | 214 | Feedback and Discussion 215 | ----------------------- 216 | 217 | Come to Gitter channel to discuss, pass any feedbacks and suggestions. If you like to be contributor, please do let me know. 218 | 219 | Important Notes 220 | --------------- 221 | 222 | Parsing Order 223 | ^^^^^^^^^^^^^ 224 | 225 | conff will only parse and resolve variable/names top to bottom order. Please ensure you arrange your configuration 226 | in the same manner, there is no auto-dependencies resolver to handle complex and advanced names currently. 227 | 228 | dict vs collections.OrderedDict 229 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 230 | 231 | In Python 3.5, the dict data type has inconsistent ordering, it is **STRONGLY** recommended to use **OrderedDict** if 232 | you manually parse object. If you load from YAML file, the library already handled it. The reason of order is important, 233 | this due to simplification and assumption of order execution. The library will parse the values from top to bottom as 234 | per order in the key-value dictionary. 235 | 236 | Install 237 | ------- 238 | 239 | .. code:: bash 240 | 241 | [sudo] pip install conff 242 | 243 | Basic Usage 244 | ----------- 245 | 246 | To get very basic parsing: 247 | 248 | Simple parse 249 | ^^^^^^^^^^^^ 250 | 251 | .. code:: python 252 | 253 | import conff 254 | p = conff.Parser() 255 | r = p.parse({'math': '1 + 3'}) 256 | assert r == {'math': 4} 257 | 258 | Load YAML file 259 | ^^^^^^^^^^^^^^ 260 | 261 | .. code:: python 262 | 263 | import conff 264 | p = conff.Parser() 265 | r = p.load('path_of_file.yml') 266 | 267 | Template based config 268 | ^^^^^^^^^^^^^^^^^^^^^ 269 | 270 | Using `jinja2 `_ to craft more powerful config. 271 | 272 | .. code:: python 273 | 274 | import conff 275 | p = conff.Parser() 276 | r = p.parse('F.template("{{ 1 + 2 }}")') 277 | assert r == 3 278 | 279 | 280 | Examples 281 | -------- 282 | 283 | More advances examples: 284 | 285 | Parse with simple expression 286 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 287 | 288 | .. code:: python 289 | 290 | import conff 291 | p = conff.Parser() 292 | r = p.parse('1 + 2') 293 | assert r == 3 294 | 295 | Parse object 296 | ^^^^^^^^^^^^ 297 | 298 | .. code:: python 299 | 300 | import conff 301 | p = conff.Parser() 302 | r = p.parse({"math": "1 + 2"}) 303 | assert r == {'math': 3} 304 | 305 | Ignore expression (declare it as string) 306 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 307 | 308 | .. code:: python 309 | 310 | import conff 311 | p = conff.Parser() 312 | r = conff.parse('"1 + 2"') 313 | assert r == '1 + 2' 314 | 315 | Parse error behaviours 316 | ^^^^^^^^^^^^^^^^^^^^^^ 317 | 318 | .. code:: python 319 | 320 | import conff 321 | p = conff.Parser() 322 | r = p.parse({'math': '1 / 0'}) 323 | # Exception raised 324 | # ZeroDivisionError: division by zero 325 | 326 | 327 | import files 328 | ^^^^^^^^^^^^ 329 | 330 | .. code:: python 331 | 332 | import conff 333 | ## y1.yml 334 | # shared_conf: 1 335 | ## y2.yml 336 | # conf: F.inc('y1.yml') 337 | 338 | p = conff.Parser() 339 | r = p.load('y2.yml') 340 | assert r == {'conf': {'shared_conf': 1}} 341 | 342 | Parse with functions 343 | ^^^^^^^^^^^^^^^^^^^^ 344 | 345 | .. code:: python 346 | 347 | import conff 348 | def fn_add(a, b): 349 | return a + b 350 | p = conff.Parser(fns={'add': fn_add}) 351 | r = p.parse('F.add(1, 2)') 352 | assert r == 3 353 | 354 | Parse with names 355 | ^^^^^^^^^^^^^^^^ 356 | 357 | .. code:: python 358 | 359 | import conff 360 | p = conff.Parser(names={'a': 1, 'b': 2}) 361 | r = conff.parse('a + b') 362 | assert r == 3 363 | 364 | Parse with extends 365 | ^^^^^^^^^^^^^^^^^^ 366 | 367 | .. code:: python 368 | 369 | import conff 370 | data = { 371 | 't1': {'a': 'a'}, 372 | 't2': { 373 | 'F.extend': 'R.t1', 374 | 'b': 'b' 375 | } 376 | } 377 | p = conff.Parser() 378 | r = p.parse(data) 379 | assert r == {'t1': {'a': 'a'}, 't2': {'a': 'a', 'b': 'b'}} 380 | 381 | Parse with updates 382 | ^^^^^^^^^^^^^^^^^^ 383 | 384 | .. code:: python 385 | 386 | import conff 387 | data = { 388 | 't1': {'a': 'a'}, 389 | 't2': { 390 | 'b': 'b', 391 | 'F.update': { 392 | 'c': 'c' 393 | }, 394 | } 395 | } 396 | p = conff.Parser() 397 | r = p.parse(data) 398 | assert r == {'t1': {'a': 'a'}, 't2': {'b': 'b', 'c': 'c'}} 399 | 400 | Parse with extends and updates 401 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 402 | 403 | .. code:: python 404 | 405 | import conff 406 | data = { 407 | 't1': {'a': 'a'}, 408 | 't2': { 409 | 'F.extend': 'R.t1', 410 | 'b': 'b', 411 | 'F.update': { 412 | 'a': 'A', 413 | 'c': 'c' 414 | }, 415 | } 416 | } 417 | p = conff.Parser() 418 | r = p.parse(data) 419 | assert r == {'t1': {'a': 'a'}, 't2': {'a': 'A', 'b': 'b', 'c': 'c'}} 420 | 421 | Create a list of values 422 | ^^^^^^^^^^^^^^^^^^^^^^^ 423 | 424 | This creates a list of floats, similar to numpy.linspace 425 | 426 | .. code:: python 427 | 428 | import conff 429 | data = {'t2': 'F.linspace(0, 10, 5)'} 430 | p = conff.Parser() 431 | r = p.parse(data) 432 | assert r == {'t2': [0.0, 2.5, 5.0, 7.5, 10.0]} 433 | 434 | This also creates a list of floats, but behaves like numpy.arange (although 435 | slightly different in that it is inclusive of the endpoint). 436 | 437 | .. code:: python 438 | 439 | import conff 440 | data = {'t2': 'F.arange(0, 10, 2)'} 441 | p = conff.Parser() 442 | r = p.parse(data) 443 | assert r == {'t2': [0, 2, 4, 6, 8, 10]} 444 | 445 | Parse with for each 446 | ^^^^^^^^^^^^^^^^^^^ 447 | 448 | One can mimic the logic of a for loop with the following example 449 | 450 | .. code:: python 451 | 452 | import conff 453 | data = {'t1': 2, 454 | 'F.foreach': { 455 | 'values': 'F.linspace(0, 10, 2)', 456 | # You have access to loop.index, loop.value, and loop.length 457 | # within the template, as well as all the usual names 458 | 'template': { 459 | '"test%i"%loop.index': 'R.t1*loop.value', 460 | 'length': 'loop.length' 461 | } 462 | } 463 | } 464 | p = conff.Parser() 465 | r = p.parse(data) 466 | assert r == {'length': 3, 't1': 2, 'test0': 0.0, 'test1': 10.0, 'test2': 20.0} 467 | 468 | Encryption 469 | ---------- 470 | 471 | This section to help you to quickly generate encryption key, initial encrypt values and test to decrypt the value. 472 | 473 | .. code:: python 474 | 475 | import conff 476 | # generate key, save it somewhere safe 477 | names = {'R': {'_': {'etype': 'fernet'}}} 478 | etype = conff.generate_key(names)() 479 | # or just 480 | ekey = conff.generate_key()('fernet') 481 | 482 | # encrypt data 483 | # BIG WARNING: this should be retrieved somewhere secured for example in ~/.secret 484 | # below just for example purposes 485 | ekey = 'FOb7DBRftamqsyRFIaP01q57ZLZZV6MVB2xg1Cg_E7g=' 486 | names = {'R': {'_': {'etype': 'fernet', 'ekey': ekey}}} 487 | # gAAAAABbBBhOJDMoQSbF9jfNgt97FwyflQEZRxv2L2buv6YD_Jiq8XNrxv8VqFis__J7YlpZQA07nDvzYwMU562Mlm978uP9BQf6M9Priy3btidL6Pm406w= 488 | encrypted_value = conff.encrypt(names)('ACCESSSECRETPLAIN1234') 489 | 490 | # decrypt data 491 | ekey = 'FOb7DBRftamqsyRFIaP01q57ZLZZV6MVB2xg1Cg_E7g=' 492 | names = {'R': {'_': {'etype': 'fernet', 'ekey': ekey}}} 493 | encrypted_value = 'gAAAAABbBBhOJDMoQSbF9jfNgt97FwyflQEZRxv2L2buv6YD_Jiq8XNrxv8VqFis__J7YlpZQA07nDvzYwMU562Mlm978uP9BQf6M9Priy3btidL6Pm406w=' 494 | conff.decrypt(names)(encrypted_value) 495 | 496 | Test 497 | ---- 498 | 499 | To test this project: 500 | 501 | .. code:: bash 502 | 503 | # default test 504 | nose2 505 | 506 | # test with coverage 507 | nose2 --with-coverage 508 | 509 | # test specific 510 | nose2 conff.test.ConffTestCase.test_sample 511 | 512 | TODO 513 | ---- 514 | 515 | - [X] Setup basic necessity 516 | 517 | - [X] Stop procrastinating 518 | - [X] Project registration in pypi 519 | - [X] Create unit tests 520 | - [X] Setup travis 521 | - [X] Setup coveralls 522 | 523 | - [ ] Add more support on `Python versions `_ 524 | 525 | - [ ] 2.7 526 | - [ ] 3.4 527 | - [X] 3.5 528 | - [X] 3.6 529 | 530 | - [ ] Features 531 | 532 | - Wish List Features now moved to `wiki page `_. 533 | 534 | - [ ] Improve docs 535 | 536 | - [ ] Add more code comments and visibilities 537 | - [ ] Make github layout code into two left -> right 538 | - [X] Put more examples 539 | - [ ] Setup readthedocs 540 | - [ ] Add code conduct, issue template into git project. 541 | - [ ] Add information that conff currently accept YML and it not limited, it can take any objects 542 | 543 | 544 | Other Open Source 545 | ----------------- 546 | 547 | This project uses other awesome projects: 548 | 549 | - `cryptography `_ 550 | - `jinja2 `_ 551 | - `munch `_ 552 | - `simpleeval `_ 553 | - `yaml `_ 554 | 555 | Who uses conff? 556 | --------------- 557 | 558 | Please send a PR to keep the list growing, if you may please add your handle and company. 559 | -------------------------------------------------------------------------------- /conff/__init__.py: -------------------------------------------------------------------------------- 1 | from conff import ee 2 | from conff import parser 3 | 4 | 5 | __all__ = ['parse', 'load', 'encrypt', 'decrypt', 'generate_key', 'update', 'Parser'] 6 | 7 | parse = ee.parse 8 | load = ee.load 9 | encrypt = ee.encrypt 10 | decrypt = ee.decrypt 11 | generate_key = ee.generate_key 12 | update = parser.update_recursive 13 | Parser = parser.Parser 14 | -------------------------------------------------------------------------------- /conff/data/malformed_foreach_01.yml: -------------------------------------------------------------------------------- 1 | # malformed foreach sections 2 | test: 3 | F.foreach: 4 | template: 5 | '"test%i"%loop.index': 6 | value: loop.value 7 | length: loop.length 8 | -------------------------------------------------------------------------------- /conff/data/malformed_foreach_02.yml: -------------------------------------------------------------------------------- 1 | test: 2 | F.foreach: 3 | values: F.arange(0, 6, 4) 4 | template: 'notadict' 5 | -------------------------------------------------------------------------------- /conff/data/sample_config_01.yml: -------------------------------------------------------------------------------- 1 | job: 2 | read_image: 3 | # R01 4 | root_path: /data/project/images_and_videos/ 5 | analyse_image: 6 | # R02 7 | api_cred: 8 | region_name: ap-southeast-2 9 | aws_access_key_id: ACCESSKEY1234 10 | # R03 11 | aws_secret_access_key: ACCESSSECRETPLAIN1234 12 | max_res: [1024, 768] 13 | analyse_video: 14 | # R04 15 | api_cred: 16 | region_name: ap-southeast-2 17 | aws_access_key_id: ACCESSKEY1234 18 | aws_secret_access_key: ACCESSSECRETPLAIN1234 19 | max_res: [800, 600] 20 | write_result: 21 | # R05 22 | output_path: /data/project/result.json 23 | -------------------------------------------------------------------------------- /conff/data/sample_config_02.yml: -------------------------------------------------------------------------------- 1 | # this can be any name, as long as not reserved in Python 2 | shared: 3 | project_path: /data/project 4 | aws_cred: 5 | region_name: ap-southeast-2 6 | aws_access_key_id: ACCESSKEY1234 7 | # F03 8 | aws_secret_access_key: F.decrypt('gAAAAABbBBhOJDMoQSbF9jfNgt97FwyflQEZRxv2L2buv6YD_Jiq8XNrxv8VqFis__J7YlpZQA07nDvzYwMU562Mlm978uP9BQf6M9Priy3btidL6Pm406w=') 9 | 10 | job: 11 | read_image: 12 | # F01 13 | root_path: R.shared.project_path + '/images_and_videos/' 14 | analyse_image: 15 | # F02 16 | api_cred: R.shared.aws_cred 17 | max_res: [1024, 768] 18 | analyse_video: 19 | # F04 20 | api_cred: R.shared.aws_cred 21 | max_res: [800, 600] 22 | write_result: 23 | # F05 24 | output_path: R.shared.project_path + '/result.json' 25 | -------------------------------------------------------------------------------- /conff/data/sample_config_03.yml: -------------------------------------------------------------------------------- 1 | # this can be any name, as long as not reserved in Python 2 | shared: 3 | project_path: /data/project 4 | analyse_image_video: 5 | api_cred: 6 | region_name: ap-southeast-2 7 | aws_access_key_id: ACCESSKEY1234 8 | aws_secret_access_key: F.decrypt('gAAAAABbBBhOJDMoQSbF9jfNgt97FwyflQEZRxv2L2buv6YD_Jiq8XNrxv8VqFis__J7YlpZQA07nDvzYwMU562Mlm978uP9BQf6M9Priy3btidL6Pm406w=') 9 | max_res: [1024, 768] 10 | job: 11 | read_image: 12 | root_path: R.shared.project_path + '/images_and_videos/' 13 | analyse_image: R.shared.analyse_image_video 14 | analyse_video: 15 | F.extend: R.shared.analyse_image_video 16 | F.update: 17 | max_res: [800, 600] 18 | write_result: 19 | output_path: R.shared.project_path + '/result.json' 20 | -------------------------------------------------------------------------------- /conff/data/test_config_01.json: -------------------------------------------------------------------------------- 1 | { 2 | "test_1": 1, 3 | "test_2": "1 + 1" 4 | } 5 | -------------------------------------------------------------------------------- /conff/data/test_config_01.yml: -------------------------------------------------------------------------------- 1 | # test 1: simple string 2 | test_1: test_1 3 | # test 2: empty value 4 | test_2: '[empty]' -------------------------------------------------------------------------------- /conff/data/test_config_02.yml: -------------------------------------------------------------------------------- 1 | # test 1: simple string 2 | test_1: test_1 3 | # test 2: simple int 4 | test_2: 2 5 | # test 3: simple list 6 | test_3: ['test_3', 3] 7 | # test 4: simple dict 8 | test_4: 9 | test_4_1: test_4_1 10 | # test 5: simple dict of dict 11 | test_5: 12 | test_5_1: test_5_1 13 | test_5_2: 14 | test_5_2_1: test_5_2_1 15 | # test 6: simple expression 16 | test_6: ('test_' + F.str(1 + 2 + 3)) 17 | # test 7: simple extend 18 | test_7: 19 | data: [1] 20 | data2: 21 | data2_1: 1 22 | data2_2: 2 23 | test_7_1: F.extend([1], [2]) 24 | test_7_2: F.extend(R.test_7.data, [2]) 25 | test_7_3: "F.extend(R.test_7.data2, {'data2_3': 1 + 2})" 26 | # test 8: complex extend 27 | test_8: 28 | F.extend: R.test_7.data2 29 | data2_2: 2a 30 | data2_3: 1 + 2 31 | data2_4: 4 32 | # test 9: complex expressions 33 | test_9: 34 | test_9_1: ([F.has([1, 2, 3], 2), F.has([1, 2, 3], 4), F.has({'a':'a'}, 'a'), F.has({'a':'a'}, 'b')]) 35 | test_9_2: ([F.next([1, 2]), F.next('RO')]) 36 | test_9_3: F.join(['1', '2', '3']) 37 | test_9_4: F.trim('/ro/ro ') 38 | # test 10: error expression 39 | test_10: F.no_exist() 40 | # test 11: encrypt/decrypt 41 | test_11: F.decrypt(F.encrypt('test_11')) 42 | # test 12: importing 43 | test_12: F.inc('test_config_01.yml') 44 | # test 13: update 45 | test_13: 46 | F.update: 47 | test_13_1: 1 48 | test_13_2: '2' 49 | test_13_3: 1 + 2 50 | test_13_5: 51 | test_13_5_1: 1 52 | test_13_6: 53 | test_13_6_1: 1 54 | # test 14: extend + update 55 | test_14: 56 | F.extend: R.test_13 57 | F.update: 58 | test_13_1: 11 59 | test_13_4: R.test_13.test_13_3 + 1 60 | test_13_5: 5 61 | test_13_6: 62 | test_13_6_2: 63 | test_13_6_2_1: 1 64 | test_13_6_2_2: 2 65 | # test 15: foreach with linspace 66 | test_15: 67 | F.foreach: 68 | values: F.linspace(0, 6, 3) 69 | template: 70 | '"test%i"%loop.index': 71 | value: loop.value 72 | length: loop.length 73 | # test 16: foreach with arange 74 | test_16: 75 | F.foreach: 76 | values: F.arange(0, 6, 3) 77 | template: 78 | '"test%i"%loop.index': 79 | value: loop.value 80 | length: loop.length 81 | test_17: 82 | F.foreach: 83 | values: F.arange(0, 6, 4) 84 | template: 85 | '"test%i"%loop.index': 86 | value: loop.value 87 | length: loop.length 88 | # test 18: test float and int functions 89 | test_18: 90 | test_18_1: 'F.int(F.float(3.5))' 91 | -------------------------------------------------------------------------------- /conff/data/test_config_03.yml: -------------------------------------------------------------------------------- 1 | # test 1: simple string 2 | test_1: {{}, {}} 3 | -------------------------------------------------------------------------------- /conff/data/test_config_04.yml: -------------------------------------------------------------------------------- 1 | test_13: 2 | F.update: 3 | test_13_1: 1 4 | test_13_2: '2' 5 | test_13_3: 1 + 2 6 | test_13_5: 7 | test_13_5_1: 1 8 | test_13_6: 9 | test_13_6_1: 1 10 | test_14: 11 | F.extend: R.test_13 12 | F.update: 13 | test_13_1: 11 14 | test_13_4: R.test_13.test_13_3 + 1 15 | test_13_5: 5 16 | test_13_6: 17 | test_13_6_2: 18 | test_13_6_2_1: 1 19 | test_13_6_2_2: 2 20 | -------------------------------------------------------------------------------- /conff/data/test_config_05.yml: -------------------------------------------------------------------------------- 1 | # test: simple value 2 | test_1: 1 3 | # test: template as string, it is seamless names from input (test) and template (test_1) 4 | test_2: 5 | F.template: "'{{ R.test_1 + test }}'" 6 | # test: template as file (borrowing F.inc capabilities), if test_tpl_01.tpl is {{1 + 2}} 7 | test_3: 8 | F.template: F.inc('test_tpl_01.tpl') 9 | # test: this where attaching more complex object 10 | test_4: 11 | test_4_0: [3, 4] 12 | F.template: | 13 | test_4_1: {{ R.test_1 }} 14 | test_4_2: {{ 1 + 1 }} 15 | {% for i in R.test_4.test_4_0 %} 16 | test_4_{{ i }}: {{ i }} 17 | {% endfor %} 18 | # data type is very important here 19 | test_4_5: {{ R.test_2 | int + R.test_3 | int }} 20 | {% if R.test_1 == 1 %} 21 | test_4_6: 6 22 | {% else %} 23 | test_4_6: 'error' 24 | {% endif %} 25 | # test behaviour of replace 26 | test_4_1: 'error' 27 | # TODO: test this 28 | # test_5: 29 | # F.template: "{{ }}" 30 | # TODO: test this 31 | # test_6: 32 | # F.template: "{{ " 33 | -------------------------------------------------------------------------------- /conff/data/test_tpl_01.tpl: -------------------------------------------------------------------------------- 1 | {{1 + 2}} -------------------------------------------------------------------------------- /conff/ee.py: -------------------------------------------------------------------------------- 1 | # TODO: for now, let user use these function, eventually, before version 1.0, we should mark this as deprecated 2 | 3 | 4 | def parse(root, names: dict = None, fns: dict = None, errors: list = None): 5 | from conff import Parser 6 | p = Parser(names=names, fns=fns) 7 | result = p.parse(root) 8 | errors = errors or [] 9 | errors.extend(p.errors) 10 | return result 11 | 12 | 13 | def load(fs_path: str, fs_root: str = '', params: dict = None, errors: list = None): 14 | from conff import Parser 15 | p = Parser(params=params) 16 | result = p.load(fs_path=fs_path, fs_root=fs_root) 17 | errors = errors or [] 18 | errors.extend(p.errors) 19 | return result 20 | 21 | 22 | def encrypt(names: dict): 23 | def fn_encrypt(data): 24 | from conff import Parser 25 | p = Parser(names=names, params=names.get('R', {}).get('_', {})) 26 | result = p.fn_encrypt(data) 27 | return result 28 | 29 | return fn_encrypt 30 | 31 | 32 | def decrypt(names: dict): 33 | def fn_decrypt(data): 34 | from conff import Parser 35 | p = Parser(names=names, params=names.get('R', {}).get('_', {})) 36 | result = p.fn_decrypt(data) 37 | return result 38 | 39 | return fn_decrypt 40 | 41 | 42 | def generate_key(names: dict): 43 | def fn_generate_key(): 44 | from conff import Parser 45 | p = Parser(names=names, params=names.get('R', {}).get('_', {})) 46 | result = p.generate_crypto_key() 47 | return result 48 | 49 | return fn_generate_key 50 | -------------------------------------------------------------------------------- /conff/parser.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import collections 5 | import copy 6 | import sys 7 | 8 | import simpleeval 9 | import warnings 10 | from jinja2 import Template 11 | from simpleeval import EvalWithCompoundTypes 12 | from cryptography.fernet import Fernet 13 | from conff import utils 14 | from conff.utils import Munch2, update_recursive, yaml_safe_load, filter_value, odict 15 | 16 | 17 | class Parser: 18 | # default params 19 | default_params = { 20 | 'etype': 'fernet', 21 | # list of simpleeval library parameters 22 | 'simpleeval': { 23 | # by default operators = simpleeval.DEFAULT_OPERATORS, 24 | 'operators': {}, 25 | 'options': { 26 | 'max_power': simpleeval.MAX_POWER, 27 | 'max_string_length': simpleeval.MAX_STRING_LENGTH, 28 | 'disallow_prefixes': simpleeval.DISALLOW_PREFIXES 29 | } 30 | } 31 | } 32 | 33 | def __init__(self, names=None, fns=None, params=None): 34 | """ 35 | :param params: A dictionary containing some parameters that will modify 36 | how the builtin functions run. For example, the type of encryption to 37 | use and the encrpyption key to use or simpleeval library parameters 38 | """ 39 | self.errors = [] 40 | self.logger = self.prepare_logger() 41 | self.params = self.prepare_params(params=params) 42 | self.fns = self.prepare_functions(fns=fns) 43 | self.names = self.prepare_names(names=names) 44 | self._evaluator = self.prepare_evaluator() 45 | 46 | def prepare_logger(self): 47 | logger = logging.getLogger('conff') 48 | return logger 49 | 50 | def prepare_params(self, params: dict = None): 51 | """ 52 | Setup parameters for the library 53 | 54 | :param params: A dictionary containing some parameters that will modify 55 | how the builtin functions run. For example, the type of encryption to 56 | use and the encrpyption key to use or simpleeval library parameters 57 | 58 | :return: Prepared parameters 59 | """ 60 | # ensure not to update mutable params 61 | params = copy.deepcopy(params or {}) 62 | # inject with default params with exception for simpleeval.operators 63 | params = utils.update_recursive(params, self.default_params) 64 | return params 65 | 66 | def prepare_functions(self, fns: dict = None): 67 | fns = fns or {} 68 | cls_fns = {fn[3:]: getattr(self, fn) for fn in dir(self) if 'fn_' in fn} 69 | result = {'F': update_recursive(fns, cls_fns)} 70 | return result 71 | 72 | def prepare_names(self, names: dict = None): 73 | names = names or {} 74 | names = names if isinstance(names, Munch2) else Munch2(names) 75 | return names 76 | 77 | def prepare_evaluator(self): 78 | """ 79 | Setup evaluator engine 80 | 81 | :return: Prepare evaluator engine 82 | """ 83 | simpleeval_params = self.params.get('simpleeval', {}) 84 | # update simpleeval safety options 85 | for k, v in simpleeval_params.get('options', {}).items(): 86 | setattr(simpleeval, k.upper(), v) 87 | evaluator = EvalWithCompoundTypes() 88 | # self._evals_functions should mirror self.fns 89 | # TODO: Make a test to ensure proper mirroring 90 | evaluator.functions = self.fns 91 | evaluator.names = self.names 92 | # set the operators 93 | if simpleeval_params.get('operators'): 94 | evaluator.operators = simpleeval_params.get('operators') 95 | 96 | return evaluator 97 | 98 | def load(self, fs_path: str, fs_root: str = '', fs_include: list = None): 99 | """ 100 | Parse configuration file on disk. 101 | 102 | :param fs_path: The path to the file on disk. If fs_root is specified, 103 | this will be interpreted as a path relative to fs_root 104 | :type fs_path: str 105 | :param fs_root: Root directory to use when parsing. Defaults to the 106 | directory of the input file. 107 | :type fs_root: str 108 | :param fs_include: A list of additional directories in which to 109 | search for included files. Always contains the directory of the input 110 | file, and will also contain fs_root if specified. 111 | :type fs_include: list 112 | """ 113 | fs_file_path = os.path.join(fs_root, fs_path) 114 | _, fs_file_ext = os.path.splitext(fs_file_path) 115 | fs_root = fs_root if fs_root is None else os.path.dirname(fs_file_path) 116 | self.params.update({'fs_path': fs_path, 'fs_root': fs_root}) 117 | with open(fs_file_path) as stream: 118 | if 'yml' in fs_file_ext: 119 | # load_yaml initial structure 120 | data = yaml_safe_load(stream) 121 | names = {'R': data} 122 | self.names.update(names) 123 | data = self._process(data) 124 | elif 'json' in fs_file_ext: 125 | data = json.loads(stream.read()) 126 | names = {'R': data} 127 | self.names.update(names) 128 | data = self._process(data) 129 | else: 130 | data = '\n'.join(stream.readlines()) 131 | # Delete anything specific to this file so we can reuse the parser 132 | for k in ('fs_path', 'fs_root', 'R'): 133 | if k in self.params: 134 | del self.params[k] 135 | return data 136 | 137 | def parse(self, data): 138 | """ 139 | Main entry point to parse arbitary data type 140 | :param data: Input can be any data type such as dict, list, string, int 141 | :return: Parsed data 142 | """ 143 | if isinstance(data, dict): 144 | if type(data) == dict: 145 | warnings.warn('argument type is in dict, please use collections.OrderedDict for guaranteed order.') 146 | self.names.update(data) 147 | result = self._process(data) 148 | else: 149 | result = self.parse_expr(data) 150 | return result 151 | 152 | def parse_expr(self, expr: str): 153 | """ 154 | Parse an expression in string 155 | """ 156 | try: 157 | v = self._evaluator.eval(expr=expr) 158 | except SyntaxError as ex: 159 | v = expr 160 | # TODO: feature T2 161 | # print("Raised simpleeval exception {} for expression {}".format(type(ex), v)) 162 | self.errors.append([expr, ex]) 163 | except simpleeval.InvalidExpression as ex: 164 | v = expr 165 | # TODO: feature T2 166 | # print("Raised simpleeval exception {} for expression {}".format(type(ex), v)) 167 | # print("Raised simpleeval exception {} for expression {}".format(type(ex), v)) 168 | # print("Message: {}".format(ex)) 169 | self.errors.append(ex) 170 | except Exception as ex: 171 | v = expr 172 | # TODO: feature T2 173 | # print('Exception on expression: {}'.format(expr)) 174 | self.errors.append(ex) 175 | raise 176 | # TODO: feature T4: include this part of the classes so user could override 177 | v = filter_value(v) 178 | return v 179 | 180 | def _process(self, root): 181 | """ 182 | The main parsing function 183 | """ 184 | root_type = type(root) 185 | if root_type == dict or root_type == odict: 186 | root_keys = list(root.keys()) 187 | for k, v in root.items(): 188 | root[k] = self._process(v) 189 | if 'F.extend' in root_keys: 190 | root = self.fn_extend(root['F.extend'], root) 191 | if isinstance(root, dict): 192 | del root['F.extend'] 193 | if 'F.template' in root_keys: 194 | root = self.fn_template(root['F.template'], root) 195 | if isinstance(root, dict): 196 | del root['F.template'] 197 | if 'F.update' in root_keys: 198 | self.fn_update(root['F.update'], root) 199 | del root['F.update'] 200 | if 'F.foreach' in root_keys: 201 | for k in ('values', 'template'): 202 | if k not in root['F.foreach']: 203 | raise ValueError('F.foreach missing key: {}'.format(k)) 204 | self.fn_foreach(root['F.foreach'], root) 205 | del root['F.foreach'] 206 | elif root_type == list: 207 | for i, v in enumerate(root): 208 | root[i] = self._process(root=v) 209 | elif root_type == str: 210 | value = root 211 | if type(value) == str: 212 | value = self.parse_expr(root) 213 | return value 214 | return root 215 | 216 | def add_functions(self, funcs: dict): 217 | """ 218 | Add functions to the list of available parsing function. Funcs should 219 | be a dict whose keys are the name you would like the function to have, 220 | and whose value is a callable that maps to that name. The functions 221 | will be callable via F.name_of_func(args_go_here) 222 | """ 223 | 224 | def add_names(self, names: dict): 225 | """ 226 | Add names to the dictionary of names available when parsing. These 227 | names are accessible via the syntax R.path.to.name 228 | """ 229 | 230 | def generate_crypto_key(self): 231 | """ 232 | Generate a cryptographic key for encrypting data. Stores the key in 233 | self.params['ekey'] so it is accessible to encrypt parsing functions. 234 | Also returns the key 235 | """ 236 | etype = self.params.get('etype') 237 | if etype == 'fernet': 238 | key = Fernet.generate_key() 239 | else: 240 | key = None 241 | self.params['ekey'] = key 242 | return key 243 | 244 | def fn_str(self, val): 245 | return str(val) 246 | 247 | def fn_float(self, val): 248 | return float(val) 249 | 250 | def fn_int(self, val): 251 | return int(val) 252 | 253 | def fn_has(self, val, name): 254 | if isinstance(val, collections.Mapping): 255 | return val.get(name, False) is not False 256 | else: 257 | return name in val 258 | 259 | def fn_next(self, vals, default=None): 260 | vals = [vals] if type(vals) != list else vals 261 | val = next(iter(vals), default) 262 | return val 263 | 264 | def fn_join(self, vals, sep=' '): 265 | vals = [val for val in vals if val] 266 | return sep.join(vals) 267 | 268 | def fn_trim(self, val: str, cs: list = None): 269 | cs = cs if cs else ['/', ' '] 270 | for c in cs: 271 | val = val.strip(c) 272 | return val 273 | 274 | def fn_linspace(self, start, end, steps): 275 | delta = (end - start) / (steps - 1) 276 | return [start + delta * i for i in range(steps)] 277 | 278 | def fn_arange(self, start, end, delta): 279 | vals = [start] 280 | while vals[-1] + delta <= end: 281 | vals.append(vals[-1] + delta) 282 | return vals 283 | 284 | def fn_extend(self, val, val2): 285 | val = copy.deepcopy(val) 286 | type_val = type(val) 287 | type_val2 = type(val2) 288 | if type_val == list and type_val2 == list: 289 | val.extend(val2) 290 | elif type_val in [dict, odict, Munch2] and type_val2 in [dict, odict, Munch2]: 291 | for k, v in val2.items(): 292 | val[k] = v 293 | return val 294 | 295 | def fn_update(self, update, parent): 296 | def walk(u, p): 297 | tu, tp = type(u), type(p) 298 | if tu in [dict, odict, Munch2] and tp in [dict, odict, Munch2]: 299 | for k, v in u.items(): 300 | p[k] = walk(v, p.get(k, v)) 301 | return p 302 | else: 303 | return u 304 | 305 | walk(update, parent) 306 | 307 | def fn_encrypt(self, data): 308 | etype = self.params.get('etype', None) 309 | ekey = self.params.get('ekey', None) 310 | token = None 311 | if etype == 'fernet': 312 | f = Fernet(ekey) 313 | token = f.encrypt(data=str(data).encode()).decode() 314 | return token 315 | 316 | def fn_decrypt(self, data): 317 | etype = self.params.get('etype', None) 318 | ekey = self.params.get('ekey', None) 319 | message = None 320 | if etype == 'fernet': 321 | f = Fernet(ekey) 322 | message = f.decrypt(token=str(data).encode()).decode() 323 | return message 324 | 325 | def fn_inc(self, fs_path, fs_root: str = None): 326 | fs_root = fs_root if fs_root else self.params['fs_root'] 327 | # Make sure to pass on any modified options to the sub parser 328 | sub_parser = Parser(params=self.params) 329 | data = sub_parser.load(fs_path=fs_path, fs_root=fs_root) 330 | return data 331 | 332 | def fn_foreach(self, foreach, parent): 333 | template = foreach['template'] 334 | if not isinstance(template, dict): 335 | raise ValueError('template item of F.foreach must be a dict') 336 | for i, v in enumerate(foreach['values']): 337 | self.names.update({'loop': {'index': i, 'value': v, 338 | 'length': len(foreach['values'])}}) 339 | result = {} 340 | for key, val in template.items(): 341 | pkey = self.parse_expr(key) 342 | pval = self._process(copy.copy(val)) 343 | result[pkey] = pval 344 | parent.update(result) 345 | # remove this specific foreach loop info from names dict so we don't 346 | # break any subsequent foreach loops 347 | del self.names['loop'] 348 | 349 | def fn_template(self, template: str, root=None): 350 | engine = Template(template) 351 | obj_str = engine.render(**self.names) 352 | result = obj_str 353 | 354 | # TODO: feature T2 355 | obj = utils.yaml_safe_load(obj_str) 356 | if root and isinstance(obj, dict): 357 | result = self.fn_extend(root, obj) 358 | return result 359 | 360 | 361 | class ParserPlugin(object): 362 | pass 363 | -------------------------------------------------------------------------------- /conff/test_parser.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | import shutil 4 | from distutils.dir_util import copy_tree 5 | from unittest import TestCase 6 | import yaml 7 | import conff 8 | from conff import utils 9 | 10 | 11 | class ConffTestCase(TestCase): 12 | def setUp(self): 13 | super(ConffTestCase, self).setUp() 14 | # set path 15 | current_path = os.path.dirname(os.path.abspath(__file__)) 16 | test_data_path = os.path.join(current_path, 'data') 17 | self.test_data_path = tempfile.mkdtemp() 18 | copy_tree(test_data_path, self.test_data_path) 19 | self.maxDiff = None 20 | 21 | def tearDown(self): 22 | shutil.rmtree(self.test_data_path) 23 | 24 | def get_test_data_path(self, fs_path: str): 25 | return os.path.join(self.test_data_path, fs_path) 26 | 27 | def test_simple_load_yaml(self): 28 | fs_path = self.get_test_data_path('test_config_01.yml') 29 | p = conff.Parser() 30 | data = p.load(fs_path=fs_path) 31 | data = data if data else {} 32 | self.assertDictEqual(data, {'test_1': 'test_1', 'test_2': ''}) 33 | 34 | def test_ext_up_load_yaml(self): 35 | fs_path = self.get_test_data_path('test_config_04.yml') 36 | p = conff.Parser() 37 | data = p.load(fs_path=fs_path) 38 | data = data if data else {} 39 | self.assertDictEqual(data, { 40 | "test_13": {"test_13_1": 1, "test_13_2": 2, "test_13_3": 3, "test_13_5": {"test_13_5_1": 1}, 41 | "test_13_6": {"test_13_6_1": 1}}, 42 | "test_14": {"test_13_1": 11, "test_13_2": 2, "test_13_3": 3, "test_13_5": 5, 43 | "test_13_6": {"test_13_6_1": 1, "test_13_6_2": {"test_13_6_2_1": 1, "test_13_6_2_2": 2}}, 44 | "test_13_4": 4}}) 45 | 46 | def test_load_json(self): 47 | fs_path = self.get_test_data_path('test_config_01.json') 48 | p = conff.Parser() 49 | data = p.load(fs_path=fs_path) 50 | data = data if data else {} 51 | self.assertDictEqual(data, {'test_1': 1, 'test_2': 2}) 52 | 53 | def test_complex_load_yml(self): 54 | p = conff.Parser() 55 | fs_path = self.get_test_data_path('test_config_02.yml') 56 | p.generate_crypto_key() 57 | data = p.load(fs_path=fs_path) 58 | data = data if data else {} 59 | # test simple types 60 | self.assertEqual(data.get('test_1'), 'test_1') 61 | self.assertEqual(data.get('test_2'), 2) 62 | self.assertListEqual(data.get('test_3'), ['test_3', 3]) 63 | self.assertDictEqual(data.get('test_4'), {'test_4_1': 'test_4_1'}) 64 | self.assertDictEqual(data.get('test_5'), {'test_5_1': 'test_5_1', 'test_5_2': {'test_5_2_1': 'test_5_2_1'}}) 65 | # test expression 66 | self.assertEqual(data.get('test_6'), 'test_6') 67 | # test extends 68 | self.assertListEqual(data.get('test_7', {}).get('test_7_1'), [1, 2]) 69 | self.assertListEqual(data.get('test_7', {}).get('test_7_2'), [1, 2]) 70 | self.assertDictEqual(data.get('test_7', {}).get('test_7_3'), {'data2_1': 1, 'data2_2': 2, 'data2_3': 3}) 71 | # test complex extends 72 | self.assertDictEqual(data.get('test_8'), {'data2_1': 1, 'data2_2': '2a', 'data2_3': 3, 'data2_4': 4}) 73 | # test complex expressions 74 | self.assertListEqual(data.get('test_9', {}).get('test_9_1'), [True, False, True, False]) 75 | self.assertListEqual(data.get('test_9', {}).get('test_9_2'), [1, 'RO']) 76 | self.assertEqual(data.get('test_9', {}).get('test_9_3'), '1 2 3') 77 | self.assertEqual(data.get('test_9', {}).get('test_9_4'), 'ro/ro') 78 | # test error expressions 79 | self.assertEqual(data.get('test_10'), 'F.no_exist()') 80 | # test encryption 81 | self.assertEqual(data.get('test_11'), 'test_11') 82 | # test importing 83 | self.assertDictEqual(data.get('test_12'), {'test_1': 'test_1', 'test_2': ''}) 84 | # test update 85 | data_test_13 = {'test_13_1': 1, 'test_13_2': 2, 'test_13_3': 3, 'test_13_5': {'test_13_5_1': 1}, 86 | 'test_13_6': {'test_13_6_1': 1}} 87 | self.assertDictEqual(data.get('test_13'), data_test_13) 88 | # test extend + update 89 | data_test_14 = {'test_13_1': 11, 'test_13_2': 2, 'test_13_3': 3, 'test_13_5': 5, 90 | 'test_13_6': {'test_13_6_1': 1, 'test_13_6_2': {'test_13_6_2_1': 1, 'test_13_6_2_2': 2}}, 91 | 'test_13_4': 4} 92 | self.assertDictEqual(data.get('test_14'), data_test_14) 93 | # test foreach with linspace 94 | data_test_15 = {'test0': {'value': 0, 'length': 3}, 95 | 'test1': {'value': 3, 'length': 3}, 96 | 'test2': {'value': 6, 'length': 3}} 97 | self.assertDictEqual(data.get('test_15'), data_test_15) 98 | # test foreach with arange. Should get same result as above 99 | data_test_16 = data_test_15 100 | self.assertDictEqual(data.get('test_16'), data_test_16) 101 | # test foreach with arange. Testing behavior of arange 102 | data_test_17 = {'test0': {'value': 0, 'length': 2}, 103 | 'test1': {'value': 4, 'length': 2}} 104 | self.assertDictEqual(data.get('test_17'), data_test_17) 105 | data_test_18 = {'test_18_1': 3} 106 | self.assertDictEqual(data.get('test_18'), data_test_18) 107 | 108 | def test_error_load_yaml(self): 109 | p = conff.Parser() 110 | fs_path = self.get_test_data_path('test_config_03.yml') 111 | with self.assertRaises(TypeError) as context: 112 | data = p.load(fs_path=fs_path) 113 | 114 | def test_error_foreach(self): 115 | p = conff.Parser() 116 | fs_path = self.get_test_data_path('malformed_foreach_01.yml') 117 | with self.assertRaises(ValueError): 118 | p.load(fs_path=fs_path) 119 | fs_path = self.get_test_data_path('malformed_foreach_02.yml') 120 | with self.assertRaises(ValueError): 121 | p.load(fs_path=fs_path) 122 | 123 | def test_parse(self): 124 | p = conff.Parser() 125 | data = p.parse('{"a": "a", "b": "1/0"}') 126 | self.assertDictEqual(data, {'a': 'a', 'b': '1/0'}) 127 | data = p.parse(utils.odict([('a', 'a'), ('b', '1 + 2')])) 128 | self.assertDictEqual(data, {'a': 'a', 'b': 3}) 129 | 130 | def test_parse_with_fns(self): 131 | def fn_add(a, b): 132 | return a + b 133 | 134 | fns = {'add': fn_add, 'test': {'add': fn_add}} 135 | p = conff.Parser(fns=fns) 136 | data = p.parse('{"a": "a", "b": "1/0", "c": F.add(1, 2), "d": F.test.add(2, 2)}') 137 | self.assertDictEqual(data, {'a': 'a', 'b': '1/0', 'c': 3, 'd': 4}) 138 | 139 | def test_parse_dict_with_names(self): 140 | names = {'c': 1, 'd': 2} 141 | p = conff.Parser(names=names) 142 | data = p.parse(utils.odict([('a', 'a'), ('b', 'c + d')])) 143 | self.assertDictEqual(data, {'a': 'a', 'b': 3}) 144 | 145 | def test_missing_operators(self): 146 | names = {'c': 1, 'd': 2} 147 | p = conff.Parser(names=names, params={'simpleeval': {'operators': {'not': 'an_operator'}}}) 148 | with self.assertRaises(KeyError) as context: 149 | p.parse(utils.odict([('a', 'a'), ('b', 'c + d')])) 150 | self.assertTrue("" in str(context.exception)) 151 | 152 | def test_generate_crypto(self): 153 | p = conff.Parser() 154 | del p.params['etype'] 155 | key = p.generate_crypto_key() 156 | self.assertTrue(key is None) 157 | self.assertTrue(p.params['ekey'] is None) 158 | p.params['etype'] = 'nonsense' 159 | key = p.generate_crypto_key() 160 | self.assertTrue(key is None) 161 | self.assertTrue(p.params['ekey'] is None) 162 | 163 | def test_encryption(self): 164 | # generate key, save it somewhere safe 165 | names = {'R': {'_': {'etype': 'fernet'}}} 166 | ekey = 'FOb7DBRftamqsyRFIaP01q57ZLZZV6MVB2xg1Cg_E7g=' 167 | names = {'R': {'_': {'etype': 'fernet', 'ekey': ekey}}} 168 | original_value = 'ACCESSSECRETPLAIN1234' 169 | encrypted_value = conff.encrypt(names)(original_value) 170 | # decrypt data 171 | value = conff.decrypt(names)(encrypted_value) 172 | self.assertEqual(original_value, value, 'Value mismatch') 173 | 174 | def test_sample(self): 175 | # nose2 conff.test.ConffTestCase.test_sample 176 | fs_path = self.get_test_data_path('sample_config_01.yml') 177 | with open(fs_path) as stream: 178 | r1 = yaml.safe_load(stream) 179 | fs_path = self.get_test_data_path('sample_config_02.yml') 180 | ekey = 'FOb7DBRftamqsyRFIaP01q57ZLZZV6MVB2xg1Cg_E7g=' 181 | errors = [] 182 | r2 = conff.load(fs_path=fs_path, params={'ekey': ekey}, errors=errors) 183 | fs_path = self.get_test_data_path('sample_config_03.yml') 184 | r3 = conff.load(fs_path=fs_path, params={'ekey': ekey}) 185 | self.assertDictEqual(r1['job'], r2['job'], 'Mismatch value') 186 | self.assertDictEqual(r2['job'], r3['job'], 'Mismatch value') 187 | 188 | def test_object(self): 189 | # nose2 conff.test.ConffTestCase.test_object 190 | # TODO: add test when trying to combine config as object with conff 191 | # test update 192 | class Test(object): 193 | test = None 194 | 195 | data = Test() 196 | conff.update(data, {'test': 'test'}) 197 | self.assertEqual('test', data.test, 'Value mismatch') 198 | 199 | def test_warning(self): 200 | p = conff.Parser() 201 | with self.assertWarns(Warning): 202 | data = p.parse({'a': 'a', 'b': '1 + 2'}) 203 | self.assertDictEqual(data, {'a': 'a', 'b': 3}) 204 | 205 | def test_update_recursive(self): 206 | fns = {'F': conff.update({'a': 1}, {'b': {'c': 2}})} 207 | self.assertDictEqual(fns, {'F': {'a': 1, 'b': {'c': 2}}}) 208 | 209 | def test_parse_old(self): 210 | data = conff.parse('{"a": "a", "b": "1/0"}') 211 | self.assertDictEqual(data, {'a': 'a', 'b': '1/0'}) 212 | data = conff.parse(utils.odict([('a', 'a'), ('b', '1 + 2')])) 213 | self.assertDictEqual(data, {'a': 'a', 'b': 3}) 214 | 215 | def test_encryption_old(self): 216 | # generate key, save it somewhere safe 217 | names = {'R': {'_': {'etype': 'fernet'}}} 218 | etype = conff.generate_key(names)() 219 | ekey = 'FOb7DBRftamqsyRFIaP01q57ZLZZV6MVB2xg1Cg_E7g=' 220 | names = {'R': {'_': {'etype': 'fernet', 'ekey': ekey}}} 221 | original_value = 'ACCESSSECRETPLAIN1234' 222 | encrypted_value = conff.encrypt(names)(original_value) 223 | # decrypt data 224 | value = conff.decrypt(names)(encrypted_value) 225 | self.assertEqual(original_value, value, 'Value mismatch') 226 | 227 | def test_fn_template(self): 228 | p = conff.Parser(names={'test': 1}) 229 | fs_path = self.get_test_data_path('test_config_05.yml') 230 | data = p.load(fs_path) 231 | data = data if data else {} 232 | # test: simple value 233 | self.assertEqual(data.get('test_1'), 1) 234 | # test: template as string, it is seamless names from input (test) and template (test_1) 235 | self.assertEqual(data.get('test_2'), '2') 236 | # test: template as file (borrowing F.inc capabilities), if test_tpl_01.tpl is {{1 + 2}} 237 | self.assertEqual(data.get('test_3'), '3') 238 | # test: this where attaching more complex object 239 | data_test_4 = { 240 | "test_4_0": [3, 4], "test_4_1": 1, "test_4_2": 2, "test_4_3": 3, "test_4_4": 4, "test_4_5": 5, 241 | "test_4_6": 6 242 | } 243 | self.assertDictEqual(data.get('test_4'), data_test_4) 244 | -------------------------------------------------------------------------------- /conff/utils.py: -------------------------------------------------------------------------------- 1 | from munch import Munch 2 | from collections import OrderedDict as odict 3 | 4 | 5 | class Munch2(Munch): 6 | """ 7 | Provide easy way to access item in object by dot-notation. 8 | Example: 9 | obj = {'item': 'value'} 10 | # old way 11 | value = obj.get('item') 12 | # with munch 13 | value = obj.item 14 | """ 15 | pass 16 | 17 | 18 | def update_recursive(d, u): 19 | """ 20 | Update dictionary recursively. It traverse any object implements 21 | "collections.Mapping", anything else, it overwrites the original value. 22 | :param d: Original dictionary to be updated 23 | :param u: Value dictionary 24 | :return: Updated dictionary after merged 25 | """ 26 | for k, v in u.items(): 27 | if isinstance(d, dict): 28 | d2 = d.get(k, {}) 29 | else: 30 | d2 = getattr(d, k, object()) 31 | 32 | if isinstance(v, dict): 33 | r = update_recursive(d2, v) 34 | u2 = r 35 | else: 36 | u2 = u[k] 37 | 38 | if isinstance(d, dict): 39 | d[k] = u2 40 | else: 41 | setattr(d, k, u2) 42 | return d 43 | 44 | 45 | def yaml_safe_load(stream): 46 | import yaml 47 | from yaml.resolver import BaseResolver 48 | 49 | def ordered_load(stream, loader_cls): 50 | class OrderedLoader(loader_cls): 51 | pass 52 | 53 | def construct_mapping(loader, node): 54 | loader.flatten_mapping(node) 55 | return odict(loader.construct_pairs(node)) 56 | 57 | OrderedLoader.add_constructor(BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) 58 | return yaml.load(stream, OrderedLoader) 59 | 60 | return ordered_load(stream, yaml.SafeLoader) 61 | 62 | 63 | def filter_value(value): 64 | if isinstance(value, str): 65 | if value == '[empty]': 66 | value = '' 67 | if isinstance(value, str): 68 | value = value.strip() 69 | return value 70 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | nose2 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pyyaml 2 | munch 3 | simpleeval 4 | cryptography 5 | jinja2 -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.rst 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from setuptools import setup 4 | 5 | 6 | def get_requirements(r: str): 7 | try: # for pip >= 10 8 | from pip._internal.req import parse_requirements 9 | except ImportError: # for pip <= 9.0.3 10 | from pip.req import parse_requirements 11 | 12 | # parse_requirements() returns generator of pip.req.InstallRequirement objects 13 | if os.path.exists(r): 14 | install_reqs = parse_requirements(r, session=pkg) 15 | return install_reqs 16 | return [] 17 | 18 | 19 | __version__ = '0.5.0' 20 | pkg = 'conff' 21 | rs = [str(ir.req) for ir in get_requirements('requirements.txt')] 22 | setup( 23 | name=pkg, 24 | packages=[pkg], 25 | include_package_data=True, 26 | version=__version__, 27 | description='Simple config parser with evaluator library.', 28 | long_description=open('README.rst', 'r').read(), 29 | author='Robertus Johansyah', 30 | author_email='kororola@gmail.com', 31 | url='https://github.com/kororo/conff', 32 | download_url='https://github.com/kororo/conff/tarball/' + __version__, 33 | keywords=['config', 'parser', 'expression', 'parse', 'eval'], 34 | test_suite='conff.test', 35 | use_2to3=True, 36 | classifiers=[ 37 | 'Intended Audience :: Developers', 38 | 'License :: OSI Approved :: MIT License', 39 | 'Natural Language :: English', 40 | 'Operating System :: MacOS :: MacOS X', 41 | 'Operating System :: POSIX', 42 | 'Operating System :: POSIX :: BSD', 43 | 'Operating System :: POSIX :: Linux', 44 | 'Operating System :: Microsoft :: Windows', 45 | 'Programming Language :: Python', 46 | 'Programming Language :: Python :: 3.5', 47 | 'Programming Language :: Python :: 3.6', 48 | 'Topic :: Software Development :: Libraries :: Python Modules', 49 | 'Programming Language :: Python', 50 | ], 51 | python_requires='>=3.5', 52 | install_requires=rs 53 | ) 54 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | import conff 2 | p = conff.Parser() 3 | r = p.parse('F.template("{{ 1 + 2 }}")') 4 | print(r) -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py{35,36} 3 | skip_missing_interpreters = True 4 | [testenv] 5 | commands = 6 | python -m nose2 -v --with-coverage 7 | deps = 8 | -Ur{toxinidir}/requirements.txt 9 | -Ur{toxinidir}/requirements-test.txt 10 | passenv = 11 | PYTHON* 12 | -------------------------------------------------------------------------------- /unittest.cfg: -------------------------------------------------------------------------------- 1 | [coverage] 2 | always-on = False --------------------------------------------------------------------------------