├── .appveyor.yml ├── .coveragerc ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── MANIFEST.in ├── Makefile ├── Pipfile ├── Pipfile.lock ├── README.md ├── scripts └── generate_md_help.py ├── setup.cfg ├── setup.py ├── src └── csv_schema │ ├── __init__.py │ ├── _version.py │ ├── cli.py │ ├── commands │ ├── __init__.py │ ├── generate_config │ │ ├── __init__.py │ │ ├── cli.py │ │ └── generate_config.py │ ├── validate_config │ │ ├── __init__.py │ │ ├── cli.py │ │ └── validate_config.py │ └── validate_csv │ │ ├── __init__.py │ │ ├── cli.py │ │ └── validate_csv.py │ └── core │ ├── __init__.py │ ├── exit_codes.py │ ├── models │ ├── __init__.py │ ├── base_column.py │ ├── base_config_object.py │ ├── column_types.py │ ├── config_property.py │ ├── decimal_column.py │ ├── enum_column.py │ ├── integer_column.py │ ├── schema_config.py │ ├── schema_config_filename.py │ └── string_column.py │ └── utils.py └── tests ├── __init__.py ├── conftest.py └── csv_schema ├── __init__.py ├── commands ├── __init__.py ├── generate_config │ ├── __init__.py │ ├── test_cli.py │ └── test_generate_config.py ├── validate_config │ ├── __init__.py │ ├── test_cli.py │ └── test_validate_config.py └── validate_csv │ ├── __init__.py │ ├── test_cli.py │ └── test_validate_csv.py └── core └── models ├── test_base_column.py ├── test_base_config_object.py ├── test_config_property.py ├── test_decimal_column.py ├── test_enum_column.py ├── test_integer_column.py ├── test_schema_config.py └── test_string_column.py /.appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | PYTHONIOENCODING: UTF-8 3 | LOG_LEVEL: DEBUG 4 | matrix: 5 | - PYTHON: "C:\\Python37-x64" 6 | 7 | install: 8 | - set PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH% 9 | - pip install pipenv 10 | - pipenv --python=%PYTHON%\\python.exe 11 | - pipenv lock -r >> requirements.txt 12 | - pipenv lock -r --dev >> requirements.txt 13 | - pip install -r requirements.txt 14 | 15 | build: off 16 | 17 | test_script: 18 | - python --version 19 | - pytest -v -s --cov 20 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = 4 | src 5 | omit = 6 | */tests/* 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .vscode 3 | .venv 4 | .DS_Store 5 | _ignore 6 | 7 | # Source for the following rules: https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | 31 | # Unit test / coverage reports 32 | htmlcov/ 33 | .tox/ 34 | .nox/ 35 | .coverage 36 | .coverage.* 37 | .cache 38 | nosetests.xml 39 | coverage.xml 40 | *.cover 41 | .hypothesis/ 42 | .pytest_cache/ 43 | 44 | # pyenv 45 | .python-version 46 | 47 | # Testing files 48 | csv-schema*.json 49 | csv-schema*.csv 50 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | matrix: 4 | include: 5 | - os: linux 6 | python: 3.7 7 | - os: osx 8 | language: generic 9 | env: PYENV_VERSION=3.7.4 10 | install: 11 | - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew update ; fi 12 | - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew outdated pyenv || brew upgrade pyenv ; fi 13 | - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then pyenv install $PYENV_VERSION ; fi 14 | - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then ~/.pyenv/versions/$PYENV_VERSION/bin/python -m venv .venv ; fi 15 | - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then source .venv/bin/activate ; fi 16 | - python --version 17 | - python -m pip install -U pip 18 | - python -m pip install -U pipenv 19 | - pipenv install --dev 20 | - pip install coveralls 21 | before_script: 22 | - python -m coverage erase 23 | script: 24 | - python --version 25 | - python -m pytest -v -s --cov 26 | after_success: 27 | - python -m coveralls 28 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | ## Version 1.1 (2022-01-24) 4 | 5 | ### Added 6 | 7 | - Do not error when an optional column (required=False) is missing from the CSV file. 8 | 9 | ## Version 1.0 (2020-07-21) 10 | 11 | ### Added 12 | 13 | - Initial release. 14 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: pip_install 2 | pip_install: 3 | pipenv install --dev 4 | 5 | 6 | .PHONY: test 7 | test: 8 | pytest -v --cov --cov-report=term --cov-report=html 9 | 10 | 11 | .PHONY: build 12 | build: clean 13 | python setup.py sdist 14 | python setup.py bdist_wheel 15 | twine check dist/* 16 | 17 | 18 | .PHONY: clean 19 | clean: 20 | rm -rf ./build/* 21 | rm -rf ./dist/* 22 | rm -rf ./htmlcov 23 | 24 | 25 | .PHONY: install_local 26 | install_local: 27 | pip install -e . 28 | 29 | 30 | .PHONY: install_test 31 | install_test: 32 | pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple csv-schema 33 | 34 | 35 | .PHONY: publish_test 36 | publish_test: build 37 | twine upload --repository-url https://test.pypi.org/legacy/ dist/* 38 | 39 | 40 | .PHONY: publish 41 | publish: build 42 | twine upload dist/* 43 | 44 | 45 | .PHONY: uninstall 46 | uninstall: 47 | pip uninstall -y csv-schema 48 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | urllib3 = ">=1.26.5" 8 | py = ">=1.10.0" 9 | Pygments = ">=2.7.4" 10 | bleach = ">=3.3.0" 11 | cryptography = ">=3.2" 12 | pytest = "*" 13 | pytest-cov = "*" 14 | coverage = "*" 15 | twine = "*" 16 | wheel = "*" 17 | autopep8 = "*" 18 | pytest-mock = "*" 19 | pytest-pylint = "*" 20 | 21 | [packages] 22 | 23 | [requires] 24 | python_version = "3.7" 25 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "7ad84b5b875a9cd165b3324eab9fea973948a424bbbdc8e17fda429d916cbf72" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.7" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": {}, 19 | "develop": { 20 | "astroid": { 21 | "hashes": [ 22 | "sha256:1efdf4e867d4d8ba4a9f6cf9ce07cd182c4c41de77f23814feb27ca93ca9d877", 23 | "sha256:506daabe5edffb7e696ad82483ad0228245a9742ed7d2d8c9cdb31537decf9f6" 24 | ], 25 | "version": "==2.9.3" 26 | }, 27 | "attrs": { 28 | "hashes": [ 29 | "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", 30 | "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" 31 | ], 32 | "version": "==21.4.0" 33 | }, 34 | "autopep8": { 35 | "hashes": [ 36 | "sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979", 37 | "sha256:ed77137193bbac52d029a52c59bec1b0629b5a186c495f1eb21b126ac466083f" 38 | ], 39 | "index": "pypi", 40 | "version": "==1.6.0" 41 | }, 42 | "bleach": { 43 | "hashes": [ 44 | "sha256:0900d8b37eba61a802ee40ac0061f8c2b5dee29c1927dd1d233e075ebf5a71da", 45 | "sha256:4d2651ab93271d1129ac9cbc679f524565cc8a1b791909c4a51eac4446a15994" 46 | ], 47 | "index": "pypi", 48 | "version": "==4.1.0" 49 | }, 50 | "certifi": { 51 | "hashes": [ 52 | "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872", 53 | "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569" 54 | ], 55 | "version": "==2021.10.8" 56 | }, 57 | "cffi": { 58 | "hashes": [ 59 | "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3", 60 | "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2", 61 | "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636", 62 | "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20", 63 | "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728", 64 | "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27", 65 | "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66", 66 | "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443", 67 | "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0", 68 | "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7", 69 | "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39", 70 | "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605", 71 | "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a", 72 | "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37", 73 | "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029", 74 | "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139", 75 | "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc", 76 | "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df", 77 | "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14", 78 | "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880", 79 | "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2", 80 | "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a", 81 | "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e", 82 | "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474", 83 | "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024", 84 | "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8", 85 | "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0", 86 | "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e", 87 | "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a", 88 | "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e", 89 | "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032", 90 | "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6", 91 | "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e", 92 | "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b", 93 | "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e", 94 | "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954", 95 | "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962", 96 | "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c", 97 | "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4", 98 | "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55", 99 | "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962", 100 | "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023", 101 | "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c", 102 | "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6", 103 | "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8", 104 | "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382", 105 | "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7", 106 | "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc", 107 | "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997", 108 | "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796" 109 | ], 110 | "version": "==1.15.0" 111 | }, 112 | "charset-normalizer": { 113 | "hashes": [ 114 | "sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd", 115 | "sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455" 116 | ], 117 | "markers": "python_version >= '3'", 118 | "version": "==2.0.10" 119 | }, 120 | "colorama": { 121 | "hashes": [ 122 | "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", 123 | "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" 124 | ], 125 | "version": "==0.4.4" 126 | }, 127 | "coverage": { 128 | "hashes": [ 129 | "sha256:01774a2c2c729619760320270e42cd9e797427ecfddd32c2a7b639cdc481f3c0", 130 | "sha256:03b20e52b7d31be571c9c06b74746746d4eb82fc260e594dc662ed48145e9efd", 131 | "sha256:0a7726f74ff63f41e95ed3a89fef002916c828bb5fcae83b505b49d81a066884", 132 | "sha256:1219d760ccfafc03c0822ae2e06e3b1248a8e6d1a70928966bafc6838d3c9e48", 133 | "sha256:13362889b2d46e8d9f97c421539c97c963e34031ab0cb89e8ca83a10cc71ac76", 134 | "sha256:174cf9b4bef0db2e8244f82059a5a72bd47e1d40e71c68ab055425172b16b7d0", 135 | "sha256:17e6c11038d4ed6e8af1407d9e89a2904d573be29d51515f14262d7f10ef0a64", 136 | "sha256:215f8afcc02a24c2d9a10d3790b21054b58d71f4b3c6f055d4bb1b15cecce685", 137 | "sha256:22e60a3ca5acba37d1d4a2ee66e051f5b0e1b9ac950b5b0cf4aa5366eda41d47", 138 | "sha256:2641f803ee9f95b1f387f3e8f3bf28d83d9b69a39e9911e5bfee832bea75240d", 139 | "sha256:276651978c94a8c5672ea60a2656e95a3cce2a3f31e9fb2d5ebd4c215d095840", 140 | "sha256:3f7c17209eef285c86f819ff04a6d4cbee9b33ef05cbcaae4c0b4e8e06b3ec8f", 141 | "sha256:3feac4084291642165c3a0d9eaebedf19ffa505016c4d3db15bfe235718d4971", 142 | "sha256:49dbff64961bc9bdd2289a2bda6a3a5a331964ba5497f694e2cbd540d656dc1c", 143 | "sha256:4e547122ca2d244f7c090fe3f4b5a5861255ff66b7ab6d98f44a0222aaf8671a", 144 | "sha256:5829192582c0ec8ca4a2532407bc14c2f338d9878a10442f5d03804a95fac9de", 145 | "sha256:5d6b09c972ce9200264c35a1d53d43ca55ef61836d9ec60f0d44273a31aa9f17", 146 | "sha256:600617008aa82032ddeace2535626d1bc212dfff32b43989539deda63b3f36e4", 147 | "sha256:619346d57c7126ae49ac95b11b0dc8e36c1dd49d148477461bb66c8cf13bb521", 148 | "sha256:63c424e6f5b4ab1cf1e23a43b12f542b0ec2e54f99ec9f11b75382152981df57", 149 | "sha256:6dbc1536e105adda7a6312c778f15aaabe583b0e9a0b0a324990334fd458c94b", 150 | "sha256:6e1394d24d5938e561fbeaa0cd3d356207579c28bd1792f25a068743f2d5b282", 151 | "sha256:86f2e78b1eff847609b1ca8050c9e1fa3bd44ce755b2ec30e70f2d3ba3844644", 152 | "sha256:8bdfe9ff3a4ea37d17f172ac0dff1e1c383aec17a636b9b35906babc9f0f5475", 153 | "sha256:8e2c35a4c1f269704e90888e56f794e2d9c0262fb0c1b1c8c4ee44d9b9e77b5d", 154 | "sha256:92b8c845527eae547a2a6617d336adc56394050c3ed8a6918683646328fbb6da", 155 | "sha256:9365ed5cce5d0cf2c10afc6add145c5037d3148585b8ae0e77cc1efdd6aa2953", 156 | "sha256:9a29311bd6429be317c1f3fe4bc06c4c5ee45e2fa61b2a19d4d1d6111cb94af2", 157 | "sha256:9a2b5b52be0a8626fcbffd7e689781bf8c2ac01613e77feda93d96184949a98e", 158 | "sha256:a4bdeb0a52d1d04123b41d90a4390b096f3ef38eee35e11f0b22c2d031222c6c", 159 | "sha256:a9c8c4283e17690ff1a7427123ffb428ad6a52ed720d550e299e8291e33184dc", 160 | "sha256:b637c57fdb8be84e91fac60d9325a66a5981f8086c954ea2772efe28425eaf64", 161 | "sha256:bf154ba7ee2fd613eb541c2bc03d3d9ac667080a737449d1a3fb342740eb1a74", 162 | "sha256:c254b03032d5a06de049ce8bca8338a5185f07fb76600afff3c161e053d88617", 163 | "sha256:c332d8f8d448ded473b97fefe4a0983265af21917d8b0cdcb8bb06b2afe632c3", 164 | "sha256:c7912d1526299cb04c88288e148c6c87c0df600eca76efd99d84396cfe00ef1d", 165 | "sha256:cfd9386c1d6f13b37e05a91a8583e802f8059bebfccde61a418c5808dea6bbfa", 166 | "sha256:d5d2033d5db1d58ae2d62f095e1aefb6988af65b4b12cb8987af409587cc0739", 167 | "sha256:dca38a21e4423f3edb821292e97cec7ad38086f84313462098568baedf4331f8", 168 | "sha256:e2cad8093172b7d1595b4ad66f24270808658e11acf43a8f95b41276162eb5b8", 169 | "sha256:e3db840a4dee542e37e09f30859f1612da90e1c5239a6a2498c473183a50e781", 170 | "sha256:edcada2e24ed68f019175c2b2af2a8b481d3d084798b8c20d15d34f5c733fa58", 171 | "sha256:f467bbb837691ab5a8ca359199d3429a11a01e6dfb3d9dcc676dc035ca93c0a9", 172 | "sha256:f506af4f27def639ba45789fa6fde45f9a217da0be05f8910458e4557eed020c", 173 | "sha256:f614fc9956d76d8a88a88bb41ddc12709caa755666f580af3a688899721efecd", 174 | "sha256:f9afb5b746781fc2abce26193d1c817b7eb0e11459510fba65d2bd77fe161d9e", 175 | "sha256:fb8b8ee99b3fffe4fd86f4c81b35a6bf7e4462cba019997af2fe679365db0c49" 176 | ], 177 | "index": "pypi", 178 | "version": "==6.2" 179 | }, 180 | "cryptography": { 181 | "hashes": [ 182 | "sha256:0a817b961b46894c5ca8a66b599c745b9a3d9f822725221f0e0fe49dc043a3a3", 183 | "sha256:2d87cdcb378d3cfed944dac30596da1968f88fb96d7fc34fdae30a99054b2e31", 184 | "sha256:30ee1eb3ebe1644d1c3f183d115a8c04e4e603ed6ce8e394ed39eea4a98469ac", 185 | "sha256:391432971a66cfaf94b21c24ab465a4cc3e8bf4a939c1ca5c3e3a6e0abebdbcf", 186 | "sha256:39bdf8e70eee6b1c7b289ec6e5d84d49a6bfa11f8b8646b5b3dfe41219153316", 187 | "sha256:4caa4b893d8fad33cf1964d3e51842cd78ba87401ab1d2e44556826df849a8ca", 188 | "sha256:53e5c1dc3d7a953de055d77bef2ff607ceef7a2aac0353b5d630ab67f7423638", 189 | "sha256:596f3cd67e1b950bc372c33f1a28a0692080625592ea6392987dba7f09f17a94", 190 | "sha256:5d59a9d55027a8b88fd9fd2826c4392bd487d74bf628bb9d39beecc62a644c12", 191 | "sha256:6c0c021f35b421ebf5976abf2daacc47e235f8b6082d3396a2fe3ccd537ab173", 192 | "sha256:73bc2d3f2444bcfeac67dd130ff2ea598ea5f20b40e36d19821b4df8c9c5037b", 193 | "sha256:74d6c7e80609c0f4c2434b97b80c7f8fdfaa072ca4baab7e239a15d6d70ed73a", 194 | "sha256:7be0eec337359c155df191d6ae00a5e8bbb63933883f4f5dffc439dac5348c3f", 195 | "sha256:94ae132f0e40fe48f310bba63f477f14a43116f05ddb69d6fa31e93f05848ae2", 196 | "sha256:bb5829d027ff82aa872d76158919045a7c1e91fbf241aec32cb07956e9ebd3c9", 197 | "sha256:ca238ceb7ba0bdf6ce88c1b74a87bffcee5afbfa1e41e173b1ceb095b39add46", 198 | "sha256:ca28641954f767f9822c24e927ad894d45d5a1e501767599647259cbf030b903", 199 | "sha256:e0344c14c9cb89e76eb6a060e67980c9e35b3f36691e15e1b7a9e58a0a6c6dc3", 200 | "sha256:ebc15b1c22e55c4d5566e3ca4db8689470a0ca2babef8e3a9ee057a8b82ce4b1", 201 | "sha256:ec63da4e7e4a5f924b90af42eddf20b698a70e58d86a72d943857c4c6045b3ee" 202 | ], 203 | "index": "pypi", 204 | "version": "==36.0.1" 205 | }, 206 | "docutils": { 207 | "hashes": [ 208 | "sha256:23010f129180089fbcd3bc08cfefccb3b890b0050e1ca00c867036e9d161b98c", 209 | "sha256:679987caf361a7539d76e584cbeddc311e3aee937877c87346f31debc63e9d06" 210 | ], 211 | "version": "==0.18.1" 212 | }, 213 | "idna": { 214 | "hashes": [ 215 | "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", 216 | "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" 217 | ], 218 | "markers": "python_version >= '3'", 219 | "version": "==3.3" 220 | }, 221 | "importlib-metadata": { 222 | "hashes": [ 223 | "sha256:899e2a40a8c4a1aec681feef45733de8a6c58f3f6a0dbed2eb6574b4387a77b6", 224 | "sha256:951f0d8a5b7260e9db5e41d429285b5f451e928479f19d80818878527d36e95e" 225 | ], 226 | "version": "==4.10.1" 227 | }, 228 | "iniconfig": { 229 | "hashes": [ 230 | "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", 231 | "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" 232 | ], 233 | "version": "==1.1.1" 234 | }, 235 | "isort": { 236 | "hashes": [ 237 | "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7", 238 | "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951" 239 | ], 240 | "version": "==5.10.1" 241 | }, 242 | "jeepney": { 243 | "hashes": [ 244 | "sha256:1b5a0ea5c0e7b166b2f5895b91a08c14de8915afda4407fb5022a195224958ac", 245 | "sha256:fa9e232dfa0c498bd0b8a3a73b8d8a31978304dcef0515adc859d4e096f96f4f" 246 | ], 247 | "markers": "sys_platform == 'linux'", 248 | "version": "==0.7.1" 249 | }, 250 | "keyring": { 251 | "hashes": [ 252 | "sha256:9012508e141a80bd1c0b6778d5c610dd9f8c464d75ac6774248500503f972fb9", 253 | "sha256:b0d28928ac3ec8e42ef4cc227822647a19f1d544f21f96457965dc01cf555261" 254 | ], 255 | "version": "==23.5.0" 256 | }, 257 | "lazy-object-proxy": { 258 | "hashes": [ 259 | "sha256:043651b6cb706eee4f91854da4a089816a6606c1428fd391573ef8cb642ae4f7", 260 | "sha256:07fa44286cda977bd4803b656ffc1c9b7e3bc7dff7d34263446aec8f8c96f88a", 261 | "sha256:12f3bb77efe1367b2515f8cb4790a11cffae889148ad33adad07b9b55e0ab22c", 262 | "sha256:2052837718516a94940867e16b1bb10edb069ab475c3ad84fd1e1a6dd2c0fcfc", 263 | "sha256:2130db8ed69a48a3440103d4a520b89d8a9405f1b06e2cc81640509e8bf6548f", 264 | "sha256:39b0e26725c5023757fc1ab2a89ef9d7ab23b84f9251e28f9cc114d5b59c1b09", 265 | "sha256:46ff647e76f106bb444b4533bb4153c7370cdf52efc62ccfc1a28bdb3cc95442", 266 | "sha256:4dca6244e4121c74cc20542c2ca39e5c4a5027c81d112bfb893cf0790f96f57e", 267 | "sha256:553b0f0d8dbf21890dd66edd771f9b1b5f51bd912fa5f26de4449bfc5af5e029", 268 | "sha256:677ea950bef409b47e51e733283544ac3d660b709cfce7b187f5ace137960d61", 269 | "sha256:6a24357267aa976abab660b1d47a34aaf07259a0c3859a34e536f1ee6e76b5bb", 270 | "sha256:6a6e94c7b02641d1311228a102607ecd576f70734dc3d5e22610111aeacba8a0", 271 | "sha256:6aff3fe5de0831867092e017cf67e2750c6a1c7d88d84d2481bd84a2e019ec35", 272 | "sha256:6ecbb350991d6434e1388bee761ece3260e5228952b1f0c46ffc800eb313ff42", 273 | "sha256:7096a5e0c1115ec82641afbdd70451a144558ea5cf564a896294e346eb611be1", 274 | "sha256:70ed0c2b380eb6248abdef3cd425fc52f0abd92d2b07ce26359fcbc399f636ad", 275 | "sha256:8561da8b3dd22d696244d6d0d5330618c993a215070f473b699e00cf1f3f6443", 276 | "sha256:85b232e791f2229a4f55840ed54706110c80c0a210d076eee093f2b2e33e1bfd", 277 | "sha256:898322f8d078f2654d275124a8dd19b079080ae977033b713f677afcfc88e2b9", 278 | "sha256:8f3953eb575b45480db6568306893f0bd9d8dfeeebd46812aa09ca9579595148", 279 | "sha256:91ba172fc5b03978764d1df5144b4ba4ab13290d7bab7a50f12d8117f8630c38", 280 | "sha256:9d166602b525bf54ac994cf833c385bfcc341b364e3ee71e3bf5a1336e677b55", 281 | "sha256:a57d51ed2997e97f3b8e3500c984db50a554bb5db56c50b5dab1b41339b37e36", 282 | "sha256:b9e89b87c707dd769c4ea91f7a31538888aad05c116a59820f28d59b3ebfe25a", 283 | "sha256:bb8c5fd1684d60a9902c60ebe276da1f2281a318ca16c1d0a96db28f62e9166b", 284 | "sha256:c19814163728941bb871240d45c4c30d33b8a2e85972c44d4e63dd7107faba44", 285 | "sha256:c4ce15276a1a14549d7e81c243b887293904ad2d94ad767f42df91e75fd7b5b6", 286 | "sha256:c7a683c37a8a24f6428c28c561c80d5f4fd316ddcf0c7cab999b15ab3f5c5c69", 287 | "sha256:d609c75b986def706743cdebe5e47553f4a5a1da9c5ff66d76013ef396b5a8a4", 288 | "sha256:d66906d5785da8e0be7360912e99c9188b70f52c422f9fc18223347235691a84", 289 | "sha256:dd7ed7429dbb6c494aa9bc4e09d94b778a3579be699f9d67da7e6804c422d3de", 290 | "sha256:df2631f9d67259dc9620d831384ed7732a198eb434eadf69aea95ad18c587a28", 291 | "sha256:e368b7f7eac182a59ff1f81d5f3802161932a41dc1b1cc45c1f757dc876b5d2c", 292 | "sha256:e40f2013d96d30217a51eeb1db28c9ac41e9d0ee915ef9d00da639c5b63f01a1", 293 | "sha256:f769457a639403073968d118bc70110e7dce294688009f5c24ab78800ae56dc8", 294 | "sha256:fccdf7c2c5821a8cbd0a9440a456f5050492f2270bd54e94360cac663398739b", 295 | "sha256:fd45683c3caddf83abbb1249b653a266e7069a09f486daa8863fb0e7496a9fdb" 296 | ], 297 | "version": "==1.7.1" 298 | }, 299 | "mccabe": { 300 | "hashes": [ 301 | "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", 302 | "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" 303 | ], 304 | "version": "==0.6.1" 305 | }, 306 | "packaging": { 307 | "hashes": [ 308 | "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", 309 | "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" 310 | ], 311 | "version": "==21.3" 312 | }, 313 | "pkginfo": { 314 | "hashes": [ 315 | "sha256:542e0d0b6750e2e21c20179803e40ab50598d8066d51097a0e382cba9eb02bff", 316 | "sha256:c24c487c6a7f72c66e816ab1796b96ac6c3d14d49338293d2141664330b55ffc" 317 | ], 318 | "version": "==1.8.2" 319 | }, 320 | "platformdirs": { 321 | "hashes": [ 322 | "sha256:1d7385c7db91728b83efd0ca99a5afb296cab9d0ed8313a45ed8ba17967ecfca", 323 | "sha256:440633ddfebcc36264232365d7840a970e75e1018d15b4327d11f91909045fda" 324 | ], 325 | "version": "==2.4.1" 326 | }, 327 | "pluggy": { 328 | "hashes": [ 329 | "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", 330 | "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" 331 | ], 332 | "version": "==1.0.0" 333 | }, 334 | "py": { 335 | "hashes": [ 336 | "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719", 337 | "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378" 338 | ], 339 | "index": "pypi", 340 | "version": "==1.11.0" 341 | }, 342 | "pycodestyle": { 343 | "hashes": [ 344 | "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", 345 | "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" 346 | ], 347 | "version": "==2.8.0" 348 | }, 349 | "pycparser": { 350 | "hashes": [ 351 | "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", 352 | "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" 353 | ], 354 | "version": "==2.21" 355 | }, 356 | "pygments": { 357 | "hashes": [ 358 | "sha256:44238f1b60a76d78fc8ca0528ee429702aae011c265fe6a8dd8b63049ae41c65", 359 | "sha256:4e426f72023d88d03b2fa258de560726ce890ff3b630f88c21cbb8b2503b8c6a" 360 | ], 361 | "index": "pypi", 362 | "version": "==2.11.2" 363 | }, 364 | "pylint": { 365 | "hashes": [ 366 | "sha256:9d945a73640e1fec07ee34b42f5669b770c759acd536ec7b16d7e4b87a9c9ff9", 367 | "sha256:daabda3f7ed9d1c60f52d563b1b854632fd90035bcf01443e234d3dc794e3b74" 368 | ], 369 | "version": "==2.12.2" 370 | }, 371 | "pyparsing": { 372 | "hashes": [ 373 | "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea", 374 | "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484" 375 | ], 376 | "version": "==3.0.7" 377 | }, 378 | "pytest": { 379 | "hashes": [ 380 | "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89", 381 | "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134" 382 | ], 383 | "index": "pypi", 384 | "version": "==6.2.5" 385 | }, 386 | "pytest-cov": { 387 | "hashes": [ 388 | "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6", 389 | "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470" 390 | ], 391 | "index": "pypi", 392 | "version": "==3.0.0" 393 | }, 394 | "pytest-mock": { 395 | "hashes": [ 396 | "sha256:30c2f2cc9759e76eee674b81ea28c9f0b94f8f0445a1b87762cadf774f0df7e3", 397 | "sha256:40217a058c52a63f1042f0784f62009e976ba824c418cced42e88d5f40ab0e62" 398 | ], 399 | "index": "pypi", 400 | "version": "==3.6.1" 401 | }, 402 | "pytest-pylint": { 403 | "hashes": [ 404 | "sha256:790c7a8019fab08e59bd3812db1657a01995a975af8b1c6ce95b9aa39d61da27", 405 | "sha256:b63aaf8b80ff33c8ceaa7f68323ed04102c7790093ccf6bdb261a4c2dc6fd564" 406 | ], 407 | "index": "pypi", 408 | "version": "==0.18.0" 409 | }, 410 | "readme-renderer": { 411 | "hashes": [ 412 | "sha256:a50a0f2123a4c1145ac6f420e1a348aafefcc9211c846e3d51df05fe3d865b7d", 413 | "sha256:b512beafa6798260c7d5af3e1b1f097e58bfcd9a575da7c4ddd5e037490a5b85" 414 | ], 415 | "version": "==32.0" 416 | }, 417 | "requests": { 418 | "hashes": [ 419 | "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", 420 | "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d" 421 | ], 422 | "version": "==2.27.1" 423 | }, 424 | "requests-toolbelt": { 425 | "hashes": [ 426 | "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", 427 | "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" 428 | ], 429 | "version": "==0.9.1" 430 | }, 431 | "rfc3986": { 432 | "hashes": [ 433 | "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd", 434 | "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c" 435 | ], 436 | "version": "==2.0.0" 437 | }, 438 | "secretstorage": { 439 | "hashes": [ 440 | "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f", 441 | "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195" 442 | ], 443 | "markers": "sys_platform == 'linux'", 444 | "version": "==3.3.1" 445 | }, 446 | "six": { 447 | "hashes": [ 448 | "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", 449 | "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" 450 | ], 451 | "version": "==1.16.0" 452 | }, 453 | "toml": { 454 | "hashes": [ 455 | "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", 456 | "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" 457 | ], 458 | "version": "==0.10.2" 459 | }, 460 | "tomli": { 461 | "hashes": [ 462 | "sha256:b5bde28da1fed24b9bd1d4d2b8cba62300bfb4ec9a6187a957e8ddb9434c5224", 463 | "sha256:c292c34f58502a1eb2bbb9f5bbc9a5ebc37bee10ffb8c2d6bbdfa8eb13cc14e1" 464 | ], 465 | "version": "==2.0.0" 466 | }, 467 | "tqdm": { 468 | "hashes": [ 469 | "sha256:8dd278a422499cd6b727e6ae4061c40b48fce8b76d1ccbf5d34fca9b7f925b0c", 470 | "sha256:d359de7217506c9851b7869f3708d8ee53ed70a1b8edbba4dbcb47442592920d" 471 | ], 472 | "version": "==4.62.3" 473 | }, 474 | "twine": { 475 | "hashes": [ 476 | "sha256:28460a3db6b4532bde6a5db6755cf2dce6c5020bada8a641bb2c5c7a9b1f35b8", 477 | "sha256:8c120845fc05270f9ee3e9d7ebbed29ea840e41f48cd059e04733f7e1d401345" 478 | ], 479 | "index": "pypi", 480 | "version": "==3.7.1" 481 | }, 482 | "typing-extensions": { 483 | "hashes": [ 484 | "sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e", 485 | "sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b" 486 | ], 487 | "markers": "python_version < '3.10'", 488 | "version": "==4.0.1" 489 | }, 490 | "urllib3": { 491 | "hashes": [ 492 | "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed", 493 | "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c" 494 | ], 495 | "index": "pypi", 496 | "version": "==1.26.8" 497 | }, 498 | "webencodings": { 499 | "hashes": [ 500 | "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", 501 | "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" 502 | ], 503 | "version": "==0.5.1" 504 | }, 505 | "wheel": { 506 | "hashes": [ 507 | "sha256:4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a", 508 | "sha256:e9a504e793efbca1b8e0e9cb979a249cf4a0a7b5b8c9e8b65a5e39d49529c1c4" 509 | ], 510 | "index": "pypi", 511 | "version": "==0.37.1" 512 | }, 513 | "wrapt": { 514 | "hashes": [ 515 | "sha256:086218a72ec7d986a3eddb7707c8c4526d677c7b35e355875a0fe2918b059179", 516 | "sha256:0877fe981fd76b183711d767500e6b3111378ed2043c145e21816ee589d91096", 517 | "sha256:0a017a667d1f7411816e4bf214646d0ad5b1da2c1ea13dec6c162736ff25a374", 518 | "sha256:0cb23d36ed03bf46b894cfec777eec754146d68429c30431c99ef28482b5c1df", 519 | "sha256:1fea9cd438686e6682271d36f3481a9f3636195578bab9ca3382e2f5f01fc185", 520 | "sha256:220a869982ea9023e163ba915077816ca439489de6d2c09089b219f4e11b6785", 521 | "sha256:25b1b1d5df495d82be1c9d2fad408f7ce5ca8a38085e2da41bb63c914baadff7", 522 | "sha256:2dded5496e8f1592ec27079b28b6ad2a1ef0b9296d270f77b8e4a3a796cf6909", 523 | "sha256:2ebdde19cd3c8cdf8df3fc165bc7827334bc4e353465048b36f7deeae8ee0918", 524 | "sha256:43e69ffe47e3609a6aec0fe723001c60c65305784d964f5007d5b4fb1bc6bf33", 525 | "sha256:46f7f3af321a573fc0c3586612db4decb7eb37172af1bc6173d81f5b66c2e068", 526 | "sha256:47f0a183743e7f71f29e4e21574ad3fa95676136f45b91afcf83f6a050914829", 527 | "sha256:498e6217523111d07cd67e87a791f5e9ee769f9241fcf8a379696e25806965af", 528 | "sha256:4b9c458732450ec42578b5642ac53e312092acf8c0bfce140ada5ca1ac556f79", 529 | "sha256:51799ca950cfee9396a87f4a1240622ac38973b6df5ef7a41e7f0b98797099ce", 530 | "sha256:5601f44a0f38fed36cc07db004f0eedeaadbdcec90e4e90509480e7e6060a5bc", 531 | "sha256:5f223101f21cfd41deec8ce3889dc59f88a59b409db028c469c9b20cfeefbe36", 532 | "sha256:610f5f83dd1e0ad40254c306f4764fcdc846641f120c3cf424ff57a19d5f7ade", 533 | "sha256:6a03d9917aee887690aa3f1747ce634e610f6db6f6b332b35c2dd89412912bca", 534 | "sha256:705e2af1f7be4707e49ced9153f8d72131090e52be9278b5dbb1498c749a1e32", 535 | "sha256:766b32c762e07e26f50d8a3468e3b4228b3736c805018e4b0ec8cc01ecd88125", 536 | "sha256:77416e6b17926d953b5c666a3cb718d5945df63ecf922af0ee576206d7033b5e", 537 | "sha256:778fd096ee96890c10ce96187c76b3e99b2da44e08c9e24d5652f356873f6709", 538 | "sha256:78dea98c81915bbf510eb6a3c9c24915e4660302937b9ae05a0947164248020f", 539 | "sha256:7dd215e4e8514004c8d810a73e342c536547038fb130205ec4bba9f5de35d45b", 540 | "sha256:7dde79d007cd6dfa65afe404766057c2409316135cb892be4b1c768e3f3a11cb", 541 | "sha256:81bd7c90d28a4b2e1df135bfbd7c23aee3050078ca6441bead44c42483f9ebfb", 542 | "sha256:85148f4225287b6a0665eef08a178c15097366d46b210574a658c1ff5b377489", 543 | "sha256:865c0b50003616f05858b22174c40ffc27a38e67359fa1495605f96125f76640", 544 | "sha256:87883690cae293541e08ba2da22cacaae0a092e0ed56bbba8d018cc486fbafbb", 545 | "sha256:8aab36778fa9bba1a8f06a4919556f9f8c7b33102bd71b3ab307bb3fecb21851", 546 | "sha256:8c73c1a2ec7c98d7eaded149f6d225a692caa1bd7b2401a14125446e9e90410d", 547 | "sha256:936503cb0a6ed28dbfa87e8fcd0a56458822144e9d11a49ccee6d9a8adb2ac44", 548 | "sha256:944b180f61f5e36c0634d3202ba8509b986b5fbaf57db3e94df11abee244ba13", 549 | "sha256:96b81ae75591a795d8c90edc0bfaab44d3d41ffc1aae4d994c5aa21d9b8e19a2", 550 | "sha256:981da26722bebb9247a0601e2922cedf8bb7a600e89c852d063313102de6f2cb", 551 | "sha256:ae9de71eb60940e58207f8e71fe113c639da42adb02fb2bcbcaccc1ccecd092b", 552 | "sha256:b73d4b78807bd299b38e4598b8e7bd34ed55d480160d2e7fdaabd9931afa65f9", 553 | "sha256:d4a5f6146cfa5c7ba0134249665acd322a70d1ea61732723c7d3e8cc0fa80755", 554 | "sha256:dd91006848eb55af2159375134d724032a2d1d13bcc6f81cd8d3ed9f2b8e846c", 555 | "sha256:e05e60ff3b2b0342153be4d1b597bbcfd8330890056b9619f4ad6b8d5c96a81a", 556 | "sha256:e6906d6f48437dfd80464f7d7af1740eadc572b9f7a4301e7dd3d65db285cacf", 557 | "sha256:e92d0d4fa68ea0c02d39f1e2f9cb5bc4b4a71e8c442207433d8db47ee79d7aa3", 558 | "sha256:e94b7d9deaa4cc7bac9198a58a7240aaf87fe56c6277ee25fa5b3aa1edebd229", 559 | "sha256:ea3e746e29d4000cd98d572f3ee2a6050a4f784bb536f4ac1f035987fc1ed83e", 560 | "sha256:ec7e20258ecc5174029a0f391e1b948bf2906cd64c198a9b8b281b811cbc04de", 561 | "sha256:ec9465dd69d5657b5d2fa6133b3e1e989ae27d29471a672416fd729b429eb554", 562 | "sha256:f122ccd12fdc69628786d0c947bdd9cb2733be8f800d88b5a37c57f1f1d73c10", 563 | "sha256:f99c0489258086308aad4ae57da9e8ecf9e1f3f30fa35d5e170b4d4896554d80", 564 | "sha256:f9c51d9af9abb899bd34ace878fbec8bf357b3194a10c4e8e0a25512826ef056", 565 | "sha256:fd76c47f20984b43d93de9a82011bb6e5f8325df6c9ed4d8310029a55fa361ea" 566 | ], 567 | "version": "==1.13.3" 568 | }, 569 | "zipp": { 570 | "hashes": [ 571 | "sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d", 572 | "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375" 573 | ], 574 | "version": "==3.7.0" 575 | } 576 | } 577 | } 578 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/pcstout/csv-schema-py.svg?branch=master)](https://travis-ci.org/pcstout/csv-schema-py) 2 | [![Build status](https://ci.appveyor.com/api/projects/status/0s8kitxrkb5neae8/branch/master?svg=true)](https://ci.appveyor.com/project/pcstout/csv-schema-py/branch/master) 3 | 4 | # CSV Schema Definition and Validation 5 | 6 | ## Install 7 | 8 | ```bash 9 | pip install csv-schema 10 | ``` 11 | 12 | ## Usage 13 | ```text 14 | usage: csv-schema [-h] [--version] {validate-config,validate-csv,generate-config} ... 15 | 16 | CSV Schema 17 | 18 | optional arguments: 19 | -h, --help show this help message and exit 20 | --version show program's version number and exit 21 | 22 | Commands: 23 | {validate-config,validate-csv,generate-config} 24 | validate-config Validates the CSV schema JSON configuration file. 25 | validate-csv Validates a CSV file against a schema. 26 | generate-config Generate a CSV schema JSON configuration file. 27 | 28 | ``` 29 | 30 | # Definition 31 | 32 | The CSV file schema definition is defined in a JSON file in the following format. 33 | 34 | ## File Definition: 35 | 36 | ```json 37 | { 38 | "name": null, 39 | "description": null, 40 | "filename": { 41 | "regex": null 42 | }, 43 | "columns": [] 44 | } 45 | ``` 46 | | Property | Description | 47 | | -------- | ----------- | 48 | | name | The name of the schema. | 49 | | description | The description of the schema. | 50 | | filename | Properties for the name of the CSV filename to validate. | 51 | | columns | List of column definitions. | 52 | 53 | 54 | ### filename 55 | ```json 56 | { 57 | "regex": null 58 | } 59 | ``` 60 | | Property | Description | 61 | | -------- | ----------- | 62 | | regex | Regular expression to validate the name of the CSV file being validated. | 63 | 64 | ## Column Definitions: 65 | 66 | ### string 67 | ```json 68 | { 69 | "type": "string", 70 | "name": null, 71 | "required": true, 72 | "null_or_empty": false, 73 | "regex": null, 74 | "min": null, 75 | "max": null 76 | } 77 | ``` 78 | | Property | Description | 79 | | -------- | ----------- | 80 | | type | The column type. | 81 | | name | The name of the column. | 82 | | required | Whether or not the column is required in the file. | 83 | | null_or_empty | Whether or not the value can be null (missing) or an empty string. | 84 | | regex | Regular expression to validate the column value. | 85 | | min | The minimum length of the string. null for no limit. | 86 | | max | The maximum length of the string. null for no limit. | 87 | 88 | ### integer 89 | ```json 90 | { 91 | "type": "integer", 92 | "name": null, 93 | "required": true, 94 | "null_or_empty": false, 95 | "regex": null, 96 | "min": null, 97 | "max": null 98 | } 99 | ``` 100 | | Property | Description | 101 | | -------- | ----------- | 102 | | type | The column type. | 103 | | name | The name of the column. | 104 | | required | Whether or not the column is required in the file. | 105 | | null_or_empty | Whether or not the value can be null (missing) or an empty string. | 106 | | regex | Regular expression to validate the column value. | 107 | | min | The minimum value. null for no limit. | 108 | | max | The maximum value. null for no limit. | 109 | 110 | ### decimal 111 | ```json 112 | { 113 | "type": "decimal", 114 | "name": null, 115 | "required": true, 116 | "null_or_empty": false, 117 | "regex": null, 118 | "min": null, 119 | "max": null, 120 | "precision": 2 121 | } 122 | ``` 123 | | Property | Description | 124 | | -------- | ----------- | 125 | | type | The column type. | 126 | | name | The name of the column. | 127 | | required | Whether or not the column is required in the file. | 128 | | null_or_empty | Whether or not the value can be null (missing) or an empty string. | 129 | | regex | Regular expression to validate the column value. | 130 | | min | The minimum value. null for no limit. | 131 | | max | The maximum value. null for no limit. | 132 | | precision | The decimal point precision. | 133 | 134 | ### enum 135 | ```json 136 | { 137 | "type": "enum", 138 | "name": null, 139 | "required": true, 140 | "null_or_empty": false, 141 | "values": [] 142 | } 143 | ``` 144 | | Property | Description | 145 | | -------- | ----------- | 146 | | type | The column type. | 147 | | name | The name of the column. | 148 | | required | Whether or not the column is required in the file. | 149 | | null_or_empty | Whether or not the value can be null (missing) or an empty string. | 150 | | values | Fixed set of constants. | 151 | 152 | ## Development 153 | 154 | ### Setup 155 | 156 | ```bash 157 | pipenv --three 158 | pipenv shell 159 | make pip_install 160 | make build 161 | make install_local 162 | ``` 163 | See [Makefile](Makefile) for all commands. 164 | 165 | ### Update README Definitions 166 | The schema definition is auto generated by running `scripts/generate_md_help.py`. 167 | Copy/paste the output into the README.md file. 168 | 169 | ### Testing 170 | 171 | - Create and activate a virtual environment: 172 | - Run the tests: `make test` 173 | -------------------------------------------------------------------------------- /scripts/generate_md_help.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # This script will generate markdown help for the README.md. 3 | import os 4 | import sys 5 | 6 | script_dir = os.path.dirname(__file__) 7 | sys.path.append(os.path.join(script_dir, '..')) 8 | import src.csv_schema.core.models as models 9 | 10 | config = models.SchemaConfig('.') 11 | 12 | markdown = '## File Definition:' + os.linesep 13 | markdown = markdown + os.linesep + config.to_md_help() + os.linesep 14 | 15 | markdown = markdown + os.linesep + '## Column Definitions:' + os.linesep 16 | 17 | for column in [models.StringColumn(), models.IntegerColumn(), models.DecimalColumn(), models.EnumColumn()]: 18 | markdown = markdown + os.linesep + '### {0}'.format(column.type.value) 19 | markdown = markdown + os.linesep + column.to_md_help() + os.linesep 20 | 21 | print(markdown) 22 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal=1 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | from src.csv_schema._version import __version__ 3 | 4 | with open("README.md", "r") as fh: 5 | long_description = fh.read() 6 | 7 | setuptools.setup( 8 | name="csv-schema", 9 | version=__version__, 10 | author="Patrick Stout", 11 | author_email="pstout@prevagroup.com", 12 | license="Unknown", 13 | description="CSV file validation.", 14 | long_description=long_description, 15 | long_description_content_type="text/markdown", 16 | url="https://github.com/pcstout/csv-schema-py", 17 | package_dir={"": "src"}, 18 | packages=setuptools.find_packages(where="src"), 19 | classifiers=( 20 | "Programming Language :: Python", 21 | "Programming Language :: Python :: 3.7", 22 | "License :: OSI Approved :: Apache Software License", 23 | "Operating System :: OS Independent", 24 | ), 25 | entry_points={ 26 | 'console_scripts': [ 27 | "csv-schema = csv_schema.cli:main" 28 | ] 29 | }, 30 | install_requires=[ 31 | 32 | ] 33 | ) 34 | -------------------------------------------------------------------------------- /src/csv_schema/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcstout/csv-schema-py/3e77dfb1a5695ef6e4547549d56f8cf023f08f1c/src/csv_schema/__init__.py -------------------------------------------------------------------------------- /src/csv_schema/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = '1.1' 2 | -------------------------------------------------------------------------------- /src/csv_schema/cli.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import argparse 3 | from .core import ExitCodes 4 | from .commands.validate_config import cli as validate_config_cli 5 | from .commands.validate_csv import cli as validate_csv_cli 6 | from .commands.generate_config import cli as generate_config_cli 7 | from ._version import __version__ 8 | 9 | ALL_ACTIONS = [validate_config_cli, validate_csv_cli, generate_config_cli] 10 | 11 | 12 | def main(args=None): 13 | shared_parser = argparse.ArgumentParser(add_help=False) 14 | 15 | main_parser = argparse.ArgumentParser(description='CSV Schema') 16 | main_parser.add_argument('--version', action='version',version='%(prog)s {0}'.format(__version__)) 17 | 18 | subparsers = main_parser.add_subparsers(title='Commands', dest='command') 19 | for action in ALL_ACTIONS: 20 | action.create(subparsers, [shared_parser]) 21 | 22 | cmd_args = main_parser.parse_args(args) 23 | 24 | if '_execute' in cmd_args: 25 | cmd_args._execute(cmd_args) 26 | else: 27 | main_parser.print_help() 28 | sys.exit(ExitCodes.FAIL) 29 | -------------------------------------------------------------------------------- /src/csv_schema/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcstout/csv-schema-py/3e77dfb1a5695ef6e4547549d56f8cf023f08f1c/src/csv_schema/commands/__init__.py -------------------------------------------------------------------------------- /src/csv_schema/commands/generate_config/__init__.py: -------------------------------------------------------------------------------- 1 | from .cli import create, execute 2 | from .generate_config import GenerateConfig 3 | -------------------------------------------------------------------------------- /src/csv_schema/commands/generate_config/cli.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from .generate_config import GenerateConfig 3 | from ...core import ExitCodes 4 | 5 | 6 | def create(subparsers, parents): 7 | parser = subparsers.add_parser('generate-config', 8 | parents=parents, 9 | help='Generate a CSV schema JSON configuration file.') 10 | parser.add_argument('path', help='Where to generate the file.') 11 | parser.add_argument('--with-csv', 12 | default=False, 13 | action='store_true', 14 | help='Generate a test CSV file to go along with the schema.') 15 | parser.add_argument('--with-csv-rows', 16 | default=100, 17 | type=int, 18 | help='How many rows to generate in the CSV file.') 19 | parser.set_defaults(_execute=execute) 20 | 21 | 22 | def execute(args): 23 | vc = GenerateConfig(args.path, with_csv=args.with_csv, with_csv_rows=args.with_csv_rows) 24 | vc.execute() 25 | if len(vc.errors) > 0: 26 | print('Errors found in: {0}'.format(vc.path)) 27 | for error in vc.errors: 28 | print(error) 29 | sys.exit(ExitCodes.FAIL) 30 | else: 31 | print('Configuration file written to: {0}'.format(vc.path)) 32 | if args.with_csv: 33 | print('CSV file written to: {0}'.format(vc.csv_path)) 34 | sys.exit(ExitCodes.SUCCESS) 35 | -------------------------------------------------------------------------------- /src/csv_schema/commands/generate_config/generate_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import random 4 | from ...core import Utils 5 | from ...core.models import SchemaConfig, ColumnTypes 6 | 7 | 8 | class GenerateConfig: 9 | def __init__(self, path, with_csv=False, with_csv_rows=100): 10 | self.path = Utils.expand_path(path) 11 | self.with_csv = with_csv 12 | self.with_csv_rows = with_csv_rows 13 | self.csv_path = None 14 | self.config = None 15 | self.errors = [] 16 | 17 | def execute(self): 18 | """Execute the generation. 19 | 20 | Returns: 21 | List of errors or empty list. 22 | """ 23 | if os.path.isdir(self.path) or not self.path.lower().endswith('.json'): 24 | self.path = os.path.join(self.path, 'csv-schema.json') 25 | 26 | Utils.ensure_dirs(os.path.dirname(self.path)) 27 | 28 | self.config = SchemaConfig(self.path) 29 | self.config.name.value = 'CSV Schema' 30 | self.config.description.value = 'Generated CSV Configuration.' 31 | 32 | count = 1 33 | for column_type in ColumnTypes.ALL: 34 | column = ColumnTypes.get_instance(column_type) 35 | column.name.value = 'col_{0}'.format(count) 36 | 37 | if column_type == ColumnTypes.ENUM: 38 | column.values.value = ['a', 'b', 'c'] 39 | 40 | self.config.columns.value.append(column) 41 | count += 1 42 | 43 | self.errors = self.config.validate() 44 | if not self.errors: 45 | self.config.save() 46 | 47 | if self.with_csv: 48 | self._generate_test_csv() 49 | 50 | return self.errors 51 | 52 | def _generate_test_csv(self): 53 | self.csv_path = self.config.path + '.csv' 54 | with open(self.csv_path, mode='w', newline='') as f: 55 | csv_writer = csv.DictWriter(f, fieldnames=[c.name.value for c in self.config.columns.value]) 56 | csv_writer.writeheader() 57 | row_count = 1 58 | while row_count <= self.with_csv_rows: 59 | row = {} 60 | for col in self.config.columns.value: 61 | if col.type.value == ColumnTypes.STRING: 62 | row[col.name.value] = 'string column value row {0}'.format(row_count) 63 | elif col.type.value == ColumnTypes.INTEGER: 64 | row[col.name.value] = row_count 65 | elif col.type.value == ColumnTypes.DECIMAL: 66 | row[col.name.value] = '{0}.{1}'.format(row_count, random.randint(10, 99)) 67 | elif col.type.value == ColumnTypes.ENUM: 68 | row[col.name.value] = col.values.value[random.randint(0, 2)] 69 | csv_writer.writerow(row) 70 | row_count += 1 71 | -------------------------------------------------------------------------------- /src/csv_schema/commands/validate_config/__init__.py: -------------------------------------------------------------------------------- 1 | from .cli import create, execute 2 | from .validate_config import ValidateConfig 3 | -------------------------------------------------------------------------------- /src/csv_schema/commands/validate_config/cli.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from .validate_config import ValidateConfig 3 | from ...core import ExitCodes 4 | 5 | 6 | def create(subparsers, parents): 7 | parser = subparsers.add_parser('validate-config', 8 | parents=parents, 9 | help='Validates the CSV schema JSON configuration file.') 10 | parser.add_argument('schema', help='The JSON file to validate.') 11 | parser.set_defaults(_execute=execute) 12 | 13 | 14 | def execute(args): 15 | vc = ValidateConfig(args.schema) 16 | vc.execute() 17 | if len(vc.errors) > 0: 18 | print('Errors found in: {0}'.format(vc.filename)) 19 | for error in vc.errors: 20 | print(error) 21 | sys.exit(ExitCodes.FAIL) 22 | else: 23 | print('No errors found in: {0}'.format(vc.filename)) 24 | sys.exit(ExitCodes.SUCCESS) 25 | -------------------------------------------------------------------------------- /src/csv_schema/commands/validate_config/validate_config.py: -------------------------------------------------------------------------------- 1 | from ...core import Utils 2 | from ...core.models import SchemaConfig 3 | 4 | 5 | class ValidateConfig: 6 | def __init__(self, schema_filename): 7 | self.filename = Utils.expand_path(schema_filename) 8 | self.config = None 9 | self.errors = [] 10 | 11 | def execute(self): 12 | """Execute the validation. 13 | 14 | Returns: 15 | List of errors or empty list. 16 | """ 17 | self.config = SchemaConfig(self.filename).load() 18 | self.errors = self.config.validate() 19 | return self.errors 20 | -------------------------------------------------------------------------------- /src/csv_schema/commands/validate_csv/__init__.py: -------------------------------------------------------------------------------- 1 | from .cli import create, execute 2 | from .validate_csv import ValidateCsv 3 | -------------------------------------------------------------------------------- /src/csv_schema/commands/validate_csv/cli.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from .validate_csv import ValidateCsv 3 | from ...core import ExitCodes 4 | 5 | 6 | def create(subparsers, parents): 7 | parser = subparsers.add_parser('validate-csv', 8 | parents=parents, 9 | help='Validates a CSV file against a schema.') 10 | parser.add_argument('csv', help='The CSV file to validate.') 11 | parser.add_argument('schema', help='The schema JSON file.') 12 | parser.set_defaults(_execute=execute) 13 | 14 | 15 | def execute(args): 16 | vc = ValidateCsv(args.csv, args.schema) 17 | vc.execute() 18 | if len(vc.errors) > 0: 19 | print('Errors found in: {0}'.format(vc.csv_path)) 20 | for error in vc.errors: 21 | print(error) 22 | sys.exit(ExitCodes.FAIL) 23 | else: 24 | print('No errors found in: {0}'.format(vc.csv_path)) 25 | sys.exit(ExitCodes.SUCCESS) 26 | -------------------------------------------------------------------------------- /src/csv_schema/commands/validate_csv/validate_csv.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import csv 4 | from ...core import Utils 5 | from ..validate_config import ValidateConfig 6 | 7 | 8 | class ValidateCsv: 9 | 10 | def __init__(self, csv_path, schema_path): 11 | self.csv_path = Utils.expand_path(csv_path) 12 | self.schema_path = Utils.expand_path(schema_path) 13 | self.config = None 14 | self.errors = [] 15 | 16 | def execute(self): 17 | validate_config = ValidateConfig(self.schema_path) 18 | self.errors = validate_config.execute() 19 | self.config = validate_config.config 20 | if not self.errors: 21 | self._validate_csv() 22 | return self.errors 23 | 24 | def _validate_csv(self): 25 | if not self._validate_filename(): 26 | return 27 | if not self._validate_columns(): 28 | return 29 | self._validate_data() 30 | 31 | def _validate_filename(self): 32 | filename = self.config.filename.value 33 | if filename is None: 34 | return True 35 | 36 | if filename.regex.value: 37 | regex = filename.regex.value 38 | csv_filename = os.path.basename(self.csv_path) 39 | is_valid = re.search(regex, csv_filename) is not None 40 | if not is_valid: 41 | self.errors.append('CSV file name: "{0}" does not match regex: "{1}"'.format(csv_filename, regex)) 42 | return is_valid 43 | else: 44 | return True 45 | 46 | def _validate_columns(self): 47 | with open(self.csv_path, 'r') as csv_file: 48 | reader = csv.DictReader(csv_file) 49 | headers = reader.fieldnames 50 | for column in self.config.columns.value: 51 | if column.required.value is True and column.name.value not in headers: 52 | self.errors.append('Required column: "{0}" not found.'.format(column.name.value)) 53 | return len(self.errors) == 0 54 | 55 | def _validate_data(self): 56 | with open(self.csv_path, 'r') as csv_file: 57 | reader = csv.DictReader(csv_file) 58 | headers = reader.fieldnames 59 | row_number = 1 60 | for row in reader: 61 | for column in self.config.columns.value: 62 | if column.name.value in headers: 63 | col_value = row[column.name.value] 64 | errors = column.validate_value(row_number, col_value) 65 | if errors: 66 | self.errors += errors 67 | row_number += 1 68 | return len(self.errors) == 0 69 | -------------------------------------------------------------------------------- /src/csv_schema/core/__init__.py: -------------------------------------------------------------------------------- 1 | from .utils import Utils 2 | from .exit_codes import ExitCodes 3 | -------------------------------------------------------------------------------- /src/csv_schema/core/exit_codes.py: -------------------------------------------------------------------------------- 1 | class ExitCodes: 2 | SUCCESS = 0 3 | FAIL = 1 4 | -------------------------------------------------------------------------------- /src/csv_schema/core/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .base_config_object import BaseConfigObject 2 | from .base_column import BaseColumn 3 | from .schema_config import SchemaConfig 4 | from .schema_config_filename import SchemaConfigFilename 5 | from .column_types import ColumnTypes 6 | from .config_property import ConfigProperty 7 | from .string_column import StringColumn 8 | from .integer_column import IntegerColumn 9 | from .decimal_column import DecimalColumn 10 | from .enum_column import EnumColumn 11 | -------------------------------------------------------------------------------- /src/csv_schema/core/models/base_column.py: -------------------------------------------------------------------------------- 1 | from .base_config_object import BaseConfigObject 2 | from .column_types import ColumnTypes 3 | from .config_property import ConfigProperty 4 | 5 | 6 | class BaseColumn(BaseConfigObject): 7 | def __init__(self, type=None, name=None, required=None, null_or_empty=None): 8 | super(BaseColumn, self).__init__() 9 | 10 | self.type = self.register_property( 11 | ConfigProperty('type', type, 'The column type.') 12 | ) 13 | self.name = self.register_property( 14 | ConfigProperty('name', name, 'The name of the column.') 15 | ) 16 | self.required = self.register_property( 17 | ConfigProperty('required', required, 'Whether or not the column is required in the file.') 18 | ) 19 | self.null_or_empty = self.register_property( 20 | ConfigProperty('null_or_empty', null_or_empty, 21 | 'Whether or not the value can be null (missing) or an empty string.') 22 | ) 23 | 24 | def on_validate(self): 25 | """Validates that each property has the correct value/type. 26 | 27 | Returns: 28 | List of error messages or an empty list. 29 | """ 30 | errors = super(BaseColumn, self).on_validate() 31 | 32 | if self.type.value not in ColumnTypes.ALL: 33 | errors.append('"type" must be one of: {0}'.format(','.join(ColumnTypes.ALL))) 34 | 35 | if self.name.value is None or len(self.name.value.strip()) == 0: 36 | errors.append('"name" must be specified.') 37 | 38 | if self.required.value not in [True, False]: 39 | errors.append('"required" must be True or False.') 40 | 41 | if self.null_or_empty.value not in [True, False]: 42 | errors.append('"null_or_empty" must be True or False.') 43 | 44 | return errors 45 | 46 | def validate_value(self, row_number, value): 47 | """Validates the value for a column from a CSV file. 48 | 49 | Args: 50 | row_number: The row number the value belongs to in the CSV file. 51 | value: The column value from the CSV file. This should always be a string. 52 | 53 | Returns: 54 | List of error messages or an empty list. 55 | """ 56 | errors = [] 57 | 58 | if not isinstance(value, str): 59 | raise ValueError('value must be a string.') 60 | 61 | if self.null_or_empty.value is False: 62 | if value is None or len(str(value).strip()) == 0: 63 | errors.append('Row number: {0}, column: "{1}", value: "{2}" cannot be null or empty.'.format( 64 | row_number, 65 | self.name.value, 66 | value) 67 | ) 68 | 69 | sub_errors = self.on_validate_value(row_number, value) 70 | if sub_errors: 71 | errors += sub_errors 72 | 73 | return errors 74 | 75 | def on_validate_value(self, row_number, value): 76 | """Override to implement value validation in sub-classes. 77 | 78 | Returns: 79 | List of error messages or an empty list. 80 | """ 81 | return [] 82 | 83 | def add_value_error(self, errors, row_number, value, error): 84 | errors.append( 85 | 'Row number: {0}, column: "{1}", value: "{2}" {3}.'.format( 86 | row_number, 87 | self.name.value, 88 | value, 89 | error) 90 | ) 91 | -------------------------------------------------------------------------------- /src/csv_schema/core/models/base_config_object.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from .column_types import ColumnTypes 4 | 5 | 6 | class BaseConfigObject(object): 7 | def __init__(self): 8 | self.properties = [] 9 | 10 | def register_property(self, prop): 11 | """Registers a configuration property. 12 | 13 | Args: 14 | prop: The ConfigProperty to register. 15 | 16 | Returns: 17 | The registered ConfigProperty. 18 | """ 19 | if prop not in self.properties: 20 | self.properties.append(prop) 21 | return prop 22 | 23 | def is_valid(self): 24 | """Get if the column data passes validation. 25 | 26 | Returns: 27 | True or False 28 | """ 29 | return len(self.validate()) == 0 30 | 31 | def validate(self): 32 | """Validates that each property has the correct value/type. 33 | 34 | Returns: 35 | List of error messages or an empty list. 36 | """ 37 | errors = self.on_validate() 38 | for prop in self.properties: 39 | if isinstance(prop.value, BaseConfigObject): 40 | child_errors = prop.value.validate() 41 | for child_error in child_errors: 42 | errors.append('"{0}" -> {1}'.format(prop.name, child_error)) 43 | elif isinstance(prop.value, list): 44 | for item in prop.value: 45 | if isinstance(item, BaseConfigObject): 46 | item_errors = item.validate() 47 | for item_error in item_errors: 48 | errors.append('"{0}" -> {1}'.format(prop.name, item_error)) 49 | 50 | return errors 51 | 52 | def on_validate(self): 53 | """Override to implement validation in sub-classes. 54 | 55 | Returns: 56 | List of error messages or an empty list. 57 | """ 58 | return [] 59 | 60 | def clear(self): 61 | """Clear all the config property values and reset to their defaults. 62 | 63 | Returns: 64 | None 65 | """ 66 | for prop in self.properties: 67 | prop.clear() 68 | 69 | def to_dict(self): 70 | """Gets the config properties as a dict. 71 | 72 | Returns: 73 | Config properties as a dict. 74 | """ 75 | result = {} 76 | for prop in self.properties: 77 | if isinstance(prop.value, BaseConfigObject): 78 | result[prop.name] = prop.value.to_dict() 79 | elif isinstance(prop.value, list): 80 | result[prop.name] = [] 81 | for item in prop.value: 82 | if isinstance(item, BaseConfigObject): 83 | result[prop.name].append(item.to_dict()) 84 | else: 85 | result[prop.name].append(item) 86 | else: 87 | result[prop.name] = prop.value 88 | 89 | return result 90 | 91 | def to_json(self): 92 | """Gets the config properties as JSON. 93 | 94 | Returns: 95 | Config properties as JSON 96 | """ 97 | return json.dumps(self.to_dict(), indent=2) 98 | 99 | def from_json(self, json_data): 100 | if isinstance(json_data, str): 101 | json_data = json.loads(json_data) 102 | 103 | for prop in self.properties: 104 | if prop.name not in json_data: 105 | raise KeyError(prop.name) 106 | 107 | json_value = json_data.get(prop.name) 108 | 109 | if isinstance(prop.value, BaseConfigObject): 110 | prop.value.from_json(json_value) 111 | elif isinstance(prop.value, list) and isinstance(json_value, list): 112 | prop.clear() 113 | for item in json_value: 114 | # List items can only be of a certain type, figure out which one it is. 115 | # Only Columns are supported right now. 116 | if 'type' in item and 'name' in item and 'required' in item and 'null_or_empty' in item: 117 | list_item = ColumnTypes.get_instance(item['type']) 118 | list_item.from_json(item) 119 | prop.value.append(list_item) 120 | else: 121 | prop.value.append(item) 122 | else: 123 | setattr(prop, 'value', json_value) 124 | return self 125 | 126 | def to_md_help(self): 127 | """Gets the help information for the config properties. 128 | 129 | Returns: 130 | String as markdown. 131 | """ 132 | lines = [] 133 | lines.append('```json') 134 | lines.append(self.to_json()) 135 | lines.append('```') 136 | 137 | lines.append('| Property | Description |') 138 | lines.append('| -------- | ----------- |') 139 | 140 | child_configs = {} 141 | 142 | for prop in self.properties: 143 | lines.append('| {0} | {1} |'.format(prop.name, prop.description)) 144 | if isinstance(prop.value, BaseConfigObject): 145 | child_configs[prop.name] = prop.value 146 | 147 | for child_name, child_value in child_configs.items(): 148 | lines.append(os.linesep) 149 | lines.append('### {0}'.format(child_name)) 150 | lines.append(child_value.to_md_help()) 151 | 152 | return os.linesep.join(lines) 153 | -------------------------------------------------------------------------------- /src/csv_schema/core/models/column_types.py: -------------------------------------------------------------------------------- 1 | class ColumnTypes: 2 | """Defines constants for all the column types.""" 3 | 4 | STRING = 'string' 5 | INTEGER = 'integer' 6 | DECIMAL = 'decimal' 7 | ENUM = 'enum' 8 | ALL = [STRING, INTEGER, DECIMAL, ENUM] 9 | 10 | @classmethod 11 | def get_instance(cls, type_str): 12 | # Load column class on-demand to avoid circular dependency errors. 13 | # TODO: Refactor this? 14 | from .string_column import StringColumn 15 | from .integer_column import IntegerColumn 16 | from .decimal_column import DecimalColumn 17 | from .enum_column import EnumColumn 18 | if type_str == cls.STRING: 19 | return StringColumn() 20 | elif type_str == cls.INTEGER: 21 | return IntegerColumn() 22 | elif type_str == cls.DECIMAL: 23 | return DecimalColumn() 24 | elif type_str == cls.ENUM: 25 | return EnumColumn() 26 | else: 27 | raise ValueError('Invalid type: {0}'.format(type_str)) 28 | -------------------------------------------------------------------------------- /src/csv_schema/core/models/config_property.py: -------------------------------------------------------------------------------- 1 | class ConfigProperty: 2 | class NotSpecified: 3 | pass 4 | 5 | def __init__(self, name, value=NotSpecified(), description=None, default=None): 6 | self.name = name 7 | self.value = value 8 | self.description = description 9 | self.default = default 10 | if isinstance(value, ConfigProperty.NotSpecified): 11 | self.clear() 12 | 13 | def clear(self): 14 | if isinstance(self.default, type): 15 | self.value = self.default() 16 | else: 17 | self.value = self.default 18 | -------------------------------------------------------------------------------- /src/csv_schema/core/models/decimal_column.py: -------------------------------------------------------------------------------- 1 | import decimal as dec 2 | import re 3 | from locale import localeconv 4 | from .base_column import BaseColumn 5 | from .column_types import ColumnTypes 6 | from .config_property import ConfigProperty 7 | 8 | 9 | class DecimalColumn(BaseColumn): 10 | COLUMN_TYPE = ColumnTypes.DECIMAL 11 | 12 | def __init__(self, name=None, required=True, null_or_empty=False, regex=None, min=None, max=None, precision=2): 13 | super(DecimalColumn, self).__init__(self.COLUMN_TYPE, name, required, null_or_empty) 14 | 15 | self.regex = self.register_property( 16 | ConfigProperty('regex', regex, 'Regular expression to validate the column value.') 17 | ) 18 | self.min = self.register_property( 19 | ConfigProperty('min', min, 'The minimum value. null for no limit.') 20 | ) 21 | self.max = self.register_property( 22 | ConfigProperty('max', max, 'The maximum value. null for no limit.') 23 | ) 24 | self.precision = self.register_property( 25 | ConfigProperty('precision', precision, 'The decimal point precision.') 26 | ) 27 | 28 | def on_validate(self): 29 | """Validates that each property has the correct value/type. 30 | 31 | Returns: 32 | List of error messages or an empty list. 33 | """ 34 | errors = super(DecimalColumn, self).on_validate() 35 | 36 | if self.regex.value is not None and not isinstance(self.regex.value, str): 37 | errors.append('"regex" must be a string.') 38 | 39 | min_set = self.min.value is not None 40 | min_is_number = isinstance(self.min.value, int) or \ 41 | isinstance(self.min.value, float) or \ 42 | isinstance(self.min.value, dec.Decimal) 43 | if min_set and not min_is_number: 44 | errors.append('"min" must be a number.') 45 | 46 | max_set = self.max.value is not None 47 | max_is_number = isinstance(self.max.value, int) or \ 48 | isinstance(self.max.value, float) or \ 49 | isinstance(self.max.value, dec.Decimal) 50 | if max_set and not max_is_number: 51 | errors.append('"max" must be a number.') 52 | 53 | if min_is_number and max_is_number: 54 | if self.min.value > self.max.value: 55 | errors.append('"min" must be less than or equal to "max".') 56 | if self.max.value < self.min.value: 57 | errors.append('"max" must be greater than or equal to "min".') 58 | 59 | if self.precision.value is not None: 60 | if not isinstance(self.precision.value, int): 61 | errors.append('"precision" must be an integer.') 62 | else: 63 | if self.precision.value < 1: 64 | errors.append('"precision" must be greater than or equal to 1.') 65 | 66 | return errors 67 | 68 | def on_validate_value(self, row_number, value): 69 | errors = [] 70 | decimal_value = None 71 | 72 | if re.search(r'^[-+]?\d*[.,]\d*$', value) is not None: 73 | decimal_value = dec.Decimal(value) 74 | elif len(value.strip()) == 0: 75 | # If the value is an empty string then convert it to None. 76 | value = None 77 | 78 | if value is not None and decimal_value is None: 79 | self.add_value_error(errors, row_number, value, 80 | 'must be a decimal') 81 | 82 | if self.regex.value and not re.search(self.regex.value, value) is not None: 83 | self.add_value_error(errors, row_number, value, 84 | 'does not match regex: "{0}"'.format(self.regex.value)) 85 | 86 | if self.min.value is not None and decimal_value is not None and decimal_value < self.min.value: 87 | self.add_value_error(errors, row_number, value, 88 | 'must be greater than or equal to: "{0}"'.format(self.min.value)) 89 | 90 | if self.max.value is not None and decimal_value is not None and decimal_value > self.max.value: 91 | self.add_value_error(errors, row_number, value, 92 | 'must be less than or equal to: "{0}"'.format(self.max.value)) 93 | 94 | if self.precision.value is not None and decimal_value is not None: 95 | decimal_char = localeconv()['decimal_point'] 96 | count = value[::-1].find(decimal_char) 97 | if count != self.precision.value: 98 | self.add_value_error(errors, row_number, value, 99 | 'precision must be: "{0}"'.format(self.precision.value)) 100 | 101 | return errors 102 | -------------------------------------------------------------------------------- /src/csv_schema/core/models/enum_column.py: -------------------------------------------------------------------------------- 1 | from .base_column import BaseColumn 2 | from .column_types import ColumnTypes 3 | from .config_property import ConfigProperty 4 | 5 | 6 | class EnumColumn(BaseColumn): 7 | COLUMN_TYPE = ColumnTypes.ENUM 8 | 9 | def __init__(self, name=None, required=True, null_or_empty=False, values=ConfigProperty.NotSpecified()): 10 | super(EnumColumn, self).__init__(self.COLUMN_TYPE, name, required, null_or_empty) 11 | 12 | self.values = self.register_property( 13 | ConfigProperty('values', values, 'Fixed set of constants.', default=list) 14 | ) 15 | 16 | def on_validate(self): 17 | """Validates that each property has the correct value/type. 18 | 19 | Returns: 20 | List of error messages or an empty list. 21 | """ 22 | errors = super(EnumColumn, self).on_validate() 23 | 24 | is_list = isinstance(self.values.value, list) 25 | 26 | if not is_list: 27 | errors.append('"values" must be a list.') 28 | 29 | if is_list and len(self.values.value) == 0: 30 | errors.append('"values" must contain at least one value.') 31 | 32 | return errors 33 | 34 | def on_validate_value(self, row_number, value): 35 | errors = [] 36 | 37 | if (value is None or len(value.strip()) == 0) and self.null_or_empty.value is True: 38 | # Null/empty values are allowed. 39 | pass 40 | elif value is not None and value not in self.values.value: 41 | self.add_value_error(errors, row_number, value, 42 | 'must be one of: "{0}"'.format(','.join(self.values.value))) 43 | 44 | return errors 45 | -------------------------------------------------------------------------------- /src/csv_schema/core/models/integer_column.py: -------------------------------------------------------------------------------- 1 | import re 2 | from .base_column import BaseColumn 3 | from .column_types import ColumnTypes 4 | from .config_property import ConfigProperty 5 | 6 | 7 | class IntegerColumn(BaseColumn): 8 | COLUMN_TYPE = ColumnTypes.INTEGER 9 | 10 | def __init__(self, name=None, required=True, null_or_empty=False, regex=None, min=None, max=None): 11 | super(IntegerColumn, self).__init__(self.COLUMN_TYPE, name, required, null_or_empty) 12 | 13 | self.regex = self.register_property( 14 | ConfigProperty('regex', regex, 'Regular expression to validate the column value.') 15 | ) 16 | self.min = self.register_property( 17 | ConfigProperty('min', min, 'The minimum value. null for no limit.') 18 | ) 19 | self.max = self.register_property( 20 | ConfigProperty('max', max, 'The maximum value. null for no limit.') 21 | ) 22 | 23 | def on_validate(self): 24 | """Validates that each property has the correct value/type. 25 | 26 | Returns: 27 | List of error messages or an empty list. 28 | """ 29 | errors = super(IntegerColumn, self).on_validate() 30 | 31 | if self.regex.value is not None and not isinstance(self.regex.value, str): 32 | errors.append('"regex" must be a string.') 33 | 34 | min_set = self.min.value is not None 35 | min_is_int = isinstance(self.min.value, int) 36 | if min_set and not min_is_int: 37 | errors.append('"min" must be an integer.') 38 | 39 | max_set = self.max.value is not None 40 | max_is_int = isinstance(self.max.value, int) 41 | if max_set and not max_is_int: 42 | errors.append('"max" must be an integer.') 43 | 44 | if min_is_int and max_is_int: 45 | if self.min.value > self.max.value: 46 | errors.append('"min" must be less than or equal to "max".') 47 | if self.max.value < self.min.value: 48 | errors.append('"max" must be greater than or equal to "min".') 49 | 50 | return errors 51 | 52 | def on_validate_value(self, row_number, value): 53 | errors = [] 54 | int_value = None 55 | 56 | if re.search(r'^[-+]?[0-9]+$', value) is not None: 57 | int_value = int(value) 58 | elif len(value.strip()) == 0: 59 | # If the value is an empty string then convert it to None. 60 | value = None 61 | 62 | if value is not None and int_value is None: 63 | self.add_value_error(errors, row_number, value, 64 | 'must be an integer') 65 | 66 | if self.regex.value and re.search(self.regex.value, value) is None: 67 | self.add_value_error(errors, row_number, value, 68 | 'does not match regex: "{0}"'.format(self.regex.value)) 69 | 70 | if self.min.value is not None and int_value is not None and int_value < self.min.value: 71 | self.add_value_error(errors, row_number, value, 72 | 'must be greater than or equal to: "{0}"'.format(self.min.value)) 73 | 74 | if self.max.value is not None and int_value is not None and int_value > self.max.value: 75 | self.add_value_error(errors, row_number, value, 76 | 'must be less than or equal to: "{0}"'.format(self.max.value)) 77 | 78 | return errors 79 | -------------------------------------------------------------------------------- /src/csv_schema/core/models/schema_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from ..utils import Utils 4 | from .base_config_object import BaseConfigObject 5 | from .config_property import ConfigProperty 6 | from .schema_config_filename import SchemaConfigFilename 7 | 8 | 9 | class SchemaConfig(BaseConfigObject): 10 | 11 | def __init__(self, path, name=None, description=None, columns=ConfigProperty.NotSpecified()): 12 | super(SchemaConfig, self).__init__() 13 | 14 | self.path = Utils.expand_path(path) 15 | self.name = self.register_property( 16 | ConfigProperty('name', name, 'The name of the schema.') 17 | ) 18 | self.description = self.register_property( 19 | ConfigProperty('description', description, 'The description of the schema.') 20 | ) 21 | self.filename = self.register_property( 22 | ConfigProperty('filename', SchemaConfigFilename(), 23 | 'Properties for the name of the CSV filename to validate.', default=SchemaConfigFilename) 24 | ) 25 | self.columns = self.register_property( 26 | ConfigProperty('columns', columns, 'List of column definitions.', default=list) 27 | ) 28 | 29 | def load(self): 30 | """Loads a JSON file from self.path into self. 31 | 32 | Returns: 33 | Self 34 | """ 35 | if not os.path.isfile(self.path): 36 | raise FileNotFoundError(self.path) 37 | 38 | with open(self.path, mode='r') as f: 39 | self.from_json(f.read()) 40 | 41 | return self 42 | 43 | def save(self): 44 | """Saves self as JSON to self.path. 45 | 46 | Returns: 47 | Self 48 | """ 49 | with open(self.path, 'w') as f: 50 | json.dump(self.to_dict(), f, indent=2) 51 | return self 52 | 53 | def on_validate(self): 54 | """Validates that each property has the correct value/type. 55 | 56 | Returns: 57 | List of error messages or an empty list. 58 | """ 59 | errors = [] 60 | 61 | if self.name.value is None or len(self.name.value.strip()) == 0: 62 | errors.append('"name" must be specified.') 63 | 64 | if self.columns.value is None or len(self.columns.value) == 0: 65 | errors.append('"columns" must have at least one item.') 66 | 67 | if self.filename.value is not None and not isinstance(self.filename.value, SchemaConfigFilename): 68 | errors.append('"filename" must be of type: SchemaConfigFilename') 69 | 70 | return errors 71 | -------------------------------------------------------------------------------- /src/csv_schema/core/models/schema_config_filename.py: -------------------------------------------------------------------------------- 1 | from .base_config_object import BaseConfigObject 2 | from .config_property import ConfigProperty 3 | 4 | 5 | class SchemaConfigFilename(BaseConfigObject): 6 | 7 | def __init__(self, regex=None): 8 | super(SchemaConfigFilename, self).__init__() 9 | self.regex = self.register_property( 10 | ConfigProperty('regex', regex, 'Regular expression to validate the name of the CSV file being validated.') 11 | ) 12 | -------------------------------------------------------------------------------- /src/csv_schema/core/models/string_column.py: -------------------------------------------------------------------------------- 1 | import re 2 | from .base_column import BaseColumn 3 | from .column_types import ColumnTypes 4 | from .config_property import ConfigProperty 5 | 6 | 7 | class StringColumn(BaseColumn): 8 | COLUMN_TYPE = ColumnTypes.STRING 9 | 10 | def __init__(self, name=None, required=True, null_or_empty=False, regex=None, min=None, max=None): 11 | super(StringColumn, self).__init__(self.COLUMN_TYPE, name, required, null_or_empty) 12 | 13 | self.regex = self.register_property( 14 | ConfigProperty('regex', regex, 'Regular expression to validate the column value.') 15 | ) 16 | self.min = self.register_property( 17 | ConfigProperty('min', min, 'The minimum length of the string. null for no limit.') 18 | ) 19 | self.max = self.register_property( 20 | ConfigProperty('max', max, 'The maximum length of the string. null for no limit.') 21 | ) 22 | 23 | def on_validate(self): 24 | """Validates that each property has the correct value/type. 25 | 26 | Returns: 27 | List of error messages or an empty list. 28 | """ 29 | errors = super(StringColumn, self).on_validate() 30 | 31 | if self.regex.value is not None and not isinstance(self.regex.value, str): 32 | errors.append('"regex" must be a string.') 33 | 34 | min_set = self.min.value is not None 35 | min_is_int = isinstance(self.min.value, int) 36 | if min_set and not min_is_int: 37 | errors.append('"min" must be an integer.') 38 | 39 | max_set = self.max.value is not None 40 | max_is_int = isinstance(self.max.value, int) 41 | if max_set and not max_is_int: 42 | errors.append('"max" must be an integer.') 43 | 44 | if min_is_int and max_is_int: 45 | if self.min.value > self.max.value: 46 | errors.append('"min" must be less than or equal to "max".') 47 | if self.max.value < self.min.value: 48 | errors.append('"max" must be greater than or equal to "min".') 49 | 50 | return errors 51 | 52 | def on_validate_value(self, row_number, value): 53 | errors = [] 54 | 55 | if self.regex.value and not re.search(self.regex.value, value) is not None: 56 | self.add_value_error(errors, row_number, value, 57 | 'does not match regex: "{0}"'.format(self.regex.value)) 58 | 59 | if self.min.value is not None and len(value) < self.min.value: 60 | self.add_value_error(errors, row_number, value, 61 | 'must be greater than or equal to: "{0}"'.format(self.min.value)) 62 | 63 | if self.max.value is not None and len(value) > self.max.value: 64 | self.add_value_error(errors, row_number, value, 65 | 'must be less than or equal to: "{0}"'.format(self.max.value)) 66 | 67 | return errors 68 | -------------------------------------------------------------------------------- /src/csv_schema/core/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import math 3 | 4 | 5 | class Utils: 6 | KB = 1024 7 | MB = KB * KB 8 | CHUNK_SIZE = 10 * MB 9 | 10 | @staticmethod 11 | def expand_path(local_path): 12 | var_path = os.path.expandvars(local_path) 13 | expanded_path = os.path.expanduser(var_path) 14 | return os.path.abspath(expanded_path) 15 | 16 | @staticmethod 17 | def norm_os_path_sep(path): 18 | """Normalizes the path separator for the current operating system. 19 | 20 | Args: 21 | path: Path to normalize. 22 | 23 | Returns: 24 | Path with normalized path separators. 25 | """ 26 | if os.sep == '/': 27 | return path.replace('\\', '/') 28 | else: 29 | return path.replace('/', '\\') 30 | 31 | @staticmethod 32 | def ensure_dirs(local_path): 33 | """Ensures the directories in local_path exist. 34 | 35 | Args: 36 | local_path: The local path to ensure. 37 | 38 | Returns: 39 | None 40 | """ 41 | if not os.path.isdir(local_path): 42 | os.makedirs(local_path) 43 | 44 | @staticmethod 45 | def split_chunk(list, chunk_size): 46 | """Yield successive n-sized chunks from a list. 47 | 48 | Args: 49 | list: The list to chunk. 50 | chunk_size: The max chunk size. 51 | 52 | Returns: 53 | List of lists. 54 | """ 55 | for i in range(0, len(list), chunk_size): 56 | yield list[i:i + chunk_size] 57 | 58 | @staticmethod 59 | def pretty_size(size): 60 | if size > 0: 61 | i = int(math.floor(math.log(size, 1024))) 62 | p = math.pow(1024, i) 63 | s = round(size / p, 2) 64 | else: 65 | i = 0 66 | s = 0 67 | return '{0} {1}'.format(s, Utils.PRETTY_SIZE_NAMES[i]) 68 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcstout/csv-schema-py/3e77dfb1a5695ef6e4547549d56f8cf023f08f1c/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | import shutil 4 | import tempfile 5 | import uuid 6 | from src.csv_schema.core.models import SchemaConfig, StringColumn 7 | 8 | 9 | @pytest.fixture 10 | def config_path(mk_tempfile): 11 | return mk_tempfile(suffix='.json') 12 | 13 | 14 | @pytest.fixture 15 | def mk_csv_file(mk_tempdir): 16 | def _mk(filename='test.csv', rows=None): 17 | tmpdir = mk_tempdir() 18 | path = os.path.join(tmpdir, filename) 19 | 20 | with open(path, mode='w') as f: 21 | f.writelines(os.linesep.join(rows)) 22 | 23 | return path 24 | 25 | yield _mk 26 | 27 | 28 | @pytest.fixture 29 | def mk_valid_csv(mk_csv_file): 30 | def _mk(filename='test.csv'): 31 | rows = ['col1,col2,col3'] 32 | rows.append('a1,b1,1') 33 | rows.append('a2,b2,2') 34 | rows.append('a3,b3,3') 35 | return mk_csv_file(filename=filename, rows=rows) 36 | 37 | yield _mk 38 | 39 | 40 | @pytest.fixture 41 | def valid_csv_path(mk_valid_csv): 42 | return mk_valid_csv() 43 | 44 | 45 | @pytest.fixture 46 | def invalid_csv_path(mk_invalid_csv): 47 | return mk_invalid_csv() 48 | 49 | 50 | @pytest.fixture 51 | def mk_invalid_csv(mk_csv_file): 52 | def _mk(filename='test.csv'): 53 | rows = ['col1,col2,col3WRONG'] 54 | rows.append('a1,b1,1') 55 | rows.append('a2,b2,2') 56 | rows.append('a3,b3,3') 57 | return mk_csv_file(filename=filename, rows=rows) 58 | 59 | yield _mk 60 | 61 | 62 | @pytest.fixture 63 | def valid_config_path(populated_config): 64 | assert populated_config.is_valid() is True 65 | populated_config.save() 66 | return populated_config.path 67 | 68 | 69 | @pytest.fixture 70 | def invalid_config_path(populated_config): 71 | populated_config.properties[0].value = None 72 | assert populated_config.is_valid() is False 73 | populated_config.save() 74 | return populated_config.path 75 | 76 | 77 | @pytest.fixture 78 | def empty_config(config_path): 79 | return SchemaConfig(config_path) 80 | 81 | 82 | @pytest.fixture 83 | def populated_config(config_path): 84 | columns = [ 85 | StringColumn('col1'), 86 | StringColumn('col2'), 87 | StringColumn('col3') 88 | ] 89 | config = SchemaConfig(config_path, 90 | name='Populated Config', 91 | description='A Description', 92 | columns=columns) 93 | config.save() 94 | return config 95 | 96 | 97 | @pytest.fixture() 98 | def mk_tempdir(): 99 | created = [] 100 | 101 | def _mk(): 102 | path = tempfile.mkdtemp() 103 | created.append(path) 104 | return path 105 | 106 | yield _mk 107 | 108 | for path in created: 109 | if os.path.isdir(path): 110 | shutil.rmtree(path) 111 | 112 | 113 | @pytest.fixture() 114 | def mk_tempfile(mk_tempdir): 115 | temp_dir = mk_tempdir() 116 | 117 | def _mk(content=uuid.uuid4().hex, suffix=None): 118 | fd, tmp_filename = tempfile.mkstemp(dir=temp_dir, suffix=suffix) 119 | with os.fdopen(fd, 'w') as tmp: 120 | tmp.write(content) 121 | return tmp_filename 122 | 123 | yield _mk 124 | 125 | if os.path.isdir(temp_dir): 126 | shutil.rmtree(temp_dir) 127 | -------------------------------------------------------------------------------- /tests/csv_schema/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcstout/csv-schema-py/3e77dfb1a5695ef6e4547549d56f8cf023f08f1c/tests/csv_schema/__init__.py -------------------------------------------------------------------------------- /tests/csv_schema/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcstout/csv-schema-py/3e77dfb1a5695ef6e4547549d56f8cf023f08f1c/tests/csv_schema/commands/__init__.py -------------------------------------------------------------------------------- /tests/csv_schema/commands/generate_config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcstout/csv-schema-py/3e77dfb1a5695ef6e4547549d56f8cf023f08f1c/tests/csv_schema/commands/generate_config/__init__.py -------------------------------------------------------------------------------- /tests/csv_schema/commands/generate_config/test_cli.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from src.csv_schema.cli import main 3 | 4 | 5 | def expect_exit_code(code, args): 6 | with pytest.raises(SystemExit) as pytest_wrapped_e: 7 | main(['generate-config'] + args) 8 | assert pytest_wrapped_e.type == SystemExit 9 | assert pytest_wrapped_e.value.code == code 10 | 11 | 12 | def test_it_passes(mk_tempdir): 13 | expect_exit_code(0, [mk_tempdir()]) 14 | 15 | 16 | def test_it_passes_with_csv(mk_tempdir): 17 | expect_exit_code(0, [mk_tempdir(), '--with-csv']) 18 | 19 | 20 | def test_it_fails(mk_tempdir, mocker): 21 | mocker.patch('src.csv_schema.core.models.schema_config.SchemaConfig.validate', return_value=['test error']) 22 | expect_exit_code(1, [mk_tempdir()]) 23 | -------------------------------------------------------------------------------- /tests/csv_schema/commands/generate_config/test_generate_config.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | from src.csv_schema.commands.generate_config import GenerateConfig 4 | from src.csv_schema.commands.validate_config import ValidateConfig 5 | from src.csv_schema.core.models import ColumnTypes, SchemaConfig 6 | 7 | 8 | def assert_success(gc): 9 | assert not gc.errors 10 | assert os.path.isfile(gc.path) 11 | found_types = [c.type.value for c in gc.config.columns.value] 12 | assert found_types == ColumnTypes.ALL 13 | loaded = SchemaConfig(gc.path).load() 14 | assert loaded.to_dict() == gc.config.to_dict() 15 | 16 | 17 | def test_it_generates_the_config_file(mk_tempdir): 18 | temp_dir = mk_tempdir() 19 | gc = GenerateConfig(temp_dir) 20 | gc.execute() 21 | assert_success(gc) 22 | 23 | 24 | def test_it_generates_a_csv_file(mk_tempdir): 25 | temp_dir = mk_tempdir() 26 | gc = GenerateConfig(temp_dir, with_csv=True, with_csv_rows=10) 27 | gc.execute() 28 | assert_success(gc) 29 | assert os.path.isfile(gc.csv_path) 30 | vc = ValidateConfig(gc.path) 31 | vc.execute() 32 | assert not vc.errors 33 | with open(gc.csv_path) as f: 34 | assert len(f.readlines()) == 11 35 | 36 | 37 | def test_it_adds_the_filename_for_dir_paths(mk_tempdir): 38 | temp_dir = mk_tempdir() 39 | gc = GenerateConfig(temp_dir) 40 | gc.execute() 41 | assert_success(gc) 42 | assert os.path.dirname(gc.path) == temp_dir 43 | assert os.path.basename(gc.path) == 'csv-schema.json' 44 | 45 | 46 | def test_it_does_not_add_the_filename_for_file_paths(mk_tempfile): 47 | temp_dir = mk_tempfile(suffix=".JsOn") 48 | gc = GenerateConfig(temp_dir) 49 | gc.execute() 50 | assert_success(gc) 51 | assert gc.path == temp_dir 52 | assert os.path.basename(gc.path) == os.path.basename(temp_dir) 53 | 54 | 55 | def test_it_creates_dirs(mk_tempdir): 56 | temp_dir = mk_tempdir() 57 | temp_path = os.path.join(temp_dir, 'dir1', 'dir2', 'dir3') 58 | gc = GenerateConfig(temp_path) 59 | gc.execute() 60 | assert_success(gc) 61 | assert os.path.dirname(gc.path) == temp_path 62 | assert os.path.basename(gc.path) == 'csv-schema.json' 63 | 64 | temp_path = os.path.join(temp_dir, 'dir1', 'dir2', 'dir3', 'test.json') 65 | gc = GenerateConfig(temp_path) 66 | gc.execute() 67 | assert_success(gc) 68 | assert os.path.dirname(gc.path) == os.path.dirname(temp_path) 69 | assert os.path.basename(gc.path) == 'test.json' 70 | -------------------------------------------------------------------------------- /tests/csv_schema/commands/validate_config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcstout/csv-schema-py/3e77dfb1a5695ef6e4547549d56f8cf023f08f1c/tests/csv_schema/commands/validate_config/__init__.py -------------------------------------------------------------------------------- /tests/csv_schema/commands/validate_config/test_cli.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from src.csv_schema.cli import main 3 | 4 | 5 | def expect_exit_code(config_path, code): 6 | with pytest.raises(SystemExit) as pytest_wrapped_e: 7 | main(['validate-config', config_path]) 8 | assert pytest_wrapped_e.type == SystemExit 9 | assert pytest_wrapped_e.value.code == code 10 | 11 | 12 | def test_it_passes(valid_config_path): 13 | expect_exit_code(valid_config_path, 0) 14 | 15 | 16 | def test_it_fails(invalid_config_path): 17 | expect_exit_code(invalid_config_path, 1) 18 | -------------------------------------------------------------------------------- /tests/csv_schema/commands/validate_config/test_validate_config.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from src.csv_schema.commands.validate_config import ValidateConfig 3 | 4 | 5 | def test_it_passes_validation(valid_config_path): 6 | vc = ValidateConfig(valid_config_path) 7 | vc.execute() 8 | assert len(vc.errors) == 0 9 | 10 | 11 | def test_it_does_not_pass_validation(invalid_config_path): 12 | vc = ValidateConfig(invalid_config_path) 13 | vc.execute() 14 | assert len(vc.errors) == 1 15 | -------------------------------------------------------------------------------- /tests/csv_schema/commands/validate_csv/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pcstout/csv-schema-py/3e77dfb1a5695ef6e4547549d56f8cf023f08f1c/tests/csv_schema/commands/validate_csv/__init__.py -------------------------------------------------------------------------------- /tests/csv_schema/commands/validate_csv/test_cli.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from src.csv_schema.cli import main 3 | 4 | 5 | def expect_exit_code(csv_path, config_path, code): 6 | with pytest.raises(SystemExit) as pytest_wrapped_e: 7 | main(['validate-csv', csv_path, config_path]) 8 | assert pytest_wrapped_e.type == SystemExit 9 | assert pytest_wrapped_e.value.code == code 10 | 11 | 12 | def test_it_passes(valid_csv_path, valid_config_path): 13 | expect_exit_code(valid_csv_path, valid_config_path, 0) 14 | 15 | 16 | def test_it_fails(invalid_csv_path, invalid_config_path): 17 | expect_exit_code(invalid_csv_path, invalid_config_path, 1) 18 | -------------------------------------------------------------------------------- /tests/csv_schema/commands/validate_csv/test_validate_csv.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | from src.csv_schema.commands.validate_csv import ValidateCsv 4 | 5 | 6 | def test_it_validates_the_csv_filename(populated_config, mk_valid_csv): 7 | correct_csv_path = mk_valid_csv('test.csv') 8 | incorrect_csv_path = mk_valid_csv('testX.csv') 9 | 10 | # Incorrect path with no regex - should pass 11 | vc = ValidateCsv(incorrect_csv_path, populated_config.path) 12 | vc.execute() 13 | assert vc.config.filename.value.regex.value is None 14 | assert len(vc.errors) == 0 15 | 16 | # Correct path with regex - should pass 17 | populated_config.filename.value.regex.value = r'^test\.csv$' 18 | populated_config.save() 19 | vc = ValidateCsv(correct_csv_path, populated_config.path) 20 | vc.execute() 21 | assert len(vc.errors) == 0 22 | 23 | # Incorrect path with regex - should not pass 24 | vc = ValidateCsv(incorrect_csv_path, populated_config.path) 25 | vc.execute() 26 | assert len(vc.errors) > 0 27 | assert 'does not match regex' in vc.errors[0] 28 | 29 | 30 | def test_it_validates_the_headers(populated_config, valid_csv_path, invalid_csv_path): 31 | vc = ValidateCsv(valid_csv_path, populated_config.path) 32 | vc.execute() 33 | assert len(vc.errors) == 0 34 | 35 | vc = ValidateCsv(invalid_csv_path, populated_config.path) 36 | vc.execute() 37 | assert len(vc.errors) > 0 38 | assert 'Required column: "col3" not found.' in vc.errors[0] 39 | 40 | 41 | def test_it_validates_the_data(populated_config, valid_csv_path, invalid_csv_path, mk_csv_file): 42 | vc = ValidateCsv(valid_csv_path, populated_config.path) 43 | vc.execute() 44 | assert len(vc.errors) == 0 45 | 46 | missing_col_value_csv = mk_csv_file(rows=['col1,col2,col3', 'a,b,']) 47 | vc = ValidateCsv(missing_col_value_csv, populated_config.path) 48 | vc.execute() 49 | assert len(vc.errors) > 0 50 | assert vc.errors[0] == 'Row number: 1, column: "col3", value: "" cannot be null or empty.' 51 | 52 | missing_optional_column_csv = mk_csv_file(rows=['col2,col3', 'b,c']) 53 | populated_config.columns.value[0].required.value = False 54 | populated_config.save() 55 | vc = ValidateCsv(missing_optional_column_csv, populated_config.path) 56 | vc.execute() 57 | assert len(vc.errors) == 0 58 | -------------------------------------------------------------------------------- /tests/csv_schema/core/models/test_base_column.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import json 3 | from src.csv_schema.core.models import BaseColumn, ColumnTypes, ConfigProperty 4 | 5 | 6 | @pytest.fixture 7 | def col_type(): 8 | return ColumnTypes.STRING 9 | 10 | 11 | @pytest.fixture 12 | def col_name(): 13 | return 'test_col_a' 14 | 15 | 16 | @pytest.fixture 17 | def col_required(): 18 | return True 19 | 20 | 21 | @pytest.fixture 22 | def col_null_or_empty(): 23 | return False 24 | 25 | 26 | @pytest.fixture 27 | def col(col_type, col_name, col_required, col_null_or_empty): 28 | return BaseColumn(col_type, col_name, col_required, col_null_or_empty) 29 | 30 | 31 | def test_it_sets_the_column_data(col_type, col_name, col_required, col_null_or_empty, col): 32 | assert col.name.value == col_name 33 | assert col.required.value == col_required 34 | assert col.null_or_empty.value == col_null_or_empty 35 | 36 | 37 | def test_register_property(col): 38 | col.properties.clear() 39 | prop = ConfigProperty('test', None, '') 40 | 41 | # Adds it and returns it. 42 | assert col.register_property(prop) == prop 43 | assert prop in col.properties 44 | 45 | # Does not duplicate it. 46 | col.register_property(prop) 47 | assert prop in col.properties 48 | assert len(col.properties) == 1 49 | 50 | 51 | def test_to_dict(col_type, col_name, col_required, col_null_or_empty, col): 52 | d = col.to_dict() 53 | assert len(d) == 4 54 | assert d['type'] == col_type 55 | assert d['name'] == col_name 56 | assert d['required'] == col_required 57 | assert d['null_or_empty'] == col_null_or_empty 58 | 59 | 60 | def test_to_json(col): 61 | d = col.to_dict() 62 | j = col.to_json() 63 | assert json.loads(j) == d 64 | 65 | 66 | def test_to_md_help(col_type, col_name, col_required, col_null_or_empty, col): 67 | # TODO: How to test this? 68 | print(col.to_md_help()) 69 | 70 | 71 | def test_validate_value_must_be_string(): 72 | col = BaseColumn(name='col1') 73 | 74 | for string_value in ['a', '1']: 75 | errors = col.validate_value(1, string_value) 76 | assert not errors 77 | 78 | for non_string_value in [0, 1, 0.00, object()]: 79 | with pytest.raises(ValueError) as ex: 80 | col.validate_value(1, non_string_value) 81 | assert str(ex.value) == 'value must be a string.' 82 | 83 | 84 | def test_validate_value_null_or_empty(): 85 | col = BaseColumn(name='col1', null_or_empty=False) 86 | empty_values = ['', ' '] 87 | 88 | for empty_value in empty_values: 89 | errors = col.validate_value(1, empty_value) 90 | assert errors 91 | assert 'cannot be null or empty' in errors[0] 92 | 93 | col.null_or_empty.value = True 94 | for empty_value in empty_values: 95 | errors = col.validate_value(1, empty_value) 96 | assert not errors 97 | 98 | 99 | def test_add_value_error(col, col_name): 100 | errors = [] 101 | col.add_value_error(errors, 9, 'a', 'ERROR_STR1') 102 | assert errors[0] == 'Row number: 9, column: "{0}", value: "a" ERROR_STR1.'.format(col_name) 103 | col.add_value_error(errors, 9, 'a', 'ERROR_STR2') 104 | assert errors[1] == 'Row number: 9, column: "{0}", value: "a" ERROR_STR2.'.format(col_name) 105 | -------------------------------------------------------------------------------- /tests/csv_schema/core/models/test_base_config_object.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import json 3 | import uuid 4 | from src.csv_schema.core.models import BaseConfigObject, ConfigProperty 5 | 6 | class TestConfigChildChild(BaseConfigObject): 7 | __test__ = False 8 | def __init__(self): 9 | super(TestConfigChildChild, self).__init__() 10 | self.prop_cca = self.register_property(ConfigProperty('prop_cca')) 11 | self.prop_ccb = self.register_property(ConfigProperty('prop_ccb')) 12 | 13 | def on_validate(self): 14 | errors = [] 15 | if self.prop_cca.value != 'prop_cca': 16 | errors.append('prop_cca value should be: "prop_cca"') 17 | return errors 18 | 19 | 20 | class TestConfigChild(BaseConfigObject): 21 | __test__ = False 22 | def __init__(self): 23 | super(TestConfigChild, self).__init__() 24 | self.prop_ca = self.register_property(ConfigProperty('prop_ca')) 25 | self.prop_cb = self.register_property(ConfigProperty('prop_cb', default=TestConfigChildChild)) 26 | 27 | 28 | class TestConfig(BaseConfigObject): 29 | __test__ = False 30 | def __init__(self): 31 | super(TestConfig, self).__init__() 32 | self.prop_a = self.register_property(ConfigProperty('prop_a')) 33 | self.prop_b = self.register_property(ConfigProperty('prop_b')) 34 | self.prop_c = self.register_property(ConfigProperty('prop_c', default=TestConfigChild)) 35 | self.prop_d = self.register_property(ConfigProperty('prop_d', default=list)) 36 | self.prop_d.value.append(TestConfigChild()) 37 | 38 | def on_validate(self): 39 | errors = [] 40 | if self.prop_a.value != 'prop_a': 41 | errors.append('prop_a value should be: "prop_a"') 42 | return errors 43 | 44 | 45 | def assert_not_default(props): 46 | """Asserts that all properties in the property hierarchy are not set to the default value.""" 47 | for prop in props: 48 | if isinstance(prop.value, BaseConfigObject): 49 | assert_not_default(prop.value.properties) 50 | else: 51 | assert prop.value != prop.default 52 | 53 | 54 | def assert_are_default(props): 55 | """Asserts that all properties in the property hierarchy are not set to the default value.""" 56 | for prop in props: 57 | if isinstance(prop.value, BaseConfigObject): 58 | assert isinstance(prop.value, prop.default) 59 | assert_are_default(prop.value.properties) 60 | elif isinstance(prop.default, type): 61 | assert prop.value != prop.default 62 | assert isinstance(prop.value, prop.default) 63 | else: 64 | assert prop.value == prop.default 65 | 66 | 67 | def populate_props(obj): 68 | """Populates all the property values in the property hierarchy.""" 69 | for prop in obj.properties: 70 | if isinstance(prop.value, BaseConfigObject): 71 | populate_props(prop.value) 72 | else: 73 | prop.value = str(uuid.uuid4()) 74 | 75 | 76 | @pytest.fixture 77 | def test_config(): 78 | config = TestConfig() 79 | populate_props(config) 80 | return config 81 | 82 | 83 | def test_register_property(): 84 | config_obj = BaseConfigObject() 85 | prop = ConfigProperty('test') 86 | 87 | # Adds it and returns it. 88 | assert config_obj.register_property(prop) == prop 89 | assert prop in config_obj.properties 90 | 91 | # Does not duplicate it. 92 | config_obj.register_property(prop) 93 | assert prop in config_obj.properties 94 | assert len(config_obj.properties) == 1 95 | 96 | 97 | def test_clear(test_config): 98 | assert_not_default(test_config.properties) 99 | test_config.clear() 100 | assert_are_default(test_config.properties) 101 | 102 | 103 | def test_to_dict(test_config): 104 | d = test_config.to_dict() 105 | assert len(d) == len(test_config.properties) 106 | 107 | def assert_check(props, _dict): 108 | for prop in props: 109 | if isinstance(prop.value, BaseConfigObject): 110 | child_d = prop.value.to_dict() 111 | assert _dict[prop.name] == child_d 112 | assert_check(prop.value.properties, child_d) 113 | else: 114 | assert _dict[prop.name] == prop.value 115 | 116 | assert_check(test_config.properties, d) 117 | 118 | 119 | def test_to_json(test_config): 120 | d = test_config.to_dict() 121 | j = test_config.to_json() 122 | assert json.loads(j) == d 123 | 124 | 125 | def test_from_json(test_config): 126 | j = test_config.to_json() 127 | test_config.clear() 128 | test_config.from_json(j) 129 | assert test_config.to_json() == j 130 | 131 | 132 | def test_to_md_help(test_config): 133 | # TODO: How to test this? 134 | print(test_config.to_md_help()) 135 | 136 | 137 | def test_is_valid(test_config): 138 | assert test_config.is_valid() is False 139 | test_config.prop_a.value = 'prop_a' 140 | test_config.prop_c.value.prop_cb.value.prop_cca.value = 'prop_cca' 141 | assert test_config.is_valid() is True 142 | 143 | 144 | def test_validate(test_config): 145 | errors = test_config.validate() 146 | assert len(errors) == 2 147 | assert errors[0] == 'prop_a value should be: "prop_a"' 148 | assert errors[1] == '"prop_c" -> "prop_cb" -> prop_cca value should be: "prop_cca"' 149 | test_config.prop_a.value = 'prop_a' 150 | test_config.prop_c.value.prop_cb.value.prop_cca.value = 'prop_cca' 151 | errors = test_config.validate() 152 | assert len(errors) == 0 153 | -------------------------------------------------------------------------------- /tests/csv_schema/core/models/test_config_property.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import uuid 3 | from src.csv_schema.core.models import ConfigProperty, BaseConfigObject 4 | 5 | 6 | def test_it_defaults_the_value(): 7 | prop = ConfigProperty('a', default=None) 8 | assert prop.value is None 9 | 10 | prop = ConfigProperty('a', default='b') 11 | assert prop.value == 'b' 12 | 13 | prop = ConfigProperty('a') 14 | assert prop.value is None 15 | 16 | prop = ConfigProperty('a', default=BaseConfigObject) 17 | assert isinstance(prop.value, BaseConfigObject) 18 | 19 | 20 | def test_clear(): 21 | # Values 22 | for default in ['', None, 1, True, False, BaseConfigObject()]: 23 | prop = ConfigProperty('prop-name', str(uuid.uuid4()), '', default=default) 24 | assert prop.value != default 25 | prop.clear() 26 | assert prop.value == default 27 | 28 | # Types 29 | for default in [str, BaseConfigObject]: 30 | prop = ConfigProperty('prop-name', str(uuid.uuid4()), '', default=default) 31 | assert prop.value != default 32 | prop.clear() 33 | assert isinstance(prop.value, default) 34 | assert prop.value != default 35 | -------------------------------------------------------------------------------- /tests/csv_schema/core/models/test_decimal_column.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from src.csv_schema.core.models import DecimalColumn 3 | 4 | 5 | def test_validate_value_must_be_string(): 6 | col = DecimalColumn(name='col1') 7 | 8 | for string_value in ['0.00', '1.00']: 9 | errors = col.validate_value(1, string_value) 10 | assert not errors 11 | 12 | for non_string_value in [0, 1, 0.00, object()]: 13 | with pytest.raises(ValueError) as ex: 14 | col.validate_value(1, non_string_value) 15 | assert str(ex.value) == 'value must be a string.' 16 | 17 | 18 | def test_validate_value_null_or_empty(): 19 | col = DecimalColumn(name='col1', null_or_empty=False) 20 | empty_values = ['', ' '] 21 | 22 | for empty_value in empty_values: 23 | errors = col.validate_value(1, empty_value) 24 | assert errors 25 | assert 'cannot be null or empty' in errors[0] 26 | 27 | col.null_or_empty.value = True 28 | for empty_value in empty_values: 29 | errors = col.validate_value(1, empty_value) 30 | assert not errors 31 | 32 | 33 | def test_validate_value_is_decimal(): 34 | col = DecimalColumn(name='col1') 35 | 36 | for decimal_value in ['-9.00', '0.00', '9.00']: 37 | errors = col.validate_value(1, decimal_value) 38 | assert not errors 39 | 40 | for non_decimal_value in ['1', '1a', 'z']: 41 | errors = col.validate_value(1, non_decimal_value) 42 | assert errors 43 | assert 'must be a decimal' in errors[0] 44 | 45 | 46 | def test_validate_value_regex(): 47 | col = DecimalColumn(name='col1', regex='^[0-3].[0-3][0-3]$') 48 | 49 | for matching_value in ['0.00', '3.10']: 50 | errors = col.validate_value(1, matching_value) 51 | assert not errors 52 | 53 | for non_matching_value in ['10', 'a', 'z5']: 54 | errors = col.validate_value(1, non_matching_value) 55 | assert errors 56 | assert 'does not match regex' in errors[-1] 57 | 58 | 59 | def test_validate_value_min(): 60 | col = DecimalColumn(name='col1', min=3.5) 61 | 62 | for min_value in ['3.50', '5.10', '10.10', '1000.10']: 63 | errors = col.validate_value(1, min_value) 64 | assert not errors 65 | 66 | for non_min_value in ['3.40', '0.00', '-1.10', '-1000.10']: 67 | errors = col.validate_value(1, non_min_value) 68 | assert errors 69 | assert 'must be greater than or equal to' in errors[0] 70 | 71 | 72 | def test_validate_value_max(): 73 | col = DecimalColumn(name='col1', max=3.5) 74 | 75 | for max_value in ['-1000.00', '-1.00', '0.00', '1.10', '3.50']: 76 | errors = col.validate_value(1, max_value) 77 | assert not errors 78 | 79 | for non_max_value in ['3.60', '10.10', '1000.10']: 80 | errors = col.validate_value(1, non_max_value) 81 | assert errors 82 | assert 'must be less than or equal to' in errors[-1] 83 | 84 | 85 | def test_validate_precision(): 86 | col = DecimalColumn(name='col1', precision=1) 87 | errors = col.validate_value(1, '1.0') 88 | assert not errors 89 | 90 | col = DecimalColumn(name='col1', precision=2) 91 | errors = col.validate_value(1, '1.00') 92 | assert not errors 93 | 94 | for invalid_precision in ['1.0', '1.000']: 95 | errors = col.validate_value(1, invalid_precision) 96 | assert errors 97 | assert 'precision must be: "2"' in errors[0] 98 | -------------------------------------------------------------------------------- /tests/csv_schema/core/models/test_enum_column.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from src.csv_schema.core.models import EnumColumn 3 | 4 | 5 | @pytest.fixture 6 | def enum_values(): 7 | return ['a', 'b', 'c'] 8 | 9 | 10 | @pytest.fixture 11 | def col(enum_values): 12 | return EnumColumn(name='col1', values=enum_values) 13 | 14 | 15 | def test_validate_value_must_be_string(col, enum_values): 16 | for string_value in enum_values: 17 | errors = col.validate_value(1, string_value) 18 | assert not errors 19 | 20 | for non_string_value in [0, 1, 0.00, object()]: 21 | with pytest.raises(ValueError) as ex: 22 | col.validate_value(1, non_string_value) 23 | assert str(ex.value) == 'value must be a string.' 24 | 25 | 26 | def test_validate_value_null_or_empty(col): 27 | empty_values = ['', ' '] 28 | 29 | for empty_value in empty_values: 30 | errors = col.validate_value(1, empty_value) 31 | assert errors 32 | assert 'cannot be null or empty' in errors[0] 33 | 34 | col.null_or_empty.value = True 35 | for empty_value in empty_values: 36 | errors = col.validate_value(1, empty_value) 37 | assert not errors 38 | 39 | 40 | def test_validate_value_must_be_one_of_values(col, enum_values): 41 | for valid_value in enum_values: 42 | errors = col.validate_value(1, valid_value) 43 | assert not errors 44 | 45 | for invalid_value in ['1', 'z']: 46 | errors = col.validate_value(1, invalid_value) 47 | assert errors 48 | assert 'must be one of' in errors[0] 49 | -------------------------------------------------------------------------------- /tests/csv_schema/core/models/test_integer_column.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from src.csv_schema.core.models import IntegerColumn 3 | 4 | 5 | def test_validate_value_must_be_string(): 6 | col = IntegerColumn(name='col1') 7 | 8 | for string_value in ['1', '1000']: 9 | errors = col.validate_value(1, string_value) 10 | assert not errors 11 | 12 | for non_string_value in [0, 1, 0.00, object()]: 13 | with pytest.raises(ValueError) as ex: 14 | col.validate_value(1, non_string_value) 15 | assert str(ex.value) == 'value must be a string.' 16 | 17 | 18 | def test_validate_value_null_or_empty(): 19 | col = IntegerColumn(name='col1', null_or_empty=False) 20 | empty_values = ['', ' '] 21 | 22 | for empty_value in empty_values: 23 | errors = col.validate_value(1, empty_value) 24 | assert errors 25 | assert 'cannot be null or empty' in errors[0] 26 | 27 | col.null_or_empty.value = True 28 | for empty_value in empty_values: 29 | errors = col.validate_value(1, empty_value) 30 | assert not errors 31 | 32 | 33 | def test_validate_value_is_integer(): 34 | col = IntegerColumn(name='col1') 35 | 36 | for int_value in ['-1', '0', '1']: 37 | errors = col.validate_value(1, int_value) 38 | assert not errors 39 | 40 | for non_int_value in ['1a', 'z']: 41 | errors = col.validate_value(1, non_int_value) 42 | assert errors 43 | assert 'must be an integer' in errors[0] 44 | 45 | 46 | def test_validate_value_regex(): 47 | col = IntegerColumn(name='col1', regex='^[0-9]$') 48 | 49 | for matching_value in ['0', '5']: 50 | errors = col.validate_value(1, matching_value) 51 | assert not errors 52 | 53 | for non_matching_value in ['10', 'a', 'z5']: 54 | errors = col.validate_value(1, non_matching_value) 55 | assert errors 56 | assert 'does not match regex' in errors[-1] 57 | 58 | 59 | def test_validate_value_min(): 60 | col = IntegerColumn(name='col1', min=3) 61 | 62 | for min_value in ['3', '5', '10', '1000']: 63 | errors = col.validate_value(1, min_value) 64 | assert not errors 65 | 66 | for non_min_value in ['2', '0', '-1', '-1000']: 67 | errors = col.validate_value(1, non_min_value) 68 | assert errors 69 | assert 'must be greater than or equal to' in errors[0] 70 | 71 | 72 | def test_validate_value_max(): 73 | col = IntegerColumn(name='col1', max=3) 74 | 75 | for max_value in ['-1000', '-1', '0', '1', '3']: 76 | errors = col.validate_value(1, max_value) 77 | assert not errors 78 | 79 | for non_max_value in ['4', '10', '1000']: 80 | errors = col.validate_value(1, non_max_value) 81 | assert errors 82 | assert 'must be less than or equal to' in errors[-1] 83 | -------------------------------------------------------------------------------- /tests/csv_schema/core/models/test_schema_config.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | import json 4 | from src.csv_schema.core.models import SchemaConfig, SchemaConfigFilename, StringColumn 5 | 6 | 7 | def test_it_has_the_properties(empty_config, config_path): 8 | assert empty_config.path == os.path.abspath(config_path) 9 | 10 | assert len(empty_config.properties) == 4 11 | 12 | assert hasattr(empty_config, 'name') 13 | assert empty_config.name.value is None 14 | 15 | assert hasattr(empty_config, 'description') 16 | assert empty_config.description.value is None 17 | 18 | assert hasattr(empty_config, 'filename') 19 | assert isinstance(empty_config.filename.value, SchemaConfigFilename) 20 | 21 | assert hasattr(empty_config, 'columns') 22 | assert isinstance(empty_config.columns.value, list) 23 | assert len(empty_config.columns.value) == 0 24 | 25 | 26 | def test_on_validate(empty_config): 27 | assert empty_config.is_valid() is False 28 | errors = empty_config.validate() 29 | assert '"name" must be specified.' in errors 30 | assert '"columns" must have at least one item.' in errors 31 | 32 | empty_config.filename.value = object() 33 | errors = empty_config.validate() 34 | assert '"filename" must be of type: SchemaConfigFilename' in errors 35 | empty_config.filename.clear() 36 | 37 | empty_config.name.value = 'test' 38 | errors = empty_config.validate() 39 | assert '"name" must be specified.' not in errors 40 | 41 | empty_config.columns.value.append(StringColumn('col1')) 42 | errors = empty_config.validate() 43 | assert '"columns" must have at least one item.' not in errors 44 | assert len(errors) == 0 45 | 46 | # Child ConfigObjects in lists are validated. 47 | empty_config.columns.value.append(StringColumn(None)) 48 | errors = empty_config.validate() 49 | assert '"columns" -> "name" must be specified.' in errors 50 | assert len(errors) == 1 51 | 52 | 53 | def test_save(populated_config): 54 | assert populated_config.save() == populated_config 55 | 56 | with open(populated_config.path) as f: 57 | json_data = json.load(f) 58 | assert populated_config.to_dict() == json_data 59 | 60 | 61 | def test_load(populated_config): 62 | populated_config.save() 63 | new_config = SchemaConfig(populated_config.path).load() 64 | assert new_config.to_dict() == populated_config.to_dict() 65 | assert new_config.to_json() == populated_config.to_json() 66 | -------------------------------------------------------------------------------- /tests/csv_schema/core/models/test_string_column.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from src.csv_schema.core.models import StringColumn 3 | 4 | 5 | def test_validate_value_must_be_string(): 6 | col = StringColumn(name='col1') 7 | 8 | for string_value in ['a', 'abc']: 9 | errors = col.validate_value(1, string_value) 10 | assert not errors 11 | 12 | for non_string_value in [0, 1, 0.00, object()]: 13 | with pytest.raises(ValueError) as ex: 14 | col.validate_value(1, non_string_value) 15 | assert str(ex.value) == 'value must be a string.' 16 | 17 | 18 | def test_validate_value_null_or_empty(): 19 | col = StringColumn(name='col1', null_or_empty=False) 20 | empty_values = ['', ' '] 21 | 22 | for empty_value in empty_values: 23 | errors = col.validate_value(1, empty_value) 24 | assert errors 25 | assert 'cannot be null or empty' in errors[0] 26 | 27 | col.null_or_empty.value = True 28 | for empty_value in empty_values: 29 | errors = col.validate_value(1, empty_value) 30 | assert not errors 31 | 32 | 33 | def test_validate_value_regex(): 34 | col = StringColumn(name='col1', regex='^[a-c]$') 35 | 36 | for matching_value in ['a', 'b', 'c']: 37 | errors = col.validate_value(1, matching_value) 38 | assert not errors 39 | 40 | for non_matching_value in ['d', 'e', 'f', '0', '1']: 41 | errors = col.validate_value(1, non_matching_value) 42 | assert errors 43 | assert 'does not match regex' in errors[0] 44 | 45 | 46 | def test_validate_value_min(): 47 | col = StringColumn(name='col1', min=3) 48 | 49 | for min_value in ['aaa', 'aaaa']: 50 | errors = col.validate_value(1, min_value) 51 | assert not errors 52 | 53 | for non_min_value in ['a', 'aa']: 54 | errors = col.validate_value(1, non_min_value) 55 | assert errors 56 | assert 'must be greater than or equal to' in errors[0] 57 | 58 | 59 | def test_validate_value_max(): 60 | col = StringColumn(name='col1', max=3) 61 | 62 | for max_value in ['a', 'aa', 'aaa']: 63 | errors = col.validate_value(1, max_value) 64 | assert not errors 65 | 66 | for non_max_value in ['aaaa', 'aaaaaa']: 67 | errors = col.validate_value(1, non_max_value) 68 | assert errors 69 | assert 'must be less than or equal to' in errors[0] 70 | --------------------------------------------------------------------------------