├── .gitignore ├── LICENSE ├── Pipfile ├── Pipfile.lock ├── README.md ├── env.ini.example ├── scrapper ├── __init__.py └── scrapper.py └── tests ├── __init__.py ├── test_login.py └── test_pesquisa.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Felipe Andrade 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | 3 | url = "https://pypi.python.org/simple" 4 | verify_ssl = true 5 | name = "pypi" 6 | 7 | 8 | [packages] 9 | 10 | requests-html = "*" 11 | peewee = "*" 12 | flask = "*" 13 | PyMySQL = "*" 14 | 15 | 16 | [dev-packages] 17 | 18 | 19 | 20 | [requires] 21 | 22 | python_version = "3.6" 23 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "d61db04f3ea5b7eb1f47125b59995c8c5d785c28ff7341cea848043e4e56b37e" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.6" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.python.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "appdirs": { 20 | "hashes": [ 21 | "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", 22 | "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" 23 | ], 24 | "version": "==1.4.3" 25 | }, 26 | "asn1crypto": { 27 | "hashes": [ 28 | "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87", 29 | "sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49" 30 | ], 31 | "version": "==0.24.0" 32 | }, 33 | "beautifulsoup4": { 34 | "hashes": [ 35 | "sha256:194ec62a25438adcb3fdb06378b26559eda1ea8a747367d34c33cef9c7f48d57", 36 | "sha256:90f8e61121d6ae58362ce3bed8cd997efb00c914eae0ff3d363c32f9a9822d10", 37 | "sha256:f0abd31228055d698bb392a826528ea08ebb9959e6bea17c606fd9c9009db938" 38 | ], 39 | "version": "==4.6.3" 40 | }, 41 | "bs4": { 42 | "hashes": [ 43 | "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a" 44 | ], 45 | "version": "==0.0.1" 46 | }, 47 | "certifi": { 48 | "hashes": [ 49 | "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", 50 | "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" 51 | ], 52 | "version": "==2018.8.24" 53 | }, 54 | "cffi": { 55 | "hashes": [ 56 | "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", 57 | "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", 58 | "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", 59 | "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", 60 | "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", 61 | "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", 62 | "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", 63 | "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", 64 | "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", 65 | "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", 66 | "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", 67 | "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", 68 | "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", 69 | "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", 70 | "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", 71 | "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", 72 | "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", 73 | "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", 74 | "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", 75 | "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", 76 | "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", 77 | "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", 78 | "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", 79 | "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", 80 | "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", 81 | "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", 82 | "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", 83 | "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", 84 | "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", 85 | "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", 86 | "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", 87 | "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" 88 | ], 89 | "version": "==1.11.5" 90 | }, 91 | "chardet": { 92 | "hashes": [ 93 | "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", 94 | "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" 95 | ], 96 | "version": "==3.0.4" 97 | }, 98 | "click": { 99 | "hashes": [ 100 | "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", 101 | "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" 102 | ], 103 | "version": "==6.7" 104 | }, 105 | "cryptography": { 106 | "hashes": [ 107 | "sha256:02602e1672b62e803e08617ec286041cc453e8d43f093a5f4162095506bc0beb", 108 | "sha256:10b48e848e1edb93c1d3b797c83c72b4c387ab0eb4330aaa26da8049a6cbede0", 109 | "sha256:17db09db9d7c5de130023657be42689d1a5f60502a14f6f745f6f65a6b8195c0", 110 | "sha256:227da3a896df1106b1a69b1e319dce218fa04395e8cc78be7e31ca94c21254bc", 111 | "sha256:2cbaa03ac677db6c821dac3f4cdfd1461a32d0615847eedbb0df54bb7802e1f7", 112 | "sha256:31db8febfc768e4b4bd826750a70c79c99ea423f4697d1dab764eb9f9f849519", 113 | "sha256:4a510d268e55e2e067715d728e4ca6cd26a8e9f1f3d174faf88e6f2cb6b6c395", 114 | "sha256:6a88d9004310a198c474d8a822ee96a6dd6c01efe66facdf17cb692512ae5bc0", 115 | "sha256:76936ec70a9b72eb8c58314c38c55a0336a2b36de0c7ee8fb874a4547cadbd39", 116 | "sha256:7e3b4aecc4040928efa8a7cdaf074e868af32c58ffc9bb77e7bf2c1a16783286", 117 | "sha256:8168bcb08403ef144ff1fb880d416f49e2728101d02aaadfe9645883222c0aa5", 118 | "sha256:8229ceb79a1792823d87779959184a1bf95768e9248c93ae9f97c7a2f60376a1", 119 | "sha256:8a19e9f2fe69f6a44a5c156968d9fc8df56d09798d0c6a34ccc373bb186cee86", 120 | "sha256:8d10113ca826a4c29d5b85b2c4e045ffa8bad74fb525ee0eceb1d38d4c70dfd6", 121 | "sha256:be495b8ec5a939a7605274b6e59fbc35e76f5ad814ae010eb679529671c9e119", 122 | "sha256:dc2d3f3b1548f4d11786616cf0f4415e25b0fbecb8a1d2cd8c07568f13fdde38", 123 | "sha256:e4aecdd9d5a3d06c337894c9a6e2961898d3f64fe54ca920a72234a3de0f9cb3", 124 | "sha256:e79ab4485b99eacb2166f3212218dd858258f374855e1568f728462b0e6ee0d9", 125 | "sha256:f995d3667301e1754c57b04e0bae6f0fa9d710697a9f8d6712e8cca02550910f" 126 | ], 127 | "version": "==2.3.1" 128 | }, 129 | "cssselect": { 130 | "hashes": [ 131 | "sha256:066d8bc5229af09617e24b3ca4d52f1f9092d9e061931f4184cd572885c23204", 132 | "sha256:3b5103e8789da9e936a68d993b70df732d06b8bb9a337a05ed4eb52c17ef7206" 133 | ], 134 | "version": "==1.0.3" 135 | }, 136 | "fake-useragent": { 137 | "hashes": [ 138 | "sha256:cc9b9ddcebc708b3deac846f5fccb16e37c02ee47435a4ec7132271dd96aec8c" 139 | ], 140 | "version": "==0.1.10" 141 | }, 142 | "flask": { 143 | "hashes": [ 144 | "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", 145 | "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05" 146 | ], 147 | "index": "pypi", 148 | "version": "==1.0.2" 149 | }, 150 | "idna": { 151 | "hashes": [ 152 | "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", 153 | "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" 154 | ], 155 | "version": "==2.7" 156 | }, 157 | "itsdangerous": { 158 | "hashes": [ 159 | "sha256:cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519" 160 | ], 161 | "version": "==0.24" 162 | }, 163 | "jinja2": { 164 | "hashes": [ 165 | "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", 166 | "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" 167 | ], 168 | "version": "==2.10" 169 | }, 170 | "lxml": { 171 | "hashes": [ 172 | "sha256:0cf1eca0652c4409e0655e04b840d6d85b7eb18718f5fba3862acad5500e3480", 173 | "sha256:10624ef1b468252309f269b13af4f837e3a82be366b5f3e49b0e83f1ad66205f", 174 | "sha256:1259e374da3a575615fe402e0966c5894bae3d2e229c2239ba4ebf2bb020c4b6", 175 | "sha256:26bb748af1ead0097eb8272b8a06f15a0015b8f312eef772a95f223a16e7de56", 176 | "sha256:27d0b13bcfcf2f6a5664e64fc3d684c76db1cdba5a5761795d154063559e0b59", 177 | "sha256:2b013fdabcbc21bc2770437099b921ec290235752b5baaac7a601f75094a378d", 178 | "sha256:2e469ea2c0b722b9b393187649e7d126c537a68512fc92a676fe86e57050c2a9", 179 | "sha256:37f7c2cdf513a0ea239c1609681880fb2f0073de0d2996e0ae9a7f0ef15d8b95", 180 | "sha256:68c6afc7a4411db2df28307e2493c945cb3d887e8f431b81811c1ea6ba087b8b", 181 | "sha256:73fe3452fc02c0b418914f842f897bdad0f1184368d8d9c315294ff7b94946f2", 182 | "sha256:7584d83d7315f641510e5f97f4d636ea225fd76e3f8aee965b2e8c93a8169b4d", 183 | "sha256:76e3ec6b26b1198dd5e6e20539d8360dcd3b224cd80cadba9307b790fda79161", 184 | "sha256:8288a889cbaa446e5fa168837456e63098b91953c89e5857968a5091b337cdca", 185 | "sha256:ad9e1fee284dec97b74cd88e925eca1575145598c974243cfb5e859f406adc32", 186 | "sha256:b360c3769cf0fd7d82577e40e37d4caf693f67744d0d61d11d66b5c31eaccf7a", 187 | "sha256:c4aaf320284a2713428163bae0e7df0db3b489237ab4830179210a12d56d3068", 188 | "sha256:c530274e43b0f376cd94e8e0a3e6ea28de1739ec4326689bdbf626e172d2e614", 189 | "sha256:ca4e79294fd0f3f9e0e5a4c309df84b5f2abc62349bfaf2aaf8965e5108ef8e2", 190 | "sha256:ce2dc5a104e885abbd48d0cc92ae74afa1d685ee65d6e3497067207d6a26e177", 191 | "sha256:d295cac30d3e13e82473081ea7df2a11352b5625cb54187fcd5a8be5d9ebf315", 192 | "sha256:d498338b39c4757ba88bdc705b3a0647d18554856cd2d394ac3bb919ac890c9d", 193 | "sha256:d537f8e613074805e17039e345edaa822534f66f07d315c89cff9824aa996d65", 194 | "sha256:df8ba3f52ef59a553b0e94593ea526c34faa4f531c1ab7f5ca7f392bc770c9e3", 195 | "sha256:e2553800d2d461a2dc329682d0a9068f238ec11d763e5454c61c4df7a0346ed2", 196 | "sha256:e2afbe403090f5893e254958d02875e0732975e73c4c0cdd33c1f009a61963ca", 197 | "sha256:e740efa625883f3c4de20c7e1411228d7ce2ab47b9e874a703f6681ec0558a30", 198 | "sha256:ec7864b62da0f5ae44973351247f2250a25b9b544fc6aff8bd6a75da1156cc70", 199 | "sha256:f26ddab491b10279b7e8e3fdcbaaaba3ab282fbaecfa48a19874dfc4d53b9d4f", 200 | "sha256:f6a16681f30918521066ddcc4ba79c1e033c9837dd94f78f5a9f6110e7572185", 201 | "sha256:f968623ac9b81a6253d4bbbe3f4d1e6be5f33707f397b566935783511bfa281a" 202 | ], 203 | "version": "==4.2.4" 204 | }, 205 | "markupsafe": { 206 | "hashes": [ 207 | "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665" 208 | ], 209 | "version": "==1.0" 210 | }, 211 | "parse": { 212 | "hashes": [ 213 | "sha256:c3cdf6206f22aeebfa00e5b954fcfea13d1b2dc271c75806b6025b94fb490939" 214 | ], 215 | "version": "==1.8.4" 216 | }, 217 | "peewee": { 218 | "hashes": [ 219 | "sha256:538d7b7f892e59ecbef6eeefb867fd37f20ac77e602b4ee94b2766836cfa24ba" 220 | ], 221 | "index": "pypi", 222 | "version": "==3.6.4" 223 | }, 224 | "pycparser": { 225 | "hashes": [ 226 | "sha256:99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226" 227 | ], 228 | "version": "==2.18" 229 | }, 230 | "pyee": { 231 | "hashes": [ 232 | "sha256:47f8fa96d6dee61c82001831e1fbba55f3f808003a322d0e6653aa01c59f6b9e", 233 | "sha256:4ec22817297b7024f89721cc34f790ee2767c5b5ca44284c565ee643abafbe32" 234 | ], 235 | "version": "==5.0.0" 236 | }, 237 | "pymysql": { 238 | "hashes": [ 239 | "sha256:95f057328357e0e13a30e67857a8c694878b0175797a9a203ee7adbfb9b1ec5f", 240 | "sha256:9ec760cbb251c158c19d6c88c17ca00a8632bac713890e465b2be01fdc30713f" 241 | ], 242 | "index": "pypi", 243 | "version": "==0.9.2" 244 | }, 245 | "pyppeteer": { 246 | "hashes": [ 247 | "sha256:3cdad9c51eb1c5b76cb2324a19bc9cdb011955834c752c520c56417f4c6aed0e" 248 | ], 249 | "version": "==0.0.21" 250 | }, 251 | "pyquery": { 252 | "hashes": [ 253 | "sha256:07987c2ed2aed5cba29ff18af95e56e9eb04a2249f42ce47bddfb37f487229a3", 254 | "sha256:4771db76bd14352eba006463656aef990a0147a0eeaf094725097acfa90442bf" 255 | ], 256 | "version": "==1.4.0" 257 | }, 258 | "requests": { 259 | "hashes": [ 260 | "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", 261 | "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" 262 | ], 263 | "version": "==2.19.1" 264 | }, 265 | "requests-html": { 266 | "hashes": [ 267 | "sha256:34257d5249b20b8ed14573eba910f48032a61205e70d11ce8a3ef6abf8edc50b", 268 | "sha256:9686f21c5753ba6c025c6ba223a8329c7b149a935a73055097faf8999eee85b1" 269 | ], 270 | "index": "pypi", 271 | "version": "==0.9.0" 272 | }, 273 | "six": { 274 | "hashes": [ 275 | "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", 276 | "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" 277 | ], 278 | "version": "==1.11.0" 279 | }, 280 | "tqdm": { 281 | "hashes": [ 282 | "sha256:5ef526702c0d265d5a960a3b27f3971fac13c26cf0fb819294bfa71fc6026c88", 283 | "sha256:a3364bd83ce4777320b862e3c8a93d7da91e20a95f06ef79bed7dd71c654cafa" 284 | ], 285 | "version": "==4.25.0" 286 | }, 287 | "urllib3": { 288 | "hashes": [ 289 | "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", 290 | "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" 291 | ], 292 | "version": "==1.23" 293 | }, 294 | "w3lib": { 295 | "hashes": [ 296 | "sha256:55994787e93b411c2d659068b51b9998d9d0c05e0df188e6daf8f45836e1ea38", 297 | "sha256:aaf7362464532b1036ab0092e2eee78e8fd7b56787baa9ed4967457b083d011b" 298 | ], 299 | "version": "==1.19.0" 300 | }, 301 | "websockets": { 302 | "hashes": [ 303 | "sha256:0e2f7d6567838369af074f0ef4d0b802d19fa1fee135d864acc656ceefa33136", 304 | "sha256:2a16dac282b2fdae75178d0ed3d5b9bc3258dabfae50196cbb30578d84b6f6a6", 305 | "sha256:5a1fa6072405648cb5b3688e9ed3b94be683ce4a4e5723e6f5d34859dee495c1", 306 | "sha256:5c1f55a1274df9d6a37553fef8cff2958515438c58920897675c9bc70f5a0538", 307 | "sha256:669d1e46f165e0ad152ed8197f7edead22854a6c90419f544e0f234cc9dac6c4", 308 | "sha256:695e34c4dbea18d09ab2c258994a8bf6a09564e762655408241f6a14592d2908", 309 | "sha256:6b2e03d69afa8d20253455e67b64de1a82ff8612db105113cccec35d3f8429f0", 310 | "sha256:79ca7cdda7ad4e3663ea3c43bfa8637fc5d5604c7737f19a8964781abbd1148d", 311 | "sha256:7fd2dd9a856f72e6ed06f82facfce01d119b88457cd4b47b7ae501e8e11eba9c", 312 | "sha256:82c0354ac39379d836719a77ee360ef865377aa6fdead87909d50248d0f05f4d", 313 | "sha256:8f3b956d11c5b301206382726210dc1d3bee1a9ccf7aadf895aaf31f71c3716c", 314 | "sha256:91ec98640220ae05b34b79ee88abf27f97ef7c61cf525eec57ea8fcea9f7dddb", 315 | "sha256:952be9540d83dba815569d5cb5f31708801e0bbfc3a8c5aef1890b57ed7e58bf", 316 | "sha256:99ac266af38ba1b1fe13975aea01ac0e14bb5f3a3200d2c69f05385768b8568e", 317 | "sha256:9fa122e7adb24232247f8a89f2d9070bf64b7869daf93ac5e19546b409e47e96", 318 | "sha256:a0873eadc4b8ca93e2e848d490809e0123eea154aa44ecd0109c4d0171869584", 319 | "sha256:cb998bd4d93af46b8b49ecf5a72c0a98e5cc6d57fdca6527ba78ad89d6606484", 320 | "sha256:e02e57346f6a68523e3c43bbdf35dde5c440318d1f827208ae455f6a2ace446d", 321 | "sha256:e79a5a896bcee7fff24a788d72e5c69f13e61369d055f28113e71945a7eb1559", 322 | "sha256:ee55eb6bcf23ecc975e6b47c127c201b913598f38b6a300075f84eeef2d3baff", 323 | "sha256:f1414e6cbcea8d22843e7eafdfdfae3dd1aba41d1945f6ca66e4806c07c4f454" 324 | ], 325 | "version": "==6.0" 326 | }, 327 | "werkzeug": { 328 | "hashes": [ 329 | "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", 330 | "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" 331 | ], 332 | "version": "==0.14.1" 333 | } 334 | }, 335 | "develop": {} 336 | } 337 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # INMET-API-REST 2 | API REST para pegar dados do Instituto Nacional de Meteorologia (INMET) 3 | -------------------------------------------------------------------------------- /env.ini.example: -------------------------------------------------------------------------------- 1 | [login] 2 | user=example@email.com 3 | password=123456 -------------------------------------------------------------------------------- /scrapper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Felipeandradee/INMET-API-REST/9a22fde88c73e3ce59fe9ea9c6334f02a622f768/scrapper/__init__.py -------------------------------------------------------------------------------- /scrapper/scrapper.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from requests_html import HTMLSession 4 | from requests import Response 5 | import re 6 | import os 7 | import configparser 8 | 9 | 10 | class ScrapperINMET: 11 | 12 | def __init__(self): 13 | 14 | env_path = os.path.join(os.path.dirname(__file__), '..', 'env.ini') 15 | env_config = configparser.ConfigParser() 16 | env_config.read(env_path) 17 | self.user = env_config.get('login', 'user') 18 | self.password = env_config.get('login', 'password') 19 | 20 | self.base_url = "http://www.inmet.gov.br" 21 | self.session = HTMLSession(mock_browser=True) 22 | 23 | @staticmethod 24 | def is_logged_in(response: Response) -> bool: 25 | return 'Modulo de Estudo e Pesquisa' in response.text 26 | 27 | def login(self) -> bool: 28 | url = '/'.join([self.base_url, "projetos/rede/pesquisa/inicio.php"]) 29 | self.session.get(url) 30 | 31 | payload = { 32 | 'mUsuario': '', 33 | 'mSenha': self.password, 34 | 'mCod': self.user, 35 | 'mGerModulo': 'PES', 36 | 'btnProcesso': ' Acessar ' 37 | } 38 | 39 | response = self.session.post(url, data=payload) 40 | response.raise_for_status() 41 | return self.is_logged_in(response) 42 | 43 | def get_dados(self, tipo_periodo: str, uf: str, cidade: str, dtInicio: datetime.datetime, dtFim: datetime.datetime): 44 | 45 | self.login() 46 | 47 | if tipo_periodo == 'mes': 48 | url = '/'.join([self.base_url, "projetos/rede/pesquisa/form_mapas_mensal.php"]) 49 | url_post = '/'.join([self.base_url, "projetos/rede/pesquisa/mapas_mensal_sem.php"]) 50 | 51 | elif tipo_periodo == 'dia': 52 | url = '/'.join([self.base_url, "projetos/rede/pesquisa/form_mapas_c_diario.php"]) 53 | url_post = '/'.join([self.base_url, "projetos/rede/pesquisa/mapas_c_diario.php"]) 54 | else: 55 | raise NotImplementedError 56 | 57 | self.session.get(url) 58 | 59 | payload = { 60 | 'mUsuario': self.user, 61 | 'mRelRegiao': '', 62 | 'mRelEstado': uf.upper(), 63 | 'mRelDtInicio': dtInicio.strftime('%d/%m/%Y'), 64 | 'mRelDtFim': dtFim.strftime('%d/%m/%Y'), 65 | 'mGerModulo': 'PES', 66 | 'mOpcaoAtrib15': '1', 67 | 'btnProcesso': ' Pesquisa ' 68 | } 69 | 70 | response = self.session.post(url_post, data=payload) 71 | pattern = cidade + '.*href=([^\>]+) target' 72 | url_result = re.search(pattern, response.text, flags=re.IGNORECASE).group(1) 73 | response = self.session.get(url_result) 74 | rtext = response.html.xpath('.//pre')[0].full_text 75 | dados = re.search('(estacao;.+)', rtext, re.IGNORECASE | re.DOTALL).group(1) 76 | 77 | dados = dados.split('\n') 78 | head = dados[0].split(';') 79 | dados.pop(0) 80 | result = list() 81 | for dado in dados: 82 | values = dado.split(';') 83 | if len(values) != len(head): 84 | continue 85 | temp = dict() 86 | for k, v in zip(head, values): 87 | if k + v == "": 88 | continue 89 | temp[k] = v 90 | result.append(temp) 91 | return result 92 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Felipeandradee/INMET-API-REST/9a22fde88c73e3ce59fe9ea9c6334f02a622f768/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_login.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from scrapper.scrapper import ScrapperINMET 3 | import requests 4 | 5 | 6 | class TestLogin(unittest.TestCase): 7 | 8 | def test_login_true(self): 9 | self.assertTrue(ScrapperINMET().login()) 10 | 11 | def test_login_false(self): 12 | response = requests.get('http://www.inmet.gov.br/projetos/rede/pesquisa/inicio.php') 13 | self.assertFalse(ScrapperINMET().is_logged_in(response)) -------------------------------------------------------------------------------- /tests/test_pesquisa.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import unittest 3 | from scrapper.scrapper import ScrapperINMET 4 | 5 | 6 | class TestPesquisa(unittest.TestCase): 7 | 8 | def test_pesquisa_mes(self): 9 | scrap = ScrapperINMET() 10 | result = scrap.get_dados('mes', 'ba', 'salvador', datetime(2016, 1, 1), datetime(2017, 1, 1)) 11 | self.assertEqual(len(result), 12) 12 | 13 | def test_pesquisa_dia(self): 14 | scrap = ScrapperINMET() 15 | result = scrap.get_dados('dia', 'ba', 'salvador', datetime(2016, 1, 1), datetime(2017, 1, 1)) 16 | self.assertEqual(len(result), 366) 17 | --------------------------------------------------------------------------------