├── .gitignore ├── .vscode └── settings.json ├── Pipfile ├── Pipfile.lock ├── Procfile ├── README.md ├── app ├── __init__.py ├── app.py ├── facebook.py ├── scraper │ ├── __init__.py │ ├── __main__.py │ ├── pipeline.py │ ├── spider.py │ └── utils.py ├── static │ ├── css │ │ └── app.css │ ├── favicon.ico │ ├── github-logo.png │ ├── js │ │ ├── app.js │ │ └── embed.js │ ├── share-template.png │ └── share.png ├── stats.py ├── templates │ ├── common │ │ ├── analytics.html │ │ ├── footer.html │ │ └── intro.html │ ├── details.html │ ├── digest.html │ ├── embed.html │ ├── index.html │ └── static │ │ ├── facebook.svg │ │ └── twitter.svg └── utils.py ├── fonts ├── SourceCodePro-Bold.otf └── SourceCodePro-Regular.otf └── importer.py /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/osx,python 3 | 4 | ### OSX ### 5 | *.DS_Store 6 | .AppleDouble 7 | .LSOverride 8 | 9 | # Icon must end with two \r 10 | Icon 11 | 12 | # Thumbnails 13 | ._* 14 | 15 | # Files that might appear in the root of a volume 16 | .DocumentRevisions-V100 17 | .fseventsd 18 | .Spotlight-V100 19 | .TemporaryItems 20 | .Trashes 21 | .VolumeIcon.icns 22 | .com.apple.timemachine.donotpresent 23 | 24 | # Directories potentially created on remote AFP share 25 | .AppleDB 26 | .AppleDesktop 27 | Network Trash Folder 28 | Temporary Items 29 | .apdisk 30 | 31 | ### Python ### 32 | # Byte-compiled / optimized / DLL files 33 | __pycache__/ 34 | *.py[cod] 35 | *$py.class 36 | 37 | # C extensions 38 | *.so 39 | 40 | # Distribution / packaging 41 | .Python 42 | build/ 43 | develop-eggs/ 44 | dist/ 45 | downloads/ 46 | eggs/ 47 | .eggs/ 48 | lib/ 49 | lib64/ 50 | parts/ 51 | sdist/ 52 | var/ 53 | wheels/ 54 | *.egg-info/ 55 | .installed.cfg 56 | *.egg 57 | 58 | # PyInstaller 59 | # Usually these files are written by a python script from a template 60 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 61 | *.manifest 62 | *.spec 63 | 64 | # Installer logs 65 | pip-log.txt 66 | pip-delete-this-directory.txt 67 | 68 | # Unit test / coverage reports 69 | htmlcov/ 70 | .tox/ 71 | .coverage 72 | .coverage.* 73 | .cache 74 | nosetests.xml 75 | coverage.xml 76 | *.cover 77 | .hypothesis/ 78 | 79 | # Translations 80 | *.mo 81 | *.pot 82 | 83 | # Django stuff: 84 | *.log 85 | local_settings.py 86 | 87 | # Flask stuff: 88 | instance/ 89 | .webassets-cache 90 | 91 | # Scrapy stuff: 92 | .scrapy 93 | 94 | # Sphinx documentation 95 | docs/_build/ 96 | 97 | # PyBuilder 98 | target/ 99 | 100 | # Jupyter Notebook 101 | .ipynb_checkpoints 102 | 103 | # pyenv 104 | .python-version 105 | 106 | # celery beat schedule file 107 | celerybeat-schedule 108 | 109 | # SageMath parsed files 110 | *.sage.py 111 | 112 | # Environments 113 | .env 114 | .venv 115 | env/ 116 | venv/ 117 | ENV/ 118 | env.bak/ 119 | venv.bak/ 120 | 121 | # Spyder project settings 122 | .spyderproject 123 | .spyproject 124 | 125 | # Rope project settings 126 | .ropeproject 127 | 128 | # mkdocs documentation 129 | /site 130 | 131 | # mypy 132 | .mypy_cache/ 133 | 134 | # End of https://www.gitignore.io/api/osx,python 135 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.formatting.provider": "yapf" 3 | } -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [requires] 2 | python_version = "3.6" 3 | 4 | [[source]] 5 | url = "https://pypi.python.org/simple" 6 | verify_ssl = true 7 | name = "pypi" 8 | 9 | [dev-packages] 10 | 11 | [packages] 12 | Flask = "==1.0.2" 13 | gunicorn = "==19.9.0" 14 | Pillow = "==5.2.0" 15 | pymongo = "==3.7.1" 16 | pytz = "==2018.5" 17 | requests = "==2.19.1" 18 | Scrapy = "==1.5.1" 19 | pylibmc = "==1.5.2" 20 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "1b1f064676f6f0b6f28ce51c38b1219445c5ac6aa864bbb9ca7553cd4893550e" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.6" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.python.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "asn1crypto": { 20 | "hashes": [ 21 | "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87", 22 | "sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49" 23 | ], 24 | "version": "==0.24.0" 25 | }, 26 | "attrs": { 27 | "hashes": [ 28 | "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", 29 | "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" 30 | ], 31 | "version": "==18.2.0" 32 | }, 33 | "automat": { 34 | "hashes": [ 35 | "sha256:cbd78b83fa2d81fe2a4d23d258e1661dd7493c9a50ee2f1a5b2cac61c1793b0e", 36 | "sha256:fdccab66b68498af9ecfa1fa43693abe546014dd25cf28543cbe9d1334916a58" 37 | ], 38 | "version": "==0.7.0" 39 | }, 40 | "certifi": { 41 | "hashes": [ 42 | "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", 43 | "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" 44 | ], 45 | "version": "==2018.8.24" 46 | }, 47 | "cffi": { 48 | "hashes": [ 49 | "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", 50 | "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", 51 | "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", 52 | "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", 53 | "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", 54 | "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", 55 | "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", 56 | "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", 57 | "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", 58 | "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", 59 | "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", 60 | "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", 61 | "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", 62 | "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", 63 | "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", 64 | "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", 65 | "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", 66 | "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", 67 | "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", 68 | "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", 69 | "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", 70 | "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", 71 | "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", 72 | "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", 73 | "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", 74 | "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", 75 | "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", 76 | "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", 77 | "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", 78 | "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", 79 | "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", 80 | "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" 81 | ], 82 | "version": "==1.11.5" 83 | }, 84 | "chardet": { 85 | "hashes": [ 86 | "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", 87 | "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" 88 | ], 89 | "version": "==3.0.4" 90 | }, 91 | "click": { 92 | "hashes": [ 93 | "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", 94 | "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" 95 | ], 96 | "version": "==6.7" 97 | }, 98 | "constantly": { 99 | "hashes": [ 100 | "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35", 101 | "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d" 102 | ], 103 | "version": "==15.1.0" 104 | }, 105 | "cryptography": { 106 | "hashes": [ 107 | "sha256:02602e1672b62e803e08617ec286041cc453e8d43f093a5f4162095506bc0beb", 108 | "sha256:10b48e848e1edb93c1d3b797c83c72b4c387ab0eb4330aaa26da8049a6cbede0", 109 | "sha256:17db09db9d7c5de130023657be42689d1a5f60502a14f6f745f6f65a6b8195c0", 110 | "sha256:227da3a896df1106b1a69b1e319dce218fa04395e8cc78be7e31ca94c21254bc", 111 | "sha256:2cbaa03ac677db6c821dac3f4cdfd1461a32d0615847eedbb0df54bb7802e1f7", 112 | "sha256:31db8febfc768e4b4bd826750a70c79c99ea423f4697d1dab764eb9f9f849519", 113 | "sha256:4a510d268e55e2e067715d728e4ca6cd26a8e9f1f3d174faf88e6f2cb6b6c395", 114 | "sha256:6a88d9004310a198c474d8a822ee96a6dd6c01efe66facdf17cb692512ae5bc0", 115 | "sha256:76936ec70a9b72eb8c58314c38c55a0336a2b36de0c7ee8fb874a4547cadbd39", 116 | "sha256:7e3b4aecc4040928efa8a7cdaf074e868af32c58ffc9bb77e7bf2c1a16783286", 117 | "sha256:8168bcb08403ef144ff1fb880d416f49e2728101d02aaadfe9645883222c0aa5", 118 | "sha256:8229ceb79a1792823d87779959184a1bf95768e9248c93ae9f97c7a2f60376a1", 119 | "sha256:8a19e9f2fe69f6a44a5c156968d9fc8df56d09798d0c6a34ccc373bb186cee86", 120 | "sha256:8d10113ca826a4c29d5b85b2c4e045ffa8bad74fb525ee0eceb1d38d4c70dfd6", 121 | "sha256:be495b8ec5a939a7605274b6e59fbc35e76f5ad814ae010eb679529671c9e119", 122 | "sha256:dc2d3f3b1548f4d11786616cf0f4415e25b0fbecb8a1d2cd8c07568f13fdde38", 123 | "sha256:e4aecdd9d5a3d06c337894c9a6e2961898d3f64fe54ca920a72234a3de0f9cb3", 124 | "sha256:e79ab4485b99eacb2166f3212218dd858258f374855e1568f728462b0e6ee0d9", 125 | "sha256:f995d3667301e1754c57b04e0bae6f0fa9d710697a9f8d6712e8cca02550910f" 126 | ], 127 | "version": "==2.3.1" 128 | }, 129 | "cssselect": { 130 | "hashes": [ 131 | "sha256:066d8bc5229af09617e24b3ca4d52f1f9092d9e061931f4184cd572885c23204", 132 | "sha256:3b5103e8789da9e936a68d993b70df732d06b8bb9a337a05ed4eb52c17ef7206" 133 | ], 134 | "markers": "python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.1.*'", 135 | "version": "==1.0.3" 136 | }, 137 | "flask": { 138 | "hashes": [ 139 | "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", 140 | "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05" 141 | ], 142 | "index": "pypi", 143 | "version": "==1.0.2" 144 | }, 145 | "gunicorn": { 146 | "hashes": [ 147 | "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471", 148 | "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3" 149 | ], 150 | "index": "pypi", 151 | "version": "==19.9.0" 152 | }, 153 | "hyperlink": { 154 | "hashes": [ 155 | "sha256:98da4218a56b448c7ec7d2655cb339af1f7d751cf541469bb4fc28c4a4245b34", 156 | "sha256:f01b4ff744f14bc5d0a22a6b9f1525ab7d6312cb0ff967f59414bbac52f0a306" 157 | ], 158 | "version": "==18.0.0" 159 | }, 160 | "idna": { 161 | "hashes": [ 162 | "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", 163 | "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" 164 | ], 165 | "version": "==2.7" 166 | }, 167 | "incremental": { 168 | "hashes": [ 169 | "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f", 170 | "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3" 171 | ], 172 | "version": "==17.5.0" 173 | }, 174 | "itsdangerous": { 175 | "hashes": [ 176 | "sha256:cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519" 177 | ], 178 | "version": "==0.24" 179 | }, 180 | "jinja2": { 181 | "hashes": [ 182 | "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", 183 | "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" 184 | ], 185 | "version": "==2.10" 186 | }, 187 | "lxml": { 188 | "hashes": [ 189 | "sha256:02bc220d61f46e9b9d5a53c361ef95e9f5e1d27171cd461dddb17677ae2289a5", 190 | "sha256:22f253b542a342755f6cfc047fe4d3a296515cf9b542bc6e261af45a80b8caf6", 191 | "sha256:2f31145c7ff665b330919bfa44aacd3a0211a76ca7e7b441039d2a0b0451e415", 192 | "sha256:36720698c29e7a9626a0dc802ef8885f8f0239bfd1689628ecd459a061f2807f", 193 | "sha256:438a1b0203545521f6616132bfe0f4bca86f8a401364008b30e2b26ec408ce85", 194 | "sha256:4815892904c336bbaf73dafd54f45f69f4021c22b5bad7332176bbf4fb830568", 195 | "sha256:5be031b0f15ad63910d8e5038b489d95a79929513b3634ad4babf77100602588", 196 | "sha256:5c93ae37c3c588e829b037fdfbd64a6e40c901d3f93f7beed6d724c44829a3ad", 197 | "sha256:60842230678674cdac4a1cf0f707ef12d75b9a4fc4a565add4f710b5fcf185d5", 198 | "sha256:62939a8bb6758d1bf923aa1c13f0bcfa9bf5b2fc0f5fa917a6e25db5fe0cfa4e", 199 | "sha256:75830c06a62fe7b8fe3bbb5f269f0b308f19f3949ac81cfd40062f47c1455faf", 200 | "sha256:81992565b74332c7c1aff6a913a3e906771aa81c9d0c68c68113cffcae45bc53", 201 | "sha256:8c892fb0ee52c594d9a7751c7d7356056a9682674b92cc1c4dc968ff0f30c52f", 202 | "sha256:9d862e3cf4fc1f2837dedce9c42269c8c76d027e49820a548ac89fdcee1e361f", 203 | "sha256:a623965c086a6e91bb703d4da62dabe59fe88888e82c4117d544e11fd74835d6", 204 | "sha256:a7783ab7f6a508b0510490cef9f857b763d796ba7476d9703f89722928d1e113", 205 | "sha256:aab09fbe8abfa3b9ce62aaf45aca2d28726b1b9ee44871dbe644050a2fff4940", 206 | "sha256:abf181934ac3ef193832fb973fd7f6149b5c531903c2ec0f1220941d73eee601", 207 | "sha256:ae07fa0c115733fce1e9da96a3ac3fa24801742ca17e917e0c79d63a01eeb843", 208 | "sha256:b9c78242219f674ab645ec571c9a95d70f381319a23911941cd2358a8e0521cf", 209 | "sha256:bccb267678b870d9782c3b44d0cefe3ba0e329f9af8c946d32bf3778e7a4f271", 210 | "sha256:c4df4d27f4c93b2cef74579f00b1d3a31a929c7d8023f870c4b476f03a274db4", 211 | "sha256:caf0e50b546bb60dfa99bb18dfa6748458a83131ecdceaf5c071d74907e7e78a", 212 | "sha256:d3266bd3ac59ac4edcd5fa75165dee80b94a3e5c91049df5f7c057ccf097551c", 213 | "sha256:db0d213987bcd4e6d41710fb4532b22315b0d8fb439ff901782234456556aed1", 214 | "sha256:dbbd5cf7690a40a9f0a9325ab480d0fccf46d16b378eefc08e195d84299bfae1", 215 | "sha256:e16e07a0ec3a75b5ee61f2b1003c35696738f937dc8148fbda9fe2147ccb6e61", 216 | "sha256:e175a006725c7faadbe69e791877d09936c0ef2cf49d01b60a6c1efcb0e8be6f", 217 | "sha256:edd9c13a97f6550f9da2236126bb51c092b3b1ce6187f2bd966533ad794bbb5e", 218 | "sha256:fa39ea60d527fbdd94215b5e5552f1c6a912624521093f1384a491a8ad89ad8b" 219 | ], 220 | "version": "==4.2.5" 221 | }, 222 | "markupsafe": { 223 | "hashes": [ 224 | "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665" 225 | ], 226 | "version": "==1.0" 227 | }, 228 | "parsel": { 229 | "hashes": [ 230 | "sha256:a4d581260eb845a762b9a354b0fc5e1c5c42df009dc8163c181097bd5314db58", 231 | "sha256:b24618fe81dce29d717aa8c4a9534c46e807dd6a5c8d5e1bb3b1fdb3fbd22b56" 232 | ], 233 | "version": "==1.5.0" 234 | }, 235 | "pillow": { 236 | "hashes": [ 237 | "sha256:00def5b638994f888d1058e4d17c86dec8e1113c3741a0a8a659039aec59a83a", 238 | "sha256:026449b64e559226cdb8e6d8c931b5965d8fc90ec18ebbb0baa04c5b36503c72", 239 | "sha256:03dbb224ee196ef30ed2156d41b579143e1efeb422974719a5392fc035e4f574", 240 | "sha256:03eb0e04f929c102ae24bc436bf1c0c60a4e63b07ebd388e84d8b219df3e6acd", 241 | "sha256:1be66b9a89e367e7d20d6cae419794997921fe105090fafd86ef39e20a3baab2", 242 | "sha256:1e977a3ed998a599bda5021fb2c2889060617627d3ae228297a529a082a3cd5c", 243 | "sha256:22cf3406d135cfcc13ec6228ade774c8461e125c940e80455f500638429be273", 244 | "sha256:24adccf1e834f82718c7fc8e3ec1093738da95144b8b1e44c99d5fc7d3e9c554", 245 | "sha256:2a3e362c97a5e6a259ee9cd66553292a1f8928a5bdfa3622fdb1501570834612", 246 | "sha256:3832e26ecbc9d8a500821e3a1d3765bda99d04ae29ffbb2efba49f5f788dc934", 247 | "sha256:4fd1f0c2dc02aaec729d91c92cd85a2df0289d88e9f68d1e8faba750bb9c4786", 248 | "sha256:4fda62030f2c515b6e2e673c57caa55cb04026a81968f3128aae10fc28e5cc27", 249 | "sha256:5044d75a68b49ce36a813c82d8201384207112d5d81643937fc758c05302f05b", 250 | "sha256:522184556921512ec484cb93bd84e0bab915d0ac5a372d49571c241a7f73db62", 251 | "sha256:5914cff11f3e920626da48e564be6818831713a3087586302444b9c70e8552d9", 252 | "sha256:6661a7908d68c4a133e03dac8178287aa20a99f841ea90beeb98a233ae3fd710", 253 | "sha256:79258a8df3e309a54c7ef2ef4a59bb8e28f7e4a8992a3ad17c24b1889ced44f3", 254 | "sha256:7d74c20b8f1c3e99d3f781d3b8ff5abfefdd7363d61e23bdeba9992ff32cc4b4", 255 | "sha256:81918afeafc16ba5d9d0d4e9445905f21aac969a4ebb6f2bff4b9886da100f4b", 256 | "sha256:8194d913ca1f459377c8a4ed8f9b7ad750068b8e0e3f3f9c6963fcc87a84515f", 257 | "sha256:84d5d31200b11b3c76fab853b89ac898bf2d05c8b3da07c1fcc23feb06359d6e", 258 | "sha256:989981db57abffb52026b114c9a1f114c7142860a6d30a352d28f8cbf186500b", 259 | "sha256:a3d7511d3fad1618a82299aab71a5fceee5c015653a77ffea75ced9ef917e71a", 260 | "sha256:b3ef168d4d6fd4fa6685aef7c91400f59f7ab1c0da734541f7031699741fb23f", 261 | "sha256:c1c5792b6e74bbf2af0f8e892272c2a6c48efa895903211f11b8342e03129fea", 262 | "sha256:c5dcb5a56aebb8a8f2585042b2f5c496d7624f0bcfe248f0cc33ceb2fd8d39e7", 263 | "sha256:e2bed4a04e2ca1050bb5f00865cf2f83c0b92fd62454d9244f690fcd842e27a4", 264 | "sha256:e87a527c06319428007e8c30511e1f0ce035cb7f14bb4793b003ed532c3b9333", 265 | "sha256:f63e420180cbe22ff6e32558b612e75f50616fc111c5e095a4631946c782e109", 266 | "sha256:f8b3d413c5a8f84b12cd4c5df1d8e211777c9852c6be3ee9c094b626644d3eab" 267 | ], 268 | "index": "pypi", 269 | "markers": "python_version != '3.2.*' and python_version >= '2.7' and python_version != '3.3.*' and python_version != '3.0.*' and python_version != '3.1.*'", 270 | "version": "==5.2.0" 271 | }, 272 | "pyasn1": { 273 | "hashes": [ 274 | "sha256:b9d3abc5031e61927c82d4d96c1cec1e55676c1a991623cfed28faea73cdd7ca", 275 | "sha256:f58f2a3d12fd754aa123e9fa74fb7345333000a035f3921dbdaa08597aa53137" 276 | ], 277 | "version": "==0.4.4" 278 | }, 279 | "pyasn1-modules": { 280 | "hashes": [ 281 | "sha256:a0cf3e1842e7c60fde97cb22d275eb6f9524f5c5250489e292529de841417547", 282 | "sha256:a38a8811ea784c0136abfdba73963876328f66172db21a05a82f9515909bfb4e" 283 | ], 284 | "version": "==0.2.2" 285 | }, 286 | "pycparser": { 287 | "hashes": [ 288 | "sha256:99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226" 289 | ], 290 | "version": "==2.18" 291 | }, 292 | "pydispatcher": { 293 | "hashes": [ 294 | "sha256:5570069e1b1769af1fe481de6dd1d3a388492acddd2cdad7a3bde145615d5caf", 295 | "sha256:5be4a8be12805ef7d712dd9a93284fb8bc53f309867e573f653a72e5fd10e433" 296 | ], 297 | "version": "==2.0.5" 298 | }, 299 | "pyhamcrest": { 300 | "hashes": [ 301 | "sha256:6b672c02fdf7470df9674ab82263841ce8333fb143f32f021f6cb26f0e512420", 302 | "sha256:8ffaa0a53da57e89de14ced7185ac746227a8894dbd5a3c718bf05ddbd1d56cd" 303 | ], 304 | "markers": "python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.1.*'", 305 | "version": "==1.9.0" 306 | }, 307 | "pylibmc": { 308 | "hashes": [ 309 | "sha256:fc54e28a9f1b5b2ec0c030da29c7ad8a15c2755bd98aaa4142eaf419d5fabb33" 310 | ], 311 | "index": "pypi", 312 | "version": "==1.5.2" 313 | }, 314 | "pymongo": { 315 | "hashes": [ 316 | "sha256:08dea6dbff33363419af7af3bf2e9a373ff71eb22833dd7063f9b953f09a0bdf", 317 | "sha256:0949110db76eb1b54cecfc0c0f8468a8b9a7fd42ba23fd0d4a37d97e0b4ca203", 318 | "sha256:0c31a39f440801cc8603547ccaacf4cb1f02b81af6ba656621c13677b27f4426", 319 | "sha256:1e10b3fda5677d360440ebd12a1185944dc81d9ea9acf0c6b0681013b3fb9bc2", 320 | "sha256:1f59440b993666a417ba1954cfb1b7fb11cb4dea1a1d2777897009f688d000ee", 321 | "sha256:2b5a3806d9f656c14e9d9b693a344fc5684fdd045155594be0c505c6e9410a94", 322 | "sha256:4a14e2d7c2c0e07b5affcfbfc5c395d767f94bb1a822934a41a3b5371cde1458", 323 | "sha256:4cb50541225208b37786fdb0de632e475c4f00ec4792579df551ef48d6999d69", 324 | "sha256:52999666ad01de885653e1f74a86c2a6520d1004afec475180bebf3d7393a8fc", 325 | "sha256:562c353079e8ce7e2ad611fd7436a72f5df97be72bca59ae9ebf789a724afd5c", 326 | "sha256:5ce2a71f473f4703daa8d6c61a00b35ce625a7f5015b4371e3af728dafca296a", 327 | "sha256:6613e633676168a4500e5e6bb6e3e64d3fdb96d2dc472eb4b99235fb4141adb1", 328 | "sha256:8330406f294df118399c721f80979f2516447bcc73e4262826687872c864751e", 329 | "sha256:8e939dfa7d16609b99eb4d1fd2fc74f7a90f4fd0aaf31d611822daaff456236f", 330 | "sha256:8fa4303e1f50d9f0c8f2f7833b5a370a94d19d41449def62b34ae072126b4dfd", 331 | "sha256:966d987975aa3b4cfcdf1495930ff6ecb152fafe8e544e40633e41b24ca3e1c5", 332 | "sha256:aec4ea43a1b8e9782246a259410f66692f2d3aa0f03c54477e506193b0781cb6", 333 | "sha256:b73f889f032fbef05863f5056b46468a8262ae83628898e20b10bbbb79a3617e", 334 | "sha256:b752088a2f819f163d11dfdbbe627b27eef9d8478c7e57d42c5e7c600fee434e", 335 | "sha256:c8669f96277f140797e0ff99f80bd706271674942672a38ed694e2bfa66f3900", 336 | "sha256:ccf00549efaf6f8d5b35b654beb9aed2b788a5b33b05606eb818ddaa4e924ea3", 337 | "sha256:ce7c91463ad21ac72fc795188292b01c8366cf625e2d1e5ed473ce127b844f60", 338 | "sha256:d776d8d47884e6ad39ff8a301f1ae6b7d2186f209218cf024f43334dbba79c64", 339 | "sha256:dab0f63841aebb2b421fadb31f3c7eef27898f21274a8e5b45c4f2bccb40f9ed", 340 | "sha256:daedcfbf3b24b2b687e35b33252a9315425c2dd06a085a36906d516135bdd60e", 341 | "sha256:e7ad1ec621db2c5ad47924f63561f75abfd4fff669c62c8cc99c169c90432f59", 342 | "sha256:f14fb6c4058772a0d74d82874d3b89d7264d89b4ed7fa0413ea0ef8112b268b9", 343 | "sha256:f16c7b6b98bc400d180f05e65e2236ef4ee9d71f3815280558582670e1e67536", 344 | "sha256:f2d9eb92b26600ae6e8092f66da4bcede1b61a647c9080d6b44c148aff3a8ea4", 345 | "sha256:ffe94f9d17800610dda5282d7f6facfc216d79a93dd728a03d2f21cff3af7cc6" 346 | ], 347 | "index": "pypi", 348 | "version": "==3.7.1" 349 | }, 350 | "pyopenssl": { 351 | "hashes": [ 352 | "sha256:26ff56a6b5ecaf3a2a59f132681e2a80afcc76b4f902f612f518f92c2a1bf854", 353 | "sha256:6488f1423b00f73b7ad5167885312bb0ce410d3312eb212393795b53c8caa580" 354 | ], 355 | "version": "==18.0.0" 356 | }, 357 | "pytz": { 358 | "hashes": [ 359 | "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", 360 | "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277" 361 | ], 362 | "index": "pypi", 363 | "version": "==2018.5" 364 | }, 365 | "queuelib": { 366 | "hashes": [ 367 | "sha256:42b413295551bdc24ed9376c1a2cd7d0b1b0fa4746b77b27ca2b797a276a1a17", 368 | "sha256:ff43b5b74b9266f8df4232a8f768dc4d67281a271905e2ed4a3689d4d304cd02" 369 | ], 370 | "version": "==1.5.0" 371 | }, 372 | "requests": { 373 | "hashes": [ 374 | "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", 375 | "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" 376 | ], 377 | "index": "pypi", 378 | "version": "==2.19.1" 379 | }, 380 | "scrapy": { 381 | "hashes": [ 382 | "sha256:5a398bf6818f87dcc817c919408a195f19ba46414ae12f259119336cfa862bb6", 383 | "sha256:5b9621731e26b0d195ca3e25ab34d559f45b0b906c0a0cc359199f1b6b612184" 384 | ], 385 | "index": "pypi", 386 | "version": "==1.5.1" 387 | }, 388 | "service-identity": { 389 | "hashes": [ 390 | "sha256:0e76f3c042cc0f5c7e6da002cf646f59dc4023962d1d1166343ce53bdad39e17", 391 | "sha256:4001fbb3da19e0df22c47a06d29681a398473af4aa9d745eca525b3b2c2302ab" 392 | ], 393 | "version": "==17.0.0" 394 | }, 395 | "six": { 396 | "hashes": [ 397 | "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", 398 | "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" 399 | ], 400 | "version": "==1.11.0" 401 | }, 402 | "twisted": { 403 | "hashes": [ 404 | "sha256:5de7b79b26aee64efe63319bb8f037af88c21287d1502b39706c818065b3d5a4", 405 | "sha256:95ae985716e8107816d8d0df249d558dbaabb677987cc2ace45272c166b267e4" 406 | ], 407 | "markers": "python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.1.*'", 408 | "version": "==18.7.0" 409 | }, 410 | "urllib3": { 411 | "hashes": [ 412 | "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", 413 | "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" 414 | ], 415 | "markers": "python_version != '3.2.*' and python_version < '4' and python_version != '3.3.*' and python_version >= '2.6' and python_version != '3.0.*' and python_version != '3.1.*'", 416 | "version": "==1.23" 417 | }, 418 | "w3lib": { 419 | "hashes": [ 420 | "sha256:55994787e93b411c2d659068b51b9998d9d0c05e0df188e6daf8f45836e1ea38", 421 | "sha256:aaf7362464532b1036ab0092e2eee78e8fd7b56787baa9ed4967457b083d011b" 422 | ], 423 | "version": "==1.19.0" 424 | }, 425 | "werkzeug": { 426 | "hashes": [ 427 | "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", 428 | "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" 429 | ], 430 | "version": "==0.14.1" 431 | }, 432 | "zope.interface": { 433 | "hashes": [ 434 | "sha256:21506674d30c009271fe68a242d330c83b1b9d76d62d03d87e1e9528c61beea6", 435 | "sha256:3d184aff0756c44fff7de69eb4cd5b5311b6f452d4de28cb08343b3f21993763", 436 | "sha256:467d364b24cb398f76ad5e90398d71b9325eb4232be9e8a50d6a3b3c7a1c8789", 437 | "sha256:57c38470d9f57e37afb460c399eb254e7193ac7fb8042bd09bdc001981a9c74c", 438 | "sha256:9ada83f4384bbb12dedc152bcdd46a3ac9f5f7720d43ac3ce3e8e8b91d733c10", 439 | "sha256:a1daf9c5120f3cc6f2b5fef8e1d2a3fb7bbbb20ed4bfdc25bc8364bc62dcf54b", 440 | "sha256:e6b77ae84f2b8502d99a7855fa33334a1eb6159de45626905cb3e454c023f339", 441 | "sha256:e881ef610ff48aece2f4ee2af03d2db1a146dc7c705561bd6089b2356f61641f", 442 | "sha256:f41037260deaacb875db250021fe883bf536bf6414a4fd25b25059b02e31b120" 443 | ], 444 | "markers": "python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.1.*'", 445 | "version": "==4.5.0" 446 | } 447 | }, 448 | "develop": {} 449 | } 450 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: gunicorn -w 4 app.app:app 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tracking status.pr 2 | 3 | This tool scrapes status.pr every hour and keeps tracks of changing metrics in order to help visualize and measure progress. Data is also made available in CSV and JSON formats. 4 | 5 | ## Running 6 | 7 | ### Web 8 | 9 | ``` 10 | $ MONGODB_URI='mongodb://localhost/tracking-status-pr' FLASK_DEBUG=1 FLASK_APP=app/app.py flask run 11 | ``` 12 | 13 | ### Scraper 14 | 15 | ``` 16 | $ MONGODB_URI='mongodb://localhost/tracking-status-pr' python scraper.py 17 | ``` 18 | -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpadilla/tracking-status-pr/5c40a1f1578a2f69dd5e99c9e8b5b5bb0a4ebb60/app/__init__.py -------------------------------------------------------------------------------- /app/app.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import datetime 4 | from io import BytesIO 5 | 6 | import pymongo 7 | from pytz import timezone 8 | from PIL import Image, ImageDraw, ImageFont 9 | from flask import (Flask, Response, request, redirect, jsonify, 10 | render_template, send_file, send_from_directory) 11 | 12 | from .stats import STATS 13 | from .utils import JSONEncoder, cached 14 | 15 | pr = timezone('America/Puerto_Rico') 16 | utc = timezone('UTC') 17 | 18 | app = Flask(__name__) 19 | app.json_encoder = JSONEncoder 20 | app.config['SERVER_NAME'] = os.getenv('SERVER_NAME') 21 | app.config['PREFERRED_URL_SCHEME'] = 'http' if app.debug else 'https' 22 | app.config['now'] = int(round(time.time() * 1000)) 23 | app.config['version'] = os.getenv('SOURCE_VERSION', app.config['now']) 24 | 25 | client = pymongo.MongoClient(os.getenv('MONGODB_URI')) 26 | db = client.get_default_database() 27 | 28 | 29 | @app.before_request 30 | def before_request(): 31 | if not app.debug and not request.is_secure: 32 | url = request.url.replace('http://', 'https://', 1) 33 | return redirect(url) 34 | 35 | 36 | def get_stats(path=None): 37 | pipeline = [{'$sort': {'created_at': -1}}] 38 | 39 | if path: 40 | pipeline.append({'$match': {'path': path}}) 41 | 42 | pipeline.append({ 43 | '$group': { 44 | '_id': '$path', 45 | 'label': { 46 | '$last': '$label' 47 | }, 48 | 'data': { 49 | '$push': { 50 | 'value': '$value', 51 | 'date': '$created_at' 52 | } 53 | } 54 | } 55 | }) 56 | 57 | return db.stats.aggregate(pipeline) 58 | 59 | 60 | def process_stats(results): 61 | paths = [] 62 | 63 | for result in results: 64 | path = result['_id'] 65 | label = result['label'] 66 | 67 | # Convert stat date from UTC to PR timezone 68 | for stat in result['data']: 69 | stat['date'] = stat['date'].replace(tzinfo=utc).astimezone(pr) 70 | 71 | data = sorted(result['data'], key=lambda k: k['date'], reverse=True) 72 | first_stat = data[-1] 73 | last_stat = data[0] 74 | stats = [] 75 | prev_value = None 76 | 77 | if STATS.get(path): 78 | label = STATS[path]['label'] 79 | 80 | for stat in data[1:-1]: 81 | if stat['value'] != prev_value: 82 | stats.append(stat) 83 | 84 | prev_value = stat['value'] 85 | 86 | if not stats: 87 | stats = [last_stat] 88 | formatted_value = "{:,}".format(last_stat['value']) 89 | 90 | if STATS[path]['percent']: 91 | formatted_value = '{}%'.format(formatted_value) 92 | 93 | paths.append({ 94 | '_id': path, 95 | 'slug': path.replace('.', '-'), 96 | 'label': label, 97 | 'data': data, 98 | 'graph_data': stats, 99 | 'last_value': formatted_value 100 | }) 101 | else: 102 | stats = sorted(stats, key=lambda k: k['date']) 103 | 104 | if first_stat['value'] != stats[0]['value']: 105 | stats.insert(0, first_stat) 106 | else: 107 | stats[0] = first_stat 108 | 109 | if last_stat['value'] != stats[-1]['value']: 110 | stats.append(last_stat) 111 | else: 112 | stats[-1] = last_stat 113 | 114 | # This prevents graphs with just one point 115 | if len(stats) == 1: 116 | stats = [first_stat, last_stat] 117 | 118 | last_stat = stats[-1] 119 | formatted_value = "{:,}".format(last_stat['value']) 120 | 121 | if STATS[path]['percent']: 122 | formatted_value = '{}%'.format(formatted_value) 123 | 124 | paths.append({ 125 | '_id': path, 126 | 'slug': path.replace('.', '-'), 127 | 'label': label, 128 | 'data': data, 129 | 'graph_data': stats, 130 | 'last_value': formatted_value 131 | }) 132 | 133 | return paths 134 | 135 | 136 | @app.route('/favicon.ico') 137 | def favicon(): 138 | directory = os.path.join(app.root_path, 'static') 139 | return send_from_directory(directory, 'favicon.ico') 140 | 141 | 142 | @app.route('/embed.js') 143 | def embed_js(): 144 | directory = os.path.join(app.root_path, 'static', 'js') 145 | return send_from_directory(directory, 'embed.js') 146 | 147 | 148 | @app.route('/') 149 | @cached() 150 | def index(): 151 | results = get_stats() 152 | stats = sorted(process_stats(results), key=lambda k: k['label']) 153 | return render_template('index.html', stats=stats) 154 | 155 | 156 | @app.route('/embed/') 157 | def embed(path): 158 | results = get_stats(path) 159 | stats = process_stats(results) 160 | 161 | if not stats: 162 | return redirect('/') 163 | 164 | return render_template('embed.html', stat=stats[0]) 165 | 166 | 167 | @app.route('/stats/') 168 | def stat_details(stat): 169 | results = get_stats(stat) 170 | stats = process_stats(results) 171 | 172 | if not stats: 173 | return redirect('/') 174 | 175 | return render_template('details.html', stat=stats[0]) 176 | 177 | 178 | @app.route('/stats/.json') 179 | def stats_json(stat): 180 | data = [] 181 | stats = db.stats.find({'path': stat}).sort('created_at') 182 | 183 | for stat in stats: 184 | data.append({'date': stat['created_at'], 'value': stat['value']}) 185 | 186 | return jsonify(data) 187 | 188 | 189 | @app.route('/stats/.csv') 190 | def stats_csv(stat): 191 | def generate(stat): 192 | stats = db.stats.find({'path': stat}).sort('created_at') 193 | yield f"Date,Value\n" 194 | 195 | for stat in stats: 196 | created_at = stat['created_at'] 197 | value = stat['value'] 198 | yield f"{created_at},{value}\n" 199 | 200 | return Response(generate(stat), mimetype='text/plain') 201 | 202 | 203 | @app.route('/stats/.png') 204 | def stat_image(stat): 205 | results = get_stats(stat) 206 | stats = process_stats(results) 207 | 208 | if not stats: 209 | return redirect('/') 210 | 211 | stat = stats[0] 212 | width = 1200 213 | height = 630 214 | image = Image.open('./app/static/share-template.png', 'r') 215 | draw = ImageDraw.Draw(image) 216 | 217 | font = ImageFont.truetype('./fonts/SourceCodePro-Bold.otf', 100) 218 | text_width, text_height = draw.textsize(stat['label'], font=font) 219 | position = ((width - text_width) / 2, (height - text_height - 400) / 2) 220 | draw.text(position, stat['label'], font=font, align='center', fill='#000') 221 | 222 | font = ImageFont.truetype('./fonts/SourceCodePro-Regular.otf', 80) 223 | text_width, text_height = draw.textsize(stat['last_value'], font=font) 224 | position = ((width - text_width) / 2, (height - text_height) / 2) 225 | draw.text( 226 | position, stat['last_value'], font=font, align='center', fill='#000') 227 | 228 | byte_io = BytesIO() 229 | image.save(byte_io, 'PNG') 230 | byte_io.seek(0) 231 | 232 | return send_file(byte_io, mimetype='image/png') 233 | 234 | 235 | @app.route('/digest/') 236 | @app.route('/digest//') 237 | def digest(date, end_date=None): 238 | created_at = datetime.datetime.strptime(date, '%Y-%m-%d').astimezone(pr) 239 | created_at = created_at - datetime.timedelta(days=1) 240 | start_date = created_at + datetime.timedelta(hours=23, minutes=59) 241 | 242 | if not end_date: 243 | end_date = created_at + datetime.timedelta( 244 | days=1, hours=23, minutes=59) 245 | else: 246 | end_date = datetime.datetime.strptime(end_date, 247 | '%Y-%m-%d').astimezone(pr) 248 | end_date = end_date + datetime.timedelta(days=1, microseconds=-1) 249 | 250 | date_range = [start_date, end_date] 251 | 252 | results = db.stats.aggregate([{ 253 | '$sort': { 254 | 'created_at': 1 255 | } 256 | }, { 257 | '$match': { 258 | 'created_at': { 259 | '$gte': start_date, 260 | '$lte': end_date 261 | } 262 | } 263 | }, { 264 | '$group': { 265 | '_id': '$path', 266 | 'first': { 267 | '$first': '$$ROOT' 268 | }, 269 | 'last': { 270 | '$last': '$$ROOT' 271 | } 272 | } 273 | }, { 274 | '$project': { 275 | '_id': '$_id', 276 | 'first': '$first', 277 | 'last': '$last', 278 | 'change': { 279 | '$subtract': ['$last.value', '$first.value'] 280 | } 281 | } 282 | }, { 283 | '$sort': { 284 | '_id': 1 285 | } 286 | }]) 287 | 288 | data = [] 289 | 290 | for result in results: 291 | path = result['_id'] 292 | 293 | if STATS.get(path): 294 | label = STATS[path]['label'] 295 | 296 | if result['change'] > 0: 297 | sign = '+' 298 | else: 299 | sign = '' 300 | 301 | if STATS[path]['percent']: 302 | percent = '%' 303 | else: 304 | percent = '' 305 | 306 | change = '{sign}{change:0.2f}{percent}'.format( 307 | sign=sign, change=result['change'], percent=percent) 308 | 309 | data.append({ 310 | '_id': path, 311 | 'label': label, 312 | 'first': result['first'], 313 | 'last': result['last'], 314 | 'change': result['change'], 315 | 'display_change': change 316 | }) 317 | 318 | return render_template('digest.html', date_range=date_range, results=data) 319 | -------------------------------------------------------------------------------- /app/facebook.py: -------------------------------------------------------------------------------- 1 | import os 2 | import requests 3 | 4 | from flask import url_for 5 | 6 | from .app import app 7 | from .stats import STATS 8 | 9 | BASE_URL = 'https://graph.facebook.com/v2.10/' 10 | ACCESS_TOKEN = os.getenv('FACEBOOK_ACCESS_TOKEN') 11 | 12 | if __name__ == '__main__': 13 | for stat in STATS.keys(): 14 | with app.app_context(): 15 | url = url_for('stat_details', stat=stat, _external=True) 16 | 17 | r = requests.post(BASE_URL, data={ 18 | 'id': url, 19 | 'scrape': True, 20 | 'access_token': ACCESS_TOKEN 21 | }) 22 | 23 | print(r.json()) 24 | -------------------------------------------------------------------------------- /app/scraper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpadilla/tracking-status-pr/5c40a1f1578a2f69dd5e99c9e8b5b5bb0a4ebb60/app/scraper/__init__.py -------------------------------------------------------------------------------- /app/scraper/__main__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from scrapy.crawler import CrawlerProcess 4 | 5 | from .spider import StatusPRJSONSpider 6 | 7 | 8 | if __name__ == '__main__': 9 | process = CrawlerProcess({ 10 | 'LOG_ENABLED': True, 11 | 'FEED_URI': 'stdout:', 12 | 'FEED_FORMAT': 'json', 13 | 'ITEM_PIPELINES': { 14 | 'app.scraper.pipeline.MongoDBPipeline': 0 15 | }, 16 | 'MONGODB_URI': os.getenv('MONGODB_URI') 17 | }) 18 | process.crawl(StatusPRJSONSpider) 19 | process.start() 20 | -------------------------------------------------------------------------------- /app/scraper/pipeline.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import pymongo 3 | 4 | 5 | class MongoDBPipeline(object): 6 | collection_name = 'stats' 7 | 8 | def __init__(self, mongodb_uri): 9 | self.mongodb_uri = mongodb_uri 10 | 11 | @classmethod 12 | def from_crawler(cls, crawler): 13 | return cls( 14 | mongodb_uri=crawler.settings.get('MONGODB_URI') 15 | ) 16 | 17 | def open_spider(self, spider): 18 | self.client = pymongo.MongoClient(self.mongodb_uri) 19 | db = self.client.get_default_database() 20 | self.collection = db[self.collection_name] 21 | 22 | def close_spider(self, spider): 23 | self.client.close() 24 | 25 | def process_item(self, item, spider): 26 | document = dict(item) 27 | document['created_at'] = datetime.datetime.utcnow() 28 | self.collection.insert_one(document) 29 | return item 30 | -------------------------------------------------------------------------------- /app/scraper/spider.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import scrapy 4 | 5 | from .utils import strip_accents, is_float, is_int, is_blank 6 | 7 | 8 | def normalize_value(value): 9 | if isinstance(value, str): 10 | value = ( 11 | value 12 | .strip() 13 | .replace('%', '') 14 | .replace(',', '') 15 | ) 16 | 17 | try: 18 | if is_int(value): 19 | return int(value) 20 | except: 21 | pass 22 | 23 | try: 24 | if is_float(value): 25 | return float(value) 26 | except: 27 | pass 28 | 29 | return value 30 | 31 | 32 | def normalize_path(path): 33 | return ( 34 | path 35 | .strip() 36 | .replace('card.', '') 37 | .replace('nav.', '') 38 | .replace('.title', '') 39 | .replace(' ', '.').strip() 40 | ) 41 | 42 | 43 | def normalize_label(label): 44 | return strip_accents(label).strip() 45 | 46 | 47 | def normalize_last_updated(last_updated): 48 | if last_updated: 49 | return last_updated.strip().replace(': ', '') 50 | 51 | 52 | def parse_businesses_processing_pan(response): 53 | for card in response.css('.card'): 54 | path = card.css( 55 | 'p.text-muted > span::attr(data-i18n)').extract_first() 56 | 57 | if path == 'card.bread.title': 58 | label = card.css('p.text-muted > span::text').extract_first() 59 | value = card.css('p.text-muted > span.info::text').extract_first() 60 | 61 | last_updated_text = card.css( 62 | '.p-small-spacing > .text-muted::text').extract() 63 | 64 | if len(last_updated_text) > 1: 65 | last_updated = last_updated_text[1] 66 | else: 67 | last_updated = None 68 | 69 | return { 70 | 'label': normalize_label(label), 71 | 'path': normalize_path(path), 72 | 'value': normalize_value(value), 73 | 'last_updated_at': normalize_last_updated(last_updated) 74 | } 75 | 76 | 77 | CUSTOM_PARSERS = { 78 | 'card.bread.title': parse_businesses_processing_pan 79 | } 80 | 81 | 82 | def parse(response): 83 | for card in response.css('.card'): 84 | label = card.css('p.text-muted > span::text').extract_first() 85 | 86 | path = card.css( 87 | 'p.text-muted > span::attr(data-i18n)').extract_first() 88 | 89 | if is_blank(path): 90 | path = card.css( 91 | 'p.text-muted::attr(data-i18n)').extract_first() 92 | 93 | if is_blank(path): 94 | path = card.css( 95 | '.card-header h2::attr(data-i18n)').extract_first() 96 | 97 | if path in CUSTOM_PARSERS: 98 | yield CUSTOM_PARSERS[path](response) 99 | continue 100 | 101 | value = card.css( 102 | '.font-large-2.text-bold-300.info::text').extract_first() 103 | 104 | if is_blank(path): 105 | label = card.css('p.text-muted::text').extract_first() 106 | 107 | if is_blank(label): 108 | label = card.css('p.text-muted::text').extract_first() 109 | 110 | if is_blank(label): 111 | label = card.css('.card-header h2::text').extract_first() 112 | 113 | if is_blank(label): 114 | label = card.css('.card-header h3.grey::text').extract_first() 115 | 116 | if is_blank(value): 117 | value = card.css('.card-header h3.success::text').extract_first() 118 | 119 | last_updated_text = card.css( 120 | '.p-small-spacing > .text-muted::text').extract() 121 | 122 | if not last_updated_text: 123 | last_updated_text = card.css( 124 | '.list-inline li .text-muted::text').extract() 125 | 126 | if len(last_updated_text) > 1: 127 | last_updated = last_updated_text[1] 128 | else: 129 | last_updated = None 130 | 131 | if not path and label: 132 | path = label.lower() 133 | 134 | if not path or not value or not label: 135 | list_items = card.css('.list-inline li') 136 | 137 | for list_item in list_items: 138 | list_item_label = list_item.css('span.info::text').extract_first() 139 | list_item_path = list_item.css('span.info::attr(data-i18n)').extract_first() 140 | list_item_value = list_item.css('h1::text').extract_first() 141 | 142 | if not list_item_path and list_item_label: 143 | list_item_path = list_item_label.lower() 144 | 145 | if label and list_item_label: 146 | label = normalize_label(label) 147 | list_item_label = '{} - {}'.format(label, list_item_label) 148 | 149 | yield { 150 | 'label': normalize_label(list_item_label), 151 | 'path': normalize_path(list_item_path), 152 | 'value': normalize_value(list_item_value), 153 | 'last_updated_at': normalize_last_updated(last_updated) 154 | } 155 | 156 | if value: 157 | yield { 158 | 'label': normalize_label(label), 159 | 'path': normalize_path(path), 160 | 'value': normalize_value(value), 161 | 'last_updated_at': normalize_last_updated(last_updated) 162 | } 163 | 164 | 165 | class StatusPRSpider(scrapy.Spider): 166 | name = 'status-pr' 167 | start_urls = ['http://estatus.pr/'] 168 | 169 | def parse(self, response): 170 | return parse(response) 171 | 172 | 173 | class StatusPRJSONSpider(scrapy.Spider): 174 | name = 'status-pr-json' 175 | start_urls = ['http://estatus.pr/card-data/card-data.json'] 176 | 177 | def parse(self, response): 178 | stats = json.loads(response.body_as_unicode()) 179 | data = [] 180 | 181 | for stat in stats: 182 | label = stat['Description'] 183 | path = stat['Language'] 184 | card = stat['Card'][0] 185 | value = card['Value'] 186 | in_percentage = card['ValueIsInPercentage'] 187 | children = stat['CardDefinitionChild'] 188 | 189 | if not path and label: 190 | path = label.lower() 191 | 192 | if not in_percentage: 193 | value = int(value) 194 | 195 | if len(children) > 0 and value == 0: 196 | for child in children: 197 | child_label = child['Description'] 198 | child_path = child['Language'] 199 | child_card = child['CardDetail'][0] 200 | child_value = child_card['Value'] 201 | child_in_percentage = child_card['ValueIsInPercentage'] 202 | 203 | if not child_path and child_label: 204 | child_path = '{}.{}'.format(path, child_label.lower()) 205 | 206 | if not child_in_percentage: 207 | child_value = int(child_value) 208 | 209 | data.append({ 210 | 'label': normalize_label(child_label), 211 | 'path': normalize_path(child_path), 212 | 'value': child_value 213 | }) 214 | else: 215 | data.append({ 216 | 'label': normalize_label(label), 217 | 'path': normalize_path(path), 218 | 'value': value 219 | }) 220 | 221 | return data 222 | 223 | -------------------------------------------------------------------------------- /app/scraper/utils.py: -------------------------------------------------------------------------------- 1 | import unicodedata 2 | 3 | 4 | def strip_accents(s): 5 | return ''.join( 6 | c for c in unicodedata.normalize('NFD', s) 7 | if unicodedata.category(c) != 'Mn' 8 | ) 9 | 10 | 11 | def is_float(x): 12 | try: 13 | float(x) 14 | except ValueError: 15 | return False 16 | else: 17 | return True 18 | 19 | 20 | def is_int(x): 21 | try: 22 | a = float(x) 23 | b = int(a) 24 | except ValueError: 25 | return False 26 | else: 27 | return a == b 28 | 29 | 30 | def is_blank(s): 31 | blank = False 32 | 33 | if not s: 34 | blank = True 35 | 36 | try: 37 | if s.isspace(): 38 | blank = True 39 | except AttributeError: 40 | pass 41 | 42 | return blank 43 | -------------------------------------------------------------------------------- /app/static/css/app.css: -------------------------------------------------------------------------------- 1 | .card-title a { 2 | color: inherit; 3 | } 4 | 5 | .card-header a { 6 | color: #212529; 7 | text-decoration: underline; 8 | } 9 | 10 | .last-value { 11 | color: #212529; 12 | } 13 | 14 | .chart { 15 | height: 180px; 16 | width: 100%; 17 | display: block; 18 | } 19 | 20 | .header-link { 21 | display: block; 22 | } 23 | .header-link:hover { 24 | text-decoration: none; 25 | } 26 | 27 | .card-header .share-link { 28 | text-decoration: none; 29 | } 30 | 31 | .share-link svg:hover #twitter-icon { 32 | fill: #1DA1F2; 33 | } 34 | 35 | .share-link svg:hover #facebook-icon { 36 | fill: #3C5B9B; 37 | } 38 | -------------------------------------------------------------------------------- /app/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpadilla/tracking-status-pr/5c40a1f1578a2f69dd5e99c9e8b5b5bb0a4ebb60/app/static/favicon.ico -------------------------------------------------------------------------------- /app/static/github-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpadilla/tracking-status-pr/5c40a1f1578a2f69dd5e99c9e8b5b5bb0a4ebb60/app/static/github-logo.png -------------------------------------------------------------------------------- /app/static/js/app.js: -------------------------------------------------------------------------------- 1 | function formatValue(value) { 2 | return value.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",") 3 | } 4 | 5 | function generateChart(path) { 6 | c3.generate({ 7 | bindto: '#chart-' + path.slug, 8 | data: { 9 | json: path.graph_data, 10 | x: 'date', 11 | xFormat: '%Y-%m-%dT%H:%M:%S.%L', 12 | keys: { 13 | x: 'date', 14 | value: ['value'] 15 | }, 16 | names: { 17 | value: path.label 18 | } 19 | }, 20 | color: { 21 | pattern: ['#212529'] 22 | }, 23 | legend: { 24 | show: false 25 | }, 26 | axis: { 27 | y: { 28 | show: false 29 | }, 30 | x: { 31 | show: false, 32 | type: 'timeseries', 33 | tick: { 34 | format: '%Y-%m-%d %I%p' 35 | } 36 | } 37 | } 38 | }); 39 | } 40 | -------------------------------------------------------------------------------- /app/static/js/embed.js: -------------------------------------------------------------------------------- 1 | // Based on https://www.charted.co/embed.js 2 | 3 | var STATS_EMBED; 4 | 5 | (function () { 6 | if (STATS_EMBED && STATS_EMBED.createIframes && STATS_EMBED.onMessage) { 7 | STATS_EMBED.createIframes(); 8 | return; 9 | } 10 | 11 | STATS_EMBED = { 12 | createIframes: function () { 13 | var scripts = document.getElementsByTagName('script'); 14 | var i, statId, iframe, link; 15 | 16 | for (var i = 0; i < scripts.length; i++) { 17 | script = scripts[i]; 18 | 19 | if (script.getAttribute('data-embed-processed')) { 20 | continue; 21 | } 22 | 23 | statId = script.getAttribute('data-stat'); 24 | 25 | if (!statId) { 26 | continue; 27 | } 28 | 29 | iframe = document.createElement('iframe') 30 | iframe.setAttribute('id', 'stat-' + statId); 31 | iframe.setAttribute('height', '450px'); 32 | iframe.setAttribute('width', '100%'); 33 | iframe.setAttribute('scrolling', 'no'); 34 | iframe.style.border = 'none'; 35 | 36 | link = document.createElement('a'); 37 | link.setAttribute('href', script.getAttribute('src')); 38 | iframe.setAttribute('src', link.origin + '/embed/' + statId); 39 | 40 | script.parentNode.insertBefore(iframe, script.nextSibling); 41 | script.setAttribute('data-embed-processed', 'yes'); 42 | } 43 | } 44 | } 45 | 46 | STATS_EMBED.createIframes(); 47 | }()) 48 | -------------------------------------------------------------------------------- /app/static/share-template.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpadilla/tracking-status-pr/5c40a1f1578a2f69dd5e99c9e8b5b5bb0a4ebb60/app/static/share-template.png -------------------------------------------------------------------------------- /app/static/share.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpadilla/tracking-status-pr/5c40a1f1578a2f69dd5e99c9e8b5b5bb0a4ebb60/app/static/share.png -------------------------------------------------------------------------------- /app/stats.py: -------------------------------------------------------------------------------- 1 | STATS = { 2 | 'gas': { 3 | 'label': 'Gas Stations', 4 | 'percent': True 5 | }, 6 | 'supermarket': { 7 | 'label': 'Supermarket', 8 | 'percent': True 9 | }, 10 | 'aee': { 11 | 'label': 'AEE Generation', 12 | 'percent': True 13 | }, 14 | 'telecomunication': { 15 | 'label': 'Telecomm. Services', 16 | 'percent': True 17 | }, 18 | 'aaa': { 19 | 'label': 'AAA', 20 | 'percent': True 21 | }, 22 | 'antenna': { 23 | 'label': 'Cell Phone Antennas', 24 | 'percent': True 25 | }, 26 | 'port': { 27 | 'label': 'Open Ports', 28 | 'percent': True 29 | }, 30 | 'goverment.mail': { 31 | 'label': 'Postal Offices', 32 | 'percent': True 33 | }, 34 | 'ama': { 35 | 'label': 'AMA Routes', 36 | 'percent': True 37 | }, 38 | 'tourism.hotels': { 39 | 'label': 'Tourism / Hotels', 40 | 'percent': True 41 | }, 42 | 'complaint': { 43 | 'label': 'Complaints', 44 | 'percent': False 45 | }, 46 | 'complaint.products': { 47 | 'label': 'Complaints / Products', 48 | 'percent': True 49 | }, 50 | 'complaint.fuel': { 51 | 'label': 'Complaints / Fuel', 52 | 'percent': True 53 | }, 54 | 'complaint.ivu': { 55 | 'label': 'Complaints / IVU', 56 | 'percent': True 57 | }, 58 | 'complaint.other': { 59 | 'label': 'Complaints / Others', 60 | 'percent': True 61 | }, 62 | 'pet': { 63 | 'label': 'Displaced Pets', 64 | 'percent': False 65 | }, 66 | 'barrel.diesel': { 67 | 'label': 'Diesel Barrels Supplied', 68 | 'percent': False 69 | }, 70 | 'flight': { 71 | 'label': 'Commercial Flights', 72 | 'percent': True 73 | }, 74 | 'tourism.casinos': { 75 | 'label': 'Tourism / Casinos', 76 | 'percent': True 77 | }, 78 | 'cooperatives': { 79 | 'label': 'Cooperatives', 80 | 'percent': False 81 | }, 82 | 'refugee': { 83 | 'label': 'Shelterees', 84 | 'percent': False 85 | }, 86 | 'shelter': { 87 | 'label': 'Shelters', 88 | 'percent': False 89 | }, 90 | 'milk-industry': { 91 | 'label': 'Milk Industry', 92 | 'percent': True 93 | }, 94 | 'atms': { 95 | 'label': 'ATMs', 96 | 'percent': False 97 | }, 98 | 'pharmacy': { 99 | 'label': 'Online Processing Pharmacies', 100 | 'percent': False 101 | }, 102 | 'container': { 103 | 'label': 'Containers', 104 | 'percent': False 105 | }, 106 | 'bread': { 107 | 'label': 'Businesses Processing PAN', 108 | 'percent': False 109 | }, 110 | 'bank': { 111 | 'label': 'Bank Branches', 112 | 'percent': True 113 | }, 114 | 'barrel.gas': { 115 | 'label': 'Gasoline Barrels Supplied', 116 | 'percent': False 117 | }, 118 | 'dialysis': { 119 | 'label': 'Assisted Dialysis Centers', 120 | 'percent': False 121 | }, 122 | 'hospital': { 123 | 'label': 'Assisted Hospitals', 124 | 'percent': False 125 | }, 126 | 'tower': { 127 | 'label': 'Cell Towers', 128 | 'percent': True 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /app/templates/common/analytics.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 10 | -------------------------------------------------------------------------------- /app/templates/common/footer.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |

This website is open-source and not affiliated with the Government of Puerto Rico or status.pr.

5 |

Built by José Padilla

6 |
7 |
8 |
9 | -------------------------------------------------------------------------------- /app/templates/common/intro.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |

6 | Tracking status.pr 7 |

8 |

This tool scrapes status.pr every hour and keeps tracks of changing metrics in order to help visualize and measure progress. Data is also made available in CSV and JSON formats.

9 | Source code on GitHub 10 | Maria Tech Brigade 11 | {% if share %} 12 |
13 | 16 | 17 | 20 |
21 | {% endif %} 22 |
23 |
24 |
25 |
26 | -------------------------------------------------------------------------------- /app/templates/details.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | {{ stat.label }} | Tracking status.pr 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 |
22 | {% with share=False %} 23 | {% include "common/intro.html" %} 24 | {% endwith %} 25 | 26 |
27 |
28 |
29 |

30 |
31 |
32 | {{ stat.label }} 33 |
34 |
35 | 38 | 41 |
42 |
43 |

44 |
45 |
46 |
47 |
48 |
49 |
50 |

{{ stat.last_value }}

51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | {% for stat in stat.data %} 68 | 69 | 70 | 71 | 72 | {% endfor %} 73 | 74 |
DateValue
{{ stat.date.strftime('%Y-%m-%d %I%p') }}{{ stat.value }}
75 |
76 |
77 | 78 | {% include "common/footer.html" %} 79 |
80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 93 | 94 | {% include "common/analytics.html" %} 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /app/templates/digest.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Changes: {{ date_range.0.strftime('%b %d') }} - {{ date_range.1.strftime('%b %d') }} | Tracking status.pr 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 |
22 | {% with share=False %} 23 | {% include "common/intro.html" %} 24 | {% endwith %} 25 | 26 |
27 |
28 |

Changes: {{ date_range.0.strftime('%b %d') }} - {{ date_range.1.strftime('%b %d') }}

29 |
30 |
31 | 32 |
33 |
34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | {% for result in results %} 43 | 44 | 47 | 58 | 59 | {% endfor %} 60 | 61 |
StatChange
45 | {{ result.label }} 46 | 48 | {% if result.change > 0 %} 49 | {% set text_color = "text-success" %} 50 | {% elif result.change < 0 %} 51 | {% set text_color = "text-danger" %} 52 | {% else %} 53 | {% set text_color = "text-secondary" %} 54 | {% endif%} 55 | 56 | {{ result.display_change }} 57 |
62 |
63 |
64 | 65 | {% include "common/footer.html" %} 66 |
67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | {% include "common/analytics.html" %} 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /app/templates/embed.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Tracking status.pr 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 |
15 |
16 |
17 |
18 |

19 |
20 |
21 | {{ stat.label }} 22 |
23 |
24 | 27 | 30 |
31 |
32 |

33 |
34 |
35 |
36 |
37 |
38 |
39 |

{{ stat.last_value }}

40 |
41 |
42 |
43 |
44 |
45 |
46 |
Tracking status.pr
47 |
48 |
49 |
Source: status.pr
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 70 | 71 | {% include "common/analytics.html" %} 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /app/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Tracking status.pr 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 |
22 | {% with share=True %} 23 | {% include "common/intro.html" %} 24 | {% endwith %} 25 | 26 | {% for stat in stats %} 27 |
28 |
29 |
30 |

31 | {{ stat.label }} 32 |

33 |
34 |
35 |
36 |
37 |
38 |
39 |

{{ stat.last_value }}

40 |
41 |
42 | JSON 43 | CSV 44 | Embed 45 | 46 | 72 |
73 |
74 |
75 |
76 | {% endfor %} 77 | 78 | {% include "common/footer.html" %} 79 |
80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 95 | 96 | {% include "common/analytics.html" %} 97 | 98 | 99 | -------------------------------------------------------------------------------- /app/templates/static/facebook.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /app/templates/static/twitter.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /app/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datetime 3 | from functools import wraps 4 | 5 | import pylibmc 6 | from flask import json, request 7 | from werkzeug.contrib.cache import SimpleCache 8 | from werkzeug.contrib.cache import MemcachedCache 9 | 10 | try: 11 | mc = pylibmc.Client( 12 | servers=[os.getenv('MEMCACHEDCLOUD_SERVERS')], 13 | username=os.getenv('MEMCACHEDCLOUD_USERNAME'), 14 | password=os.getenv('MEMCACHEDCLOUD_PASSWORD') 15 | ) 16 | 17 | cache = MemcachedCache(mc) 18 | except Exception: 19 | cache = SimpleCache() 20 | 21 | 22 | class JSONEncoder(json.JSONEncoder): 23 | def default(self, obj): 24 | if isinstance(obj, datetime.datetime): 25 | return obj.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] 26 | 27 | return json.JSONEncoder.default(self, obj) 28 | 29 | 30 | # https://docs.djangoproject.com/en/1.8/howto/outputting-csv/ 31 | class Echo(object): 32 | """ 33 | An object that implements just the write method of the file-like 34 | interface. 35 | """ 36 | 37 | def write(self, value): 38 | """ 39 | Write the value by returning it, instead of storing 40 | in a buffer. 41 | """ 42 | return value 43 | 44 | 45 | # http://flask.pocoo.org/docs/0.12/patterns/viewdecorators/ 46 | def cached(timeout=5 * 60, key='view/%s'): 47 | def decorator(f): 48 | @wraps(f) 49 | def decorated_function(*args, **kwargs): 50 | cache_key = key % request.path 51 | rv = cache.get(cache_key) 52 | if rv is not None: 53 | return rv 54 | rv = f(*args, **kwargs) 55 | cache.set(cache_key, rv, timeout=timeout) 56 | return rv 57 | 58 | return decorated_function 59 | 60 | return decorator 61 | -------------------------------------------------------------------------------- /fonts/SourceCodePro-Bold.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpadilla/tracking-status-pr/5c40a1f1578a2f69dd5e99c9e8b5b5bb0a4ebb60/fonts/SourceCodePro-Bold.otf -------------------------------------------------------------------------------- /fonts/SourceCodePro-Regular.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpadilla/tracking-status-pr/5c40a1f1578a2f69dd5e99c9e8b5b5bb0a4ebb60/fonts/SourceCodePro-Regular.otf -------------------------------------------------------------------------------- /importer.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | 4 | import pymongo 5 | 6 | from scraper import ( 7 | normalize_value, normalize_label, normalize_path, normalize_last_updated 8 | ) 9 | 10 | client = pymongo.MongoClient(os.getenv('MONGODB_URI')) 11 | db = client.get_default_database() 12 | 13 | objects = [ 14 | { 15 | "Type": "AEE", 16 | "path": "aee", 17 | "Fecha Actualizado": "9/26/2017", 18 | "Status": "5.00%", 19 | "Fecha Obtenido": "9/26/2017" 20 | }, 21 | { 22 | "Type": "Antenas Celular", 23 | "path": "antenna", 24 | "Fecha Actualizado": "9/26/2017", 25 | "Status": "8.00%", 26 | "Fecha Obtenido": "9/26/2017" 27 | }, 28 | { 29 | "Type": "Gasolineras", 30 | "path": "gas", 31 | "Fecha Actualizado": "9/26/2017", 32 | "Status": "40.91%", 33 | "Fecha Obtenido": "9/26/2017" 34 | }, 35 | { 36 | "Type": "Puertos Abiertos", 37 | "path": "port", 38 | "Fecha Actualizado": "9/26/2017", 39 | "Status": "32.00%", 40 | "Fecha Obtenido": "9/26/2017" 41 | }, 42 | { 43 | "Type": "AEE", 44 | "path": "aee", 45 | "Fecha Actualizado": "9/27/2017", 46 | "Status": "5.00%", 47 | "Fecha Obtenido": "9/27/2017" 48 | }, 49 | { 50 | "Type": "Antenas Celular", 51 | "path": "antenna", 52 | "Fecha Actualizado": "9/27/2017", 53 | "Status": "9.00%", 54 | "Fecha Obtenido": "9/27/2017" 55 | }, 56 | { 57 | "Type": "ATMs", 58 | "path": "atms", 59 | "Fecha Actualizado": "9/27/2017", 60 | "Status": "150", 61 | "Fecha Obtenido": "9/27/2017" 62 | }, 63 | { 64 | "Type": "Correos", 65 | "path": "goverment.mail", 66 | "Fecha Actualizado": "9/27/2017", 67 | "Status": "5", 68 | "Fecha Obtenido": "9/27/2017" 69 | }, 70 | { 71 | "Type": "Gasolineras", 72 | "path": "gas", 73 | "Fecha Actualizado": "9/27/2017", 74 | "Status": "48.45%", 75 | "Fecha Obtenido": "9/27/2017" 76 | }, 77 | { 78 | "Type": "Hospitales Asistidos", 79 | "path": "hospital", 80 | "Fecha Actualizado": "9/27/2017", 81 | "Status": "29", 82 | "Fecha Obtenido": "9/27/2017" 83 | }, 84 | { 85 | "Type": "Puertos Abiertos", 86 | "path": "port", 87 | "Fecha Actualizado": "9/27/2017", 88 | "Status": "44.00%", 89 | "Fecha Obtenido": "9/27/2017" 90 | }, 91 | { 92 | "Type": "Refugiados", 93 | "path": "refugee", 94 | "Fecha Actualizado": "9/27/2017", 95 | "Status": "10,389", 96 | "Fecha Obtenido": "9/27/2017" 97 | }, 98 | { 99 | "Type": "Refugios", 100 | "path": "shelter", 101 | "Fecha Actualizado": "9/27/2017", 102 | "Status": "182", 103 | "Fecha Obtenido": "9/27/2017" 104 | }, 105 | { 106 | "Type": "Supermercados", 107 | "path": "supermarket", 108 | "Fecha Actualizado": "9/27/2017", 109 | "Status": "202.00", 110 | "Fecha Obtenido": "9/27/2017" 111 | }, 112 | { 113 | "Type": "AEE", 114 | "path": "aee", 115 | "Fecha Actualizado": "9/28/2017", 116 | "Status": "5.00%", 117 | "Fecha Obtenido": "9/28/2017" 118 | }, 119 | { 120 | "Type": "Antenas Celular", 121 | "path": "antenna", 122 | "Fecha Actualizado": "9/28/2017", 123 | "Status": "9.00%", 124 | "Fecha Obtenido": "9/28/2017" 125 | }, 126 | { 127 | "Type": "Correos", 128 | "path": "goverment.mail", 129 | "Fecha Actualizado": "9/28/2017", 130 | "Status": "28", 131 | "Fecha Obtenido": "9/28/2017" 132 | }, 133 | { 134 | "Type": "Diesel (abastos en barriles)", 135 | "path": "barrel.diesel", 136 | "Fecha Actualizado": "9/28/2017", 137 | "Status": "574,158", 138 | "Fecha Obtenido": "9/28/2017" 139 | }, 140 | { 141 | "Type": "Gasolina (abastos en barriles)", 142 | "path": "barrel.gas", 143 | "Fecha Actualizado": "9/28/2017", 144 | "Status": "402,781", 145 | "Fecha Obtenido": "9/28/2017" 146 | }, 147 | { 148 | "Type": "Gasolineras", 149 | "path": "gas", 150 | "Fecha Actualizado": "9/28/2017", 151 | "Status": "62.64%", 152 | "Fecha Obtenido": "9/28/2017" 153 | }, 154 | { 155 | "Type": "Hospitales Asistidos", 156 | "path": "hospital", 157 | "Fecha Actualizado": "9/28/2017", 158 | "Status": "33", 159 | "Fecha Obtenido": "9/28/2017" 160 | }, 161 | { 162 | "Type": "Puertos Abiertos", 163 | "path": "port", 164 | "Fecha Actualizado": "9/28/2017", 165 | "Status": "50.00%", 166 | "Fecha Obtenido": "9/28/2017" 167 | }, 168 | { 169 | "Type": "Supermercados", 170 | "path": "supermarket", 171 | "Fecha Actualizado": "9/28/2017", 172 | "Status": "224.00", 173 | "Fecha Obtenido": "9/28/2017" 174 | }, 175 | { 176 | "Type": "Telecomunicaciones", 177 | "path": "telecomunication", 178 | "Fecha Actualizado": "9/28/2017", 179 | "Status": "28.00%", 180 | "Fecha Obtenido": "9/28/2017" 181 | }, 182 | { 183 | "Type": "Torres celular", 184 | "path": "tower", 185 | "Fecha Actualizado": "9/28/2017", 186 | "Status": "28.00", 187 | "Fecha Obtenido": "9/28/2017" 188 | }, 189 | { 190 | "Type": "AAA Este", 191 | "Fecha Actualizado": "9/29/2017", 192 | "Status": "30.00%", 193 | "Fecha Obtenido": "9/29/2017" 194 | }, 195 | { 196 | "Type": "AAA Metro", 197 | "Fecha Actualizado": "9/29/2017", 198 | "Status": "49.00%", 199 | "Fecha Obtenido": "9/29/2017" 200 | }, 201 | { 202 | "Type": "AAA Norte", 203 | "Fecha Actualizado": "9/29/2017", 204 | "Status": "26.00%", 205 | "Fecha Obtenido": "9/29/2017" 206 | }, 207 | { 208 | "Type": "AAA Oeste", 209 | "Fecha Actualizado": "9/29/2017", 210 | "Status": "58.00%", 211 | "Fecha Obtenido": "9/29/2017" 212 | }, 213 | { 214 | "Type": "AAA Sur", 215 | "Fecha Actualizado": "9/29/2017", 216 | "Status": "39.00%", 217 | "Fecha Obtenido": "9/29/2017" 218 | }, 219 | { 220 | "Type": "AEE", 221 | "path": "aee", 222 | "Fecha Actualizado": "9/29/2017", 223 | "Status": "5.00%", 224 | "Fecha Obtenido": "9/29/2017" 225 | }, 226 | { 227 | "Type": "Antenas Celular", 228 | "path": "antenna", 229 | "Fecha Actualizado": "9/29/2017", 230 | "Status": "10.00%", 231 | "Fecha Obtenido": "9/29/2017" 232 | }, 233 | { 234 | "Type": "ATMs", 235 | "path": "atms", 236 | "Fecha Actualizado": "9/29/2017", 237 | "Status": "230", 238 | "Fecha Obtenido": "9/29/2017" 239 | }, 240 | { 241 | "Type": "Cooperativas", 242 | "path": "cooperatives", 243 | "Fecha Actualizado": "9/29/2017", 244 | "Status": "63", 245 | "Fecha Obtenido": "9/29/2017" 246 | }, 247 | { 248 | "Type": "Correos", 249 | "path": "goverment.mail", 250 | "Fecha Actualizado": "9/29/2017", 251 | "Status": "56", 252 | "Fecha Obtenido": "9/29/2017" 253 | }, 254 | { 255 | "Type": "Diesel (abastos en barriles)", 256 | "path": "barrel.diesel", 257 | "Fecha Actualizado": "9/29/2017", 258 | "Status": "562,935", 259 | "Fecha Obtenido": "9/29/2017" 260 | }, 261 | { 262 | "Type": "Diesel (abastos en barriles)", 263 | "path": "barrel.diesel", 264 | "Fecha Actualizado": "9/29/2017", 265 | "Status": "467,766", 266 | "Fecha Obtenido": "9/29/2017" 267 | }, 268 | { 269 | "Type": "Farmacias", 270 | "path": "pharmacy", 271 | "Fecha Actualizado": "9/29/2017", 272 | "Status": "311", 273 | "Fecha Obtenido": "9/29/2017" 274 | }, 275 | { 276 | "Type": "Gasolineras", 277 | "path": "gas", 278 | "Fecha Actualizado": "9/29/2017", 279 | "Status": "61.36%", 280 | "Fecha Obtenido": "9/29/2017" 281 | }, 282 | { 283 | "Type": "Hospitales Asistidos", 284 | "path": "hospital", 285 | "Fecha Actualizado": "9/29/2017", 286 | "Status": "33", 287 | "Fecha Obtenido": "9/29/2017" 288 | }, 289 | { 290 | "Type": "Mascotas Refugiadas", 291 | "path": "pet", 292 | "Fecha Actualizado": "9/29/2017", 293 | "Status": "193", 294 | "Fecha Obtenido": "9/29/2017" 295 | }, 296 | { 297 | "Type": "Puertos Abiertos", 298 | "path": "port", 299 | "Fecha Actualizado": "9/29/2017", 300 | "Status": "54.00%", 301 | "Fecha Obtenido": "9/29/2017" 302 | }, 303 | { 304 | "Type": "Refugiados", 305 | "path": "refugee", 306 | "Fecha Actualizado": "9/29/2017", 307 | "Status": "10201", 308 | "Fecha Obtenido": "9/29/2017" 309 | }, 310 | { 311 | "Type": "Refugios", 312 | "path": "shelter", 313 | "Fecha Actualizado": "9/29/2017", 314 | "Status": "161", 315 | "Fecha Obtenido": "9/29/2017" 316 | }, 317 | { 318 | "Type": "Telecomunicaciones", 319 | "path": "telecomunication", 320 | "Fecha Actualizado": "9/29/2017", 321 | "Status": "30.00%", 322 | "Fecha Obtenido": "9/29/2017" 323 | }, 324 | { 325 | "Type": "Torres celular", 326 | "path": "tower", 327 | "Fecha Actualizado": "9/29/2017", 328 | "Status": "96.00", 329 | "Fecha Obtenido": "9/29/2017" 330 | }, 331 | { 332 | "Type": "AAA Este", 333 | "Fecha Actualizado": "9/30/2017", 334 | "Status": "46.00%", 335 | "Fecha Obtenido": "9/30/2017" 336 | }, 337 | { 338 | "Type": "AAA Metro", 339 | "Fecha Actualizado": "9/30/2017", 340 | "Status": "55.00%", 341 | "Fecha Obtenido": "9/30/2017" 342 | }, 343 | { 344 | "Type": "AAA Norte", 345 | "Fecha Actualizado": "9/30/2017", 346 | "Status": "29.00%", 347 | "Fecha Obtenido": "9/30/2017" 348 | }, 349 | { 350 | "Type": "AAA Oeste", 351 | "Fecha Actualizado": "9/30/2017", 352 | "Status": "19.00%", 353 | "Fecha Obtenido": "9/30/2017" 354 | }, 355 | { 356 | "Type": "AAA Sur", 357 | "Fecha Actualizado": "9/30/2017", 358 | "Status": "72.00%", 359 | "Fecha Obtenido": "9/30/2017" 360 | }, 361 | { 362 | "Type": "AEE", 363 | "path": "aee", 364 | "Fecha Actualizado": "9/29/2017", 365 | "Status": "5.00%", 366 | "Fecha Obtenido": "9/30/2017" 367 | }, 368 | { 369 | "Type": "Centros de Dialisis Asistidos", 370 | "path": "dialysis", 371 | "Fecha Actualizado": "9/30/2017", 372 | "Status": "46", 373 | "Fecha Obtenido": "9/30/2017" 374 | }, 375 | { 376 | "Type": "Farmacias", 377 | "path": "pharmacy", 378 | "Fecha Actualizado": "9/28/2017", 379 | "Status": "337", 380 | "Fecha Obtenido": "9/30/2017" 381 | }, 382 | { 383 | "Type": "Gasolineras", 384 | "path": "gas", 385 | "Fecha Actualizado": "9/30/2017", 386 | "Status": "64.91%", 387 | "Fecha Obtenido": "9/30/2017" 388 | }, 389 | { 390 | "Type": "Hospitales Asistidos", 391 | "path": "hospital", 392 | "Fecha Actualizado": "9/30/2017", 393 | "Status": "51", 394 | "Fecha Obtenido": "9/30/2017" 395 | }, 396 | { 397 | "Type": "Mascotas Refugiadas", 398 | "path": "pet", 399 | "Fecha Actualizado": "9/30/2017", 400 | "Status": "234", 401 | "Fecha Obtenido": "9/30/2017" 402 | }, 403 | { 404 | "Type": "Puertos (capacidad operacional diaria)", 405 | "Fecha Actualizado": "9/30/2017", 406 | "Status": "62.92%", 407 | "Fecha Obtenido": "9/30/2017" 408 | }, 409 | { 410 | "Type": "Puertos Abiertos", 411 | "path": "port", 412 | "Fecha Actualizado": "9/29/2017", 413 | "Status": "75.00%", 414 | "Fecha Obtenido": "9/30/2017" 415 | }, 416 | { 417 | "Type": "Refugiados", 418 | "path": "refugee", 419 | "Fecha Actualizado": "9/30/2017", 420 | "Status": "10,201", 421 | "Fecha Obtenido": "9/30/2017" 422 | }, 423 | { 424 | "Type": "Refugios", 425 | "path": "shelter", 426 | "Fecha Actualizado": "9/30/2017", 427 | "Status": "150", 428 | "Fecha Obtenido": "9/30/2017" 429 | }, 430 | { 431 | "Type": "Supermercados", 432 | "path": "supermarket", 433 | "Fecha Actualizado": "9/28/2017", 434 | "Status": "49.12%", 435 | "Fecha Obtenido": "9/30/2017" 436 | }, 437 | { 438 | "Type": "Telecomunicaciones", 439 | "path": "telecomunication", 440 | "Fecha Actualizado": "9/30/2017", 441 | "Status": "30.50%", 442 | "Fecha Obtenido": "9/30/2017" 443 | }, 444 | { 445 | "Type": "Torres celular", 446 | "path": "tower", 447 | "Fecha Actualizado": "9/30/2017", 448 | "Status": "10.70%", 449 | "Fecha Obtenido": "9/30/2017" 450 | }, 451 | { 452 | "Type": "AAA Este", 453 | "Fecha Actualizado": "9/30/2017", 454 | "Status": "46.00%", 455 | "Fecha Obtenido": "10/1/2017" 456 | }, 457 | { 458 | "Type": "AAA Metro", 459 | "Fecha Actualizado": "9/30/2017", 460 | "Status": "55.00%", 461 | "Fecha Obtenido": "10/1/2017" 462 | }, 463 | { 464 | "Type": "AAA Norte", 465 | "Fecha Actualizado": "9/30/2017", 466 | "Status": "29.00%", 467 | "Fecha Obtenido": "10/1/2017" 468 | }, 469 | { 470 | "Type": "AAA Oeste", 471 | "Fecha Actualizado": "9/30/2017", 472 | "Status": "19.00%", 473 | "Fecha Obtenido": "10/1/2017" 474 | }, 475 | { 476 | "Type": "AAA Sur", 477 | "Fecha Actualizado": "9/30/2017", 478 | "Status": "72.00%", 479 | "Fecha Obtenido": "10/1/2017" 480 | }, 481 | { 482 | "Type": "AEE", 483 | "path": "aee", 484 | "Fecha Actualizado": "9/29/2017", 485 | "Status": "5.00%", 486 | "Fecha Obtenido": "10/1/2017" 487 | }, 488 | { 489 | "Type": "Centros de Dialisis Asistidos", 490 | "path": "dialysis", 491 | "Fecha Actualizado": "9/30/2017", 492 | "Status": "46", 493 | "Fecha Obtenido": "10/1/2017" 494 | }, 495 | { 496 | "Type": "Farmacias", 497 | "path": "pharmacy", 498 | "Fecha Actualizado": "9/28/2017", 499 | "Status": "337", 500 | "Fecha Obtenido": "10/1/2017" 501 | }, 502 | { 503 | "Type": "Gasolineras", 504 | "path": "gas", 505 | "Fecha Actualizado": "10/1/2017", 506 | "Status": "65.55%", 507 | "Fecha Obtenido": "10/1/2017" 508 | }, 509 | { 510 | "Type": "Hospitales Asistidos", 511 | "path": "hospital", 512 | "Fecha Actualizado": "9/30/2017", 513 | "Status": "51", 514 | "Fecha Obtenido": "10/1/2017" 515 | }, 516 | { 517 | "Type": "Mascotas Refugiadas", 518 | "path": "pet", 519 | "Fecha Actualizado": "9/30/2017", 520 | "Status": "234", 521 | "Fecha Obtenido": "10/1/2017" 522 | }, 523 | { 524 | "Type": "Puertos (capacidad operacional diaria)", 525 | "Fecha Actualizado": "9/30/2017", 526 | "Status": "62.92%", 527 | "Fecha Obtenido": "10/1/2017" 528 | }, 529 | { 530 | "Type": "Puertos Abiertos", 531 | "path": "port", 532 | "Fecha Actualizado": "9/29/2017", 533 | "Status": "75.00%", 534 | "Fecha Obtenido": "10/1/2017" 535 | }, 536 | { 537 | "Type": "Refugiados", 538 | "path": "refugee", 539 | "Fecha Actualizado": "9/30/2017", 540 | "Status": "8,800", 541 | "Fecha Obtenido": "10/1/2017" 542 | }, 543 | { 544 | "Type": "Refugios", 545 | "path": "shelter", 546 | "Fecha Actualizado": "9/30/2017", 547 | "Status": "139", 548 | "Fecha Obtenido": "10/1/2017" 549 | }, 550 | { 551 | "Type": "Supermercados", 552 | "path": "supermarket", 553 | "Fecha Actualizado": "9/28/2017", 554 | "Status": "49.12%", 555 | "Fecha Obtenido": "10/1/2017" 556 | }, 557 | { 558 | "Type": "Telecomunicaciones", 559 | "path": "telecomunication", 560 | "Fecha Actualizado": "9/30/2017", 561 | "Status": "30.80%", 562 | "Fecha Obtenido": "10/1/2017" 563 | }, 564 | { 565 | "Type": "Torres celular", 566 | "path": "tower", 567 | "Fecha Actualizado": "9/30/2017", 568 | "Status": "11.30%", 569 | "Fecha Obtenido": "10/1/2017" 570 | }, 571 | { 572 | "Type": "AAA Este", 573 | "Fecha Actualizado": "9/30/2017", 574 | "Status": "46.00%", 575 | "Fecha Obtenido": "10/2/2017" 576 | }, 577 | { 578 | "Type": "AAA Metro", 579 | "Fecha Actualizado": "9/30/2017", 580 | "Status": "55.00%", 581 | "Fecha Obtenido": "10/2/2017" 582 | }, 583 | { 584 | "Type": "AAA Norte", 585 | "Fecha Actualizado": "9/30/2017", 586 | "Status": "29.00%", 587 | "Fecha Obtenido": "10/2/2017" 588 | }, 589 | { 590 | "Type": "AAA Oeste", 591 | "Fecha Actualizado": "9/30/2017", 592 | "Status": "19.00%", 593 | "Fecha Obtenido": "10/2/2017" 594 | }, 595 | { 596 | "Type": "AAA Sur", 597 | "Fecha Actualizado": "9/30/2017", 598 | "Status": "72.00%", 599 | "Fecha Obtenido": "10/2/2017" 600 | }, 601 | { 602 | "Type": "AEE", 603 | "path": "aee", 604 | "Fecha Actualizado": "10/1/2017", 605 | "Status": "5.00%", 606 | "Fecha Obtenido": "10/2/2017" 607 | }, 608 | { 609 | "Type": "Centros de Dialisis Asistidos", 610 | "path": "dialysis", 611 | "Fecha Actualizado": "9/30/2017", 612 | "Status": "46", 613 | "Fecha Obtenido": "10/2/2017" 614 | }, 615 | { 616 | "Type": "Farmacias", 617 | "path": "pharmacy", 618 | "Fecha Actualizado": "9/28/2017", 619 | "Status": "337", 620 | "Fecha Obtenido": "10/2/2017" 621 | }, 622 | { 623 | "Type": "Gasolineras", 624 | "path": "gas", 625 | "Fecha Actualizado": "10/1/2017", 626 | "Status": "65.55%", 627 | "Fecha Obtenido": "10/2/2017" 628 | }, 629 | { 630 | "Type": "Hospitales Asistidos", 631 | "path": "hospital", 632 | "Fecha Actualizado": "9/30/2017", 633 | "Status": "51", 634 | "Fecha Obtenido": "10/2/2017" 635 | }, 636 | { 637 | "Type": "Mascotas Refugiadas", 638 | "path": "pet", 639 | "Fecha Actualizado": "10/1/2017", 640 | "Status": "234", 641 | "Fecha Obtenido": "10/2/2017" 642 | }, 643 | { 644 | "Type": "Puertos (capacidad operacional diaria)", 645 | "Fecha Actualizado": "9/30/2017", 646 | "Status": "62.92%", 647 | "Fecha Obtenido": "10/2/2017" 648 | }, 649 | { 650 | "Type": "Puertos Abiertos", 651 | "path": "port", 652 | "Fecha Actualizado": "9/29/2017", 653 | "Status": "75.00%", 654 | "Fecha Obtenido": "10/2/2017" 655 | }, 656 | { 657 | "Type": "Refugiados", 658 | "path": "refugee", 659 | "Fecha Actualizado": "10/1/2017", 660 | "Status": "8,867", 661 | "Fecha Obtenido": "10/2/2017" 662 | }, 663 | { 664 | "Type": "Refugios", 665 | "path": "shelter", 666 | "Fecha Actualizado": "10/1/2017", 667 | "Status": "139", 668 | "Fecha Obtenido": "10/2/2017" 669 | }, 670 | { 671 | "Type": "Supermercados", 672 | "path": "supermarket", 673 | "Fecha Actualizado": "9/30/2017", 674 | "Status": "64.69%", 675 | "Fecha Obtenido": "10/2/2017" 676 | }, 677 | { 678 | "Type": "Telecomunicaciones", 679 | "path": "telecomunication", 680 | "Fecha Actualizado": "10/1/2017", 681 | "Status": "35.80%", 682 | "Fecha Obtenido": "10/2/2017" 683 | }, 684 | { 685 | "Type": "Torres celular", 686 | "path": "tower", 687 | "Fecha Actualizado": "10/1/2017", 688 | "Status": "14.21%", 689 | "Fecha Obtenido": "10/2/2017" 690 | }, 691 | { 692 | "Type": "AAA", 693 | "Fecha Actualizado": "10/3/2017", 694 | "Status": "45.00%", 695 | "Fecha Obtenido": "10/3/2017" 696 | }, 697 | { 698 | "Type": "AAA Este", 699 | "Fecha Actualizado": "10/3/2017", 700 | "Status": "45.00%", 701 | "Fecha Obtenido": "10/3/2017" 702 | }, 703 | { 704 | "Type": "AAA Metro", 705 | "Fecha Actualizado": "10/3/2017", 706 | "Status": "57.00%", 707 | "Fecha Obtenido": "10/3/2017" 708 | }, 709 | { 710 | "Type": "AAA Norte", 711 | "Fecha Actualizado": "10/3/2017", 712 | "Status": "13.00%", 713 | "Fecha Obtenido": "10/3/2017" 714 | }, 715 | { 716 | "Type": "AAA Oeste", 717 | "Fecha Actualizado": "10/3/2017", 718 | "Status": "25.00%", 719 | "Fecha Obtenido": "10/3/2017" 720 | }, 721 | { 722 | "Type": "AAA Sur", 723 | "Fecha Actualizado": "10/3/2017", 724 | "Status": "73.00%", 725 | "Fecha Obtenido": "10/3/2017" 726 | }, 727 | { 728 | "Type": "AEE", 729 | "path": "aee", 730 | "Fecha Actualizado": "10/3/2017", 731 | "Status": "6.89%", 732 | "Fecha Obtenido": "10/3/2017" 733 | }, 734 | { 735 | "Type": "Antenas Celular", 736 | "path": "antenna", 737 | "Fecha Actualizado": "10/2/2017", 738 | "Status": "11.68%", 739 | "Fecha Obtenido": "10/3/2017" 740 | }, 741 | { 742 | "Type": "ATMs", 743 | "path": "atms", 744 | "Fecha Actualizado": "10/1/2017", 745 | "Status": "315", 746 | "Fecha Obtenido": "10/3/2017" 747 | }, 748 | { 749 | "Type": "Centros de Dialisis Asistidos", 750 | "path": "dialysis", 751 | "Fecha Actualizado": "10/3/2017", 752 | "Status": "46", 753 | "Fecha Obtenido": "10/3/2017" 754 | }, 755 | { 756 | "Type": "Cooperativas", 757 | "path": "cooperatives", 758 | "Fecha Actualizado": "10/3/2017", 759 | "Status": "85", 760 | "Fecha Obtenido": "10/3/2017" 761 | }, 762 | { 763 | "Type": "Diesel (abastos en barriles)", 764 | "path": "barrel.diesel", 765 | "Fecha Actualizado": "10/1/2017", 766 | "Status": "539,715", 767 | "Fecha Obtenido": "10/3/2017" 768 | }, 769 | { 770 | "Type": "Farmacias", 771 | "path": "pharmacy", 772 | "Fecha Actualizado": "10/2/2017", 773 | "Status": "606", 774 | "Fecha Obtenido": "10/3/2017" 775 | }, 776 | { 777 | "Type": "Gasolina (abastos en barriles)", 778 | "path": "barrel.gas", 779 | "Fecha Actualizado": "10/1/2017", 780 | "Status": "467,766", 781 | "Fecha Obtenido": "10/3/2017" 782 | }, 783 | { 784 | "Type": "Gasolineras", 785 | "path": "gas", 786 | "Fecha Actualizado": "10/3/2017", 787 | "Status": "74.00%", 788 | "Fecha Obtenido": "10/3/2017" 789 | }, 790 | { 791 | "Type": "Hospitales Asistidos", 792 | "path": "hospital", 793 | "Fecha Actualizado": "10/2/2017", 794 | "Status": "51", 795 | "Fecha Obtenido": "10/3/2017" 796 | }, 797 | { 798 | "Type": "Mascotas Refugiadas", 799 | "path": "pet", 800 | "Fecha Actualizado": "10/2/2017", 801 | "Status": "232", 802 | "Fecha Obtenido": "10/3/2017" 803 | }, 804 | { 805 | "Type": "Puertos (capacidad operacional diaria)", 806 | "Fecha Actualizado": "10/3/2017", 807 | "Status": "61.07%", 808 | "Fecha Obtenido": "10/3/2017" 809 | }, 810 | { 811 | "Type": "Puertos Abiertos", 812 | "path": "port", 813 | "Fecha Actualizado": "10/3/2017", 814 | "Status": "75.00%", 815 | "Fecha Obtenido": "10/3/2017" 816 | }, 817 | { 818 | "Type": "Refugiados", 819 | "path": "refugee", 820 | "Fecha Actualizado": "10/2/2017", 821 | "Status": "8,867", 822 | "Fecha Obtenido": "10/3/2017" 823 | }, 824 | { 825 | "Type": "Refugios", 826 | "path": "shelter", 827 | "Fecha Actualizado": "10/2/2017", 828 | "Status": "149", 829 | "Fecha Obtenido": "10/3/2017" 830 | }, 831 | { 832 | "Type": "Sucursales Bancarias", 833 | "path": "bank", 834 | "Fecha Actualizado": "10/3/2017", 835 | "Status": "153", 836 | "Fecha Obtenido": "10/3/2017" 837 | }, 838 | { 839 | "Type": "Supermercados", 840 | "path": "supermarket", 841 | "Fecha Actualizado": "10/3/2017", 842 | "Status": "64.69%", 843 | "Fecha Obtenido": "10/3/2017" 844 | }, 845 | { 846 | "Type": "Telecomunicaciones", 847 | "path": "telecomunication", 848 | "Fecha Actualizado": "10/3/2017", 849 | "Status": "40.00%", 850 | "Fecha Obtenido": "10/3/2017" 851 | }, 852 | { 853 | "Type": "Torres celular", 854 | "path": "tower", 855 | "Fecha Actualizado": "10/2/2017", 856 | "Status": "22.54%", 857 | "Fecha Obtenido": "10/3/2017" 858 | }, 859 | { 860 | "Type": "Vuelos Comerciales", 861 | "path": "flight", 862 | "Fecha Actualizado": "10/2/2017", 863 | "Status": "26.00%", 864 | "Fecha Obtenido": "10/3/2017" 865 | }, 866 | { 867 | "Type": "Vuelos Comerciales (Capacidad Diaria)", 868 | "Fecha Actualizado": "10/2/2017", 869 | "Status": "25.63%", 870 | "Fecha Obtenido": "10/3/2017" 871 | }, 872 | { 873 | "Type": "Vuelos Comerciales (Domesticos)", 874 | "Fecha Actualizado": "10/2/2017", 875 | "Status": "38", 876 | "Fecha Obtenido": "10/3/2017" 877 | }, 878 | { 879 | "Type": "Vuelos Comerciales (Internacionales)", 880 | "Fecha Actualizado": "10/2/2017", 881 | "Status": "3", 882 | "Fecha Obtenido": "10/3/2017" 883 | }, 884 | { 885 | "Type": "AAA", 886 | "Fecha Actualizado": "10/4/2017", 887 | "Status": "48.20%", 888 | "Fecha Obtenido": "10/4/2017" 889 | }, 890 | { 891 | "Type": "AAA Este", 892 | "Fecha Actualizado": "10/4/2017", 893 | "Status": "45.20%", 894 | "Fecha Obtenido": "10/4/2017" 895 | }, 896 | { 897 | "Type": "AAA Metro", 898 | "Fecha Actualizado": "10/4/2017", 899 | "Status": "63.33%", 900 | "Fecha Obtenido": "10/4/2017" 901 | }, 902 | { 903 | "Type": "AAA Norte", 904 | "Fecha Actualizado": "10/4/2017", 905 | "Status": "14.67%", 906 | "Fecha Obtenido": "10/4/2017" 907 | }, 908 | { 909 | "Type": "AAA Oeste", 910 | "Fecha Actualizado": "10/4/2017", 911 | "Status": "30.31%", 912 | "Fecha Obtenido": "10/4/2017" 913 | }, 914 | { 915 | "Type": "AAA Sur", 916 | "Fecha Actualizado": "10/4/2017", 917 | "Status": "77.68%", 918 | "Fecha Obtenido": "10/4/2017" 919 | }, 920 | { 921 | "Type": "AEE", 922 | "path": "aee", 923 | "Fecha Actualizado": "10/4/2017", 924 | "Status": "8.60%", 925 | "Fecha Obtenido": "10/4/2017" 926 | }, 927 | { 928 | "Type": "Antenas Celular", 929 | "path": "antenna", 930 | "Fecha Actualizado": "10/2/2017", 931 | "Status": "11.68%", 932 | "Fecha Obtenido": "10/4/2017" 933 | }, 934 | { 935 | "Type": "ATMs", 936 | "path": "atms", 937 | "Fecha Actualizado": "10/4/2017", 938 | "Status": "430", 939 | "Fecha Obtenido": "10/4/2017" 940 | }, 941 | { 942 | "Type": "Centros de Dialisis Asistidos", 943 | "path": "dialysis", 944 | "Fecha Actualizado": "10/3/2017", 945 | "Status": "46", 946 | "Fecha Obtenido": "10/4/2017" 947 | }, 948 | { 949 | "Type": "Comercios procesando PAN", 950 | "path": "comercios.procesando.pan", 951 | "Fecha Actualizado": "10/3/2017", 952 | "Status": "510", 953 | "Fecha Obtenido": "10/4/2017" 954 | }, 955 | { 956 | "Type": "Contenedores", 957 | "path": "container", 958 | "Fecha Actualizado": "10/4/2017", 959 | "Status": "64.29%", 960 | "Fecha Obtenido": "10/4/2017" 961 | }, 962 | { 963 | "Type": "Cooperativas", 964 | "path": "cooperatives", 965 | "Fecha Actualizado": "10/4/2017", 966 | "Status": "100", 967 | "Fecha Obtenido": "10/4/2017" 968 | }, 969 | { 970 | "Type": "Diesel (abastos en barriles)", 971 | "path": "barrel.diesel", 972 | "Fecha Actualizado": "10/4/2017", 973 | "Status": "443,593", 974 | "Fecha Obtenido": "10/4/2017" 975 | }, 976 | { 977 | "Type": "Farmacias", 978 | "path": "pharmacy", 979 | "Fecha Actualizado": "10/3/2017", 980 | "Status": "462", 981 | "Fecha Obtenido": "10/4/2017" 982 | }, 983 | { 984 | "Type": "Gasolina (abastos en barriles)", 985 | "path": "barrel.gas", 986 | "Fecha Actualizado": "10/4/2017", 987 | "Status": "402,781", 988 | "Fecha Obtenido": "10/4/2017" 989 | }, 990 | { 991 | "Type": "Gasolineras", 992 | "path": "gas", 993 | "Fecha Actualizado": "10/4/2017", 994 | "Status": "76.09%", 995 | "Fecha Obtenido": "10/4/2017" 996 | }, 997 | { 998 | "Type": "Hospitales Asistidos", 999 | "path": "hospital", 1000 | "Fecha Actualizado": "10/3/2017", 1001 | "Status": "51", 1002 | "Fecha Obtenido": "10/4/2017" 1003 | }, 1004 | { 1005 | "Type": "Mascotas Refugiadas", 1006 | "path": "pet", 1007 | "Fecha Actualizado": "10/4/2017", 1008 | "Status": "193", 1009 | "Fecha Obtenido": "10/4/2017" 1010 | }, 1011 | { 1012 | "Type": "Puertos (capacidad operacional diaria)", 1013 | "Fecha Actualizado": "10/3/2017", 1014 | "Status": "", 1015 | "Fecha Obtenido": "10/4/2017" 1016 | }, 1017 | { 1018 | "Type": "Puertos Abiertos", 1019 | "path": "port", 1020 | "Fecha Actualizado": "10/3/2017", 1021 | "Status": "75.00%", 1022 | "Fecha Obtenido": "10/4/2017" 1023 | }, 1024 | { 1025 | "Type": "Refugiados", 1026 | "path": "refugee", 1027 | "Fecha Actualizado": "10/4/2017", 1028 | "Status": "8,802", 1029 | "Fecha Obtenido": "10/4/2017" 1030 | }, 1031 | { 1032 | "Type": "Refugios", 1033 | "path": "shelter", 1034 | "Fecha Actualizado": "10/4/2017", 1035 | "Status": "135", 1036 | "Fecha Obtenido": "10/4/2017" 1037 | }, 1038 | { 1039 | "Type": "Rutas AMA", 1040 | "path": "ama", 1041 | "Fecha Actualizado": "10/4/2017", 1042 | "Status": "75.00%", 1043 | "Fecha Obtenido": "10/4/2017" 1044 | }, 1045 | { 1046 | "Type": "Sucursales Bancarias", 1047 | "path": "bank", 1048 | "Fecha Actualizado": "10/3/2017", 1049 | "Status": "153", 1050 | "Fecha Obtenido": "10/4/2017" 1051 | }, 1052 | { 1053 | "Type": "Supermercados", 1054 | "path": "supermarket", 1055 | "Fecha Actualizado": "10/3/2017", 1056 | "Status": "69.74%", 1057 | "Fecha Obtenido": "10/4/2017" 1058 | }, 1059 | { 1060 | "Type": "Telecomunicaciones", 1061 | "path": "telecomunication", 1062 | "Fecha Actualizado": "10/4/2017", 1063 | "Status": "43.32%", 1064 | "Fecha Obtenido": "10/4/2017" 1065 | }, 1066 | { 1067 | "Type": "Torres celular", 1068 | "path": "tower", 1069 | "Fecha Actualizado": "10/3/2017", 1070 | "Status": "22.54%", 1071 | "Fecha Obtenido": "10/4/2017" 1072 | }, 1073 | { 1074 | "Type": "Vuelos Comerciales", 1075 | "path": "flight", 1076 | "Fecha Actualizado": "10/2/2017", 1077 | "Status": "100.00%", 1078 | "Fecha Obtenido": "10/4/2017" 1079 | }, 1080 | { 1081 | "Type": "Vuelos Comerciales (Capacidad Diaria)", 1082 | "Fecha Actualizado": "10/2/2017", 1083 | "Status": "100.00%", 1084 | "Fecha Obtenido": "10/4/2017" 1085 | }, 1086 | { 1087 | "Type": "Vuelos Comerciales (Domesticos)", 1088 | "Fecha Actualizado": "10/2/2017", 1089 | "Status": "", 1090 | "Fecha Obtenido": "10/4/2017" 1091 | }, 1092 | { 1093 | "Type": "Vuelos Comerciales (Internacionales)", 1094 | "Fecha Actualizado": "10/2/2017", 1095 | "Status": "", 1096 | "Fecha Obtenido": "10/4/2017" 1097 | }, 1098 | { 1099 | "Type": "AAA", 1100 | "Fecha Actualizado": "10/5/2017", 1101 | "Status": "54.20%", 1102 | "Fecha Obtenido": "10/5/2017" 1103 | }, 1104 | { 1105 | "Type": "AAA Este", 1106 | "Fecha Actualizado": "10/5/2017", 1107 | "Status": "62.73%", 1108 | "Fecha Obtenido": "10/5/2017" 1109 | }, 1110 | { 1111 | "Type": "AAA Metro", 1112 | "Fecha Actualizado": "10/5/2017", 1113 | "Status": "63.87%", 1114 | "Fecha Obtenido": "10/5/2017" 1115 | }, 1116 | { 1117 | "Type": "AAA Norte", 1118 | "Fecha Actualizado": "10/5/2017", 1119 | "Status": "19.93%", 1120 | "Fecha Obtenido": "10/5/2017" 1121 | }, 1122 | { 1123 | "Type": "AAA Oeste", 1124 | "Fecha Actualizado": "10/5/2017", 1125 | "Status": "39.85%", 1126 | "Fecha Obtenido": "10/5/2017" 1127 | }, 1128 | { 1129 | "Type": "AAA Sur", 1130 | "Fecha Actualizado": "10/5/2017", 1131 | "Status": "77.23%", 1132 | "Fecha Obtenido": "10/5/2017" 1133 | }, 1134 | { 1135 | "Type": "AEE", 1136 | "path": "aee", 1137 | "Fecha Actualizado": "10/5/2017", 1138 | "Status": "9.20%", 1139 | "Fecha Obtenido": "10/5/2017" 1140 | }, 1141 | { 1142 | "Type": "Antenas Celular", 1143 | "path": "antenna", 1144 | "Fecha Actualizado": "10/5/2017", 1145 | "Status": "13.55%", 1146 | "Fecha Obtenido": "10/5/2017" 1147 | }, 1148 | { 1149 | "Type": "ATMs", 1150 | "path": "atms", 1151 | "Fecha Actualizado": "10/5/2017", 1152 | "Status": "409", 1153 | "Fecha Obtenido": "10/5/2017" 1154 | }, 1155 | { 1156 | "Type": "Centros de Dialisis Asistidos", 1157 | "path": "dialysis", 1158 | "Fecha Actualizado": "10/3/2017", 1159 | "Status": "46", 1160 | "Fecha Obtenido": "10/5/2017" 1161 | }, 1162 | { 1163 | "Type": "Comercios procesando PAN", 1164 | "path": "comercios.procesando.pan", 1165 | "Fecha Actualizado": "10/5/2017", 1166 | "Status": "683", 1167 | "Fecha Obtenido": "10/5/2017" 1168 | }, 1169 | { 1170 | "Type": "Contenedores", 1171 | "path": "container", 1172 | "Fecha Actualizado": "10/5/2017", 1173 | "Status": "71.71%", 1174 | "Fecha Obtenido": "10/5/2017" 1175 | }, 1176 | { 1177 | "Type": "Cooperativas", 1178 | "path": "cooperatives", 1179 | "Fecha Actualizado": "10/4/2017", 1180 | "Status": "100", 1181 | "Fecha Obtenido": "10/5/2017" 1182 | }, 1183 | { 1184 | "Type": "Correos", 1185 | "path": "goverment.mail", 1186 | "Fecha Actualizado": "10/5/2017", 1187 | "Status": "90.37%", 1188 | "Fecha Obtenido": "10/5/2017" 1189 | }, 1190 | { 1191 | "Type": "Diesel (abastos en barriles)", 1192 | "path": "barrel.diesel", 1193 | "Fecha Actualizado": "10/5/2017", 1194 | "Status": "538,624", 1195 | "Fecha Obtenido": "10/5/2017" 1196 | }, 1197 | { 1198 | "Type": "Farmacias", 1199 | "path": "pharmacy", 1200 | "Fecha Actualizado": "10/4/2017", 1201 | "Status": "514", 1202 | "Fecha Obtenido": "10/5/2017" 1203 | }, 1204 | { 1205 | "Type": "Gasolina (abastos en barriles)", 1206 | "path": "barrel.gas", 1207 | "Fecha Actualizado": "10/5/2017", 1208 | "Status": "447,041", 1209 | "Fecha Obtenido": "10/5/2017" 1210 | }, 1211 | { 1212 | "Type": "Gasolineras", 1213 | "path": "gas", 1214 | "Fecha Actualizado": "10/5/2017", 1215 | "Status": "77.91%", 1216 | "Fecha Obtenido": "10/5/2017" 1217 | }, 1218 | { 1219 | "Type": "Hospitales Asistidos", 1220 | "path": "hospital", 1221 | "Fecha Actualizado": "10/5/2017", 1222 | "Status": "64", 1223 | "Fecha Obtenido": "10/5/2017" 1224 | }, 1225 | { 1226 | "Type": "Mascotas Refugiadas", 1227 | "path": "pet", 1228 | "Fecha Actualizado": "10/5/2017", 1229 | "Status": "196", 1230 | "Fecha Obtenido": "10/5/2017" 1231 | }, 1232 | { 1233 | "Type": "Puertos (capacidad operacional diaria)", 1234 | "Fecha Actualizado": "10/3/2017", 1235 | "Status": "", 1236 | "Fecha Obtenido": "10/5/2017" 1237 | }, 1238 | { 1239 | "Type": "Puertos Abiertos", 1240 | "path": "port", 1241 | "Fecha Actualizado": "10/3/2017", 1242 | "Status": "75.00%", 1243 | "Fecha Obtenido": "10/5/2017" 1244 | }, 1245 | { 1246 | "Type": "Refugiados", 1247 | "path": "refugee", 1248 | "Fecha Actualizado": "10/5/2017", 1249 | "Status": "8,585", 1250 | "Fecha Obtenido": "10/5/2017" 1251 | }, 1252 | { 1253 | "Type": "Refugios", 1254 | "path": "shelter", 1255 | "Fecha Actualizado": "10/5/2017", 1256 | "Status": "132", 1257 | "Fecha Obtenido": "10/5/2017" 1258 | }, 1259 | { 1260 | "Type": "Rutas AMA", 1261 | "path": "ama", 1262 | "Fecha Actualizado": "10/4/2017", 1263 | "Status": "75.00%", 1264 | "Fecha Obtenido": "10/5/2017" 1265 | }, 1266 | { 1267 | "Type": "Sucursales Bancarias", 1268 | "path": "bank", 1269 | "Fecha Actualizado": "10/5/2017", 1270 | "Status": "55.27%", 1271 | "Fecha Obtenido": "10/5/2017" 1272 | }, 1273 | { 1274 | "Type": "Supermercados", 1275 | "path": "supermarket", 1276 | "Fecha Actualizado": "10/5/2017", 1277 | "Status": "79.17%", 1278 | "Fecha Obtenido": "10/5/2017" 1279 | }, 1280 | { 1281 | "Type": "Telecomunicaciones", 1282 | "path": "telecomunication", 1283 | "Fecha Actualizado": "10/5/2017", 1284 | "Status": "45.00%", 1285 | "Fecha Obtenido": "10/5/2017" 1286 | }, 1287 | { 1288 | "Type": "Torres celular", 1289 | "path": "tower", 1290 | "Fecha Actualizado": "10/5/2017", 1291 | "Status": "26.13%", 1292 | "Fecha Obtenido": "10/5/2017" 1293 | }, 1294 | { 1295 | "Type": "Vuelos Comerciales", 1296 | "path": "flight", 1297 | "Fecha Actualizado": "10/5/2017", 1298 | "Status": "100.00%", 1299 | "Fecha Obtenido": "10/5/2017" 1300 | }, 1301 | { 1302 | "Type": "Vuelos Comerciales (Capacidad Diaria)", 1303 | "Fecha Actualizado": "10/2/2017", 1304 | "Status": "", 1305 | "Fecha Obtenido": "10/5/2017" 1306 | }, 1307 | { 1308 | "Type": "Vuelos Comerciales (Domesticos)", 1309 | "Fecha Actualizado": "10/2/2017", 1310 | "Status": "", 1311 | "Fecha Obtenido": "10/5/2017" 1312 | }, 1313 | { 1314 | "Type": "Vuelos Comerciales (Internacionales)", 1315 | "Fecha Actualizado": "10/2/2017", 1316 | "Status": "", 1317 | "Fecha Obtenido": "10/5/2017" 1318 | }, 1319 | { 1320 | "Type": "AAA", 1321 | "Fecha Actualizado": "10/6/2017", 1322 | "Status": "55.50%", 1323 | "Fecha Obtenido": "10/6/2017" 1324 | }, 1325 | { 1326 | "Type": "AAA Este", 1327 | "Fecha Actualizado": "10/6/2017", 1328 | "Status": "63.00%", 1329 | "Fecha Obtenido": "10/6/2017" 1330 | }, 1331 | { 1332 | "Type": "AAA Metro", 1333 | "Fecha Actualizado": "10/6/2017", 1334 | "Status": "64.00%", 1335 | "Fecha Obtenido": "10/6/2017" 1336 | }, 1337 | { 1338 | "Type": "AAA Norte", 1339 | "Fecha Actualizado": "10/6/2017", 1340 | "Status": "28.00%", 1341 | "Fecha Obtenido": "10/6/2017" 1342 | }, 1343 | { 1344 | "Type": "AAA Oeste", 1345 | "Fecha Actualizado": "10/6/2017", 1346 | "Status": "69.00%", 1347 | "Fecha Obtenido": "10/6/2017" 1348 | }, 1349 | { 1350 | "Type": "AAA Sur", 1351 | "Fecha Actualizado": "10/6/2017", 1352 | "Status": "69.00%", 1353 | "Fecha Obtenido": "10/6/2017" 1354 | }, 1355 | { 1356 | "Type": "AEE", 1357 | "path": "aee", 1358 | "Fecha Actualizado": "10/6/2017", 1359 | "Status": "10.70%", 1360 | "Fecha Obtenido": "10/6/2017" 1361 | }, 1362 | { 1363 | "Type": "Antenas Celular", 1364 | "path": "antenna", 1365 | "Fecha Actualizado": "10/6/2017", 1366 | "Status": "15.20%", 1367 | "Fecha Obtenido": "10/6/2017" 1368 | }, 1369 | { 1370 | "Type": "ATMs", 1371 | "path": "atms", 1372 | "Fecha Actualizado": "10/6/2017", 1373 | "Status": "481", 1374 | "Fecha Obtenido": "10/6/2017" 1375 | }, 1376 | { 1377 | "Type": "Centros de Dialisis Asistidos", 1378 | "path": "dialysis", 1379 | "Fecha Actualizado": "10/6/2017", 1380 | "Status": "46", 1381 | "Fecha Obtenido": "10/6/2017" 1382 | }, 1383 | { 1384 | "Type": "Comercios procesando PAN", 1385 | "path": "comercios.procesando.pan", 1386 | "Fecha Actualizado": "10/6/2017", 1387 | "Status": "490", 1388 | "Fecha Obtenido": "10/6/2017" 1389 | }, 1390 | { 1391 | "Type": "Contenedores", 1392 | "path": "container", 1393 | "Fecha Actualizado": "10/6/2017", 1394 | "Status": "81.64%", 1395 | "Fecha Obtenido": "10/6/2017" 1396 | }, 1397 | { 1398 | "Type": "Cooperativas", 1399 | "path": "cooperatives", 1400 | "Fecha Actualizado": "10/6/2017", 1401 | "Status": "116", 1402 | "Fecha Obtenido": "10/6/2017" 1403 | }, 1404 | { 1405 | "Type": "Correos", 1406 | "path": "goverment.mail", 1407 | "Fecha Actualizado": "10/6/2017", 1408 | "Status": "92.42%", 1409 | "Fecha Obtenido": "10/6/2017" 1410 | }, 1411 | { 1412 | "Type": "Diesel (abastos en barriles)", 1413 | "path": "barrel.diesel", 1414 | "Fecha Actualizado": "10/6/2017", 1415 | "Status": "513,892", 1416 | "Fecha Obtenido": "10/6/2017" 1417 | }, 1418 | { 1419 | "Type": "Farmacias", 1420 | "path": "pharmacy", 1421 | "Fecha Actualizado": "10/6/2017", 1422 | "Status": "558", 1423 | "Fecha Obtenido": "10/6/2017" 1424 | }, 1425 | { 1426 | "Type": "Gasolina (abastos en barriles)", 1427 | "path": "barrel.gas", 1428 | "Fecha Actualizado": "10/6/2017", 1429 | "Status": "512,426", 1430 | "Fecha Obtenido": "10/6/2017" 1431 | }, 1432 | { 1433 | "Type": "Gasolineras", 1434 | "path": "gas", 1435 | "Fecha Actualizado": "10/6/2017", 1436 | "Status": "78.18%", 1437 | "Fecha Obtenido": "10/6/2017" 1438 | }, 1439 | { 1440 | "Type": "Hospitales Asistidos", 1441 | "path": "hospital", 1442 | "Fecha Actualizado": "10/6/2017", 1443 | "Status": "68", 1444 | "Fecha Obtenido": "10/6/2017" 1445 | }, 1446 | { 1447 | "Type": "Mascotas Refugiadas", 1448 | "path": "pet", 1449 | "Fecha Actualizado": "10/6/2017", 1450 | "Status": "216", 1451 | "Fecha Obtenido": "10/6/2017" 1452 | }, 1453 | { 1454 | "Type": "Puertos (capacidad operacional diaria)", 1455 | "Fecha Actualizado": "10/6/2017", 1456 | "Status": "", 1457 | "Fecha Obtenido": "10/6/2017" 1458 | }, 1459 | { 1460 | "Type": "Puertos Abiertos", 1461 | "path": "port", 1462 | "Fecha Actualizado": "10/6/2017", 1463 | "Status": "75.00%", 1464 | "Fecha Obtenido": "10/6/2017" 1465 | }, 1466 | { 1467 | "Type": "Refugiados", 1468 | "path": "refugee", 1469 | "Fecha Actualizado": "10/6/2017", 1470 | "Status": "8,349", 1471 | "Fecha Obtenido": "10/6/2017" 1472 | }, 1473 | { 1474 | "Type": "Refugios", 1475 | "path": "shelter", 1476 | "Fecha Actualizado": "10/6/2017", 1477 | "Status": "68", 1478 | "Fecha Obtenido": "10/6/2017" 1479 | }, 1480 | { 1481 | "Type": "Rutas AMA", 1482 | "path": "ama", 1483 | "Fecha Actualizado": "10/6/2017", 1484 | "Status": "75.00%", 1485 | "Fecha Obtenido": "10/6/2017" 1486 | }, 1487 | { 1488 | "Type": "Sucursales Bancarias", 1489 | "path": "bank", 1490 | "Fecha Actualizado": "10/6/2017", 1491 | "Status": "55.59%", 1492 | "Fecha Obtenido": "10/6/2017" 1493 | }, 1494 | { 1495 | "Type": "Supermercados", 1496 | "path": "supermarket", 1497 | "Fecha Actualizado": "10/6/2017", 1498 | "Status": "73.03%", 1499 | "Fecha Obtenido": "10/6/2017" 1500 | }, 1501 | { 1502 | "Type": "Telecomunicaciones", 1503 | "path": "telecomunication", 1504 | "Fecha Actualizado": "10/6/2017", 1505 | "Status": "42.00%", 1506 | "Fecha Obtenido": "10/6/2017" 1507 | }, 1508 | { 1509 | "Type": "Torres celular", 1510 | "path": "tower", 1511 | "Fecha Actualizado": "10/6/2017", 1512 | "Status": "24.09%", 1513 | "Fecha Obtenido": "10/6/2017" 1514 | }, 1515 | { 1516 | "Type": "Vuelos Comerciales", 1517 | "path": "flight", 1518 | "Fecha Actualizado": "10/6/2017", 1519 | "Status": "100.00%", 1520 | "Fecha Obtenido": "10/6/2017" 1521 | }, 1522 | { 1523 | "Type": "AAA", 1524 | "Fecha Actualizado": "10/7/2017", 1525 | "Status": "56.24%", 1526 | "Fecha Obtenido": "10/7/2017" 1527 | }, 1528 | { 1529 | "Type": "AAA Este", 1530 | "Fecha Actualizado": "10/8/2017", 1531 | "Status": "64.00%", 1532 | "Fecha Obtenido": "10/7/2017" 1533 | }, 1534 | { 1535 | "Type": "AAA Metro", 1536 | "Fecha Actualizado": "10/9/2017", 1537 | "Status": "65.00%", 1538 | "Fecha Obtenido": "10/7/2017" 1539 | }, 1540 | { 1541 | "Type": "AAA Norte", 1542 | "Fecha Actualizado": "10/10/2017", 1543 | "Status": "20.00%", 1544 | "Fecha Obtenido": "10/7/2017" 1545 | }, 1546 | { 1547 | "Type": "AAA Oeste", 1548 | "Fecha Actualizado": "10/11/2017", 1549 | "Status": "48.00%", 1550 | "Fecha Obtenido": "10/7/2017" 1551 | }, 1552 | { 1553 | "Type": "AAA Sur", 1554 | "Fecha Actualizado": "10/12/2017", 1555 | "Status": "78.00%", 1556 | "Fecha Obtenido": "10/7/2017" 1557 | }, 1558 | { 1559 | "Type": "AEE", 1560 | "path": "aee", 1561 | "Fecha Actualizado": "10/13/2017", 1562 | "Status": "11.70%", 1563 | "Fecha Obtenido": "10/7/2017" 1564 | }, 1565 | { 1566 | "Type": "Antenas Celular", 1567 | "path": "antenna", 1568 | "Fecha Actualizado": "10/14/2017", 1569 | "Status": "15.80%", 1570 | "Fecha Obtenido": "10/7/2017" 1571 | }, 1572 | { 1573 | "Type": "ATMs", 1574 | "path": "atms", 1575 | "Fecha Actualizado": "10/15/2017", 1576 | "Status": "612", 1577 | "Fecha Obtenido": "10/7/2017" 1578 | }, 1579 | { 1580 | "Type": "Centros de Dialisis Asistidos", 1581 | "path": "dialysis", 1582 | "Fecha Actualizado": "10/16/2017", 1583 | "Status": "46", 1584 | "Fecha Obtenido": "10/7/2017" 1585 | }, 1586 | { 1587 | "Type": "Comercios procesando PAN", 1588 | "path": "comercios.procesando.pan", 1589 | "Fecha Actualizado": "10/17/2017", 1590 | "Status": "490", 1591 | "Fecha Obtenido": "10/7/2017" 1592 | }, 1593 | { 1594 | "Type": "Contenedores", 1595 | "path": "container", 1596 | "Fecha Actualizado": "10/18/2017", 1597 | "Status": "88.07%", 1598 | "Fecha Obtenido": "10/7/2017" 1599 | }, 1600 | { 1601 | "Type": "Cooperativas", 1602 | "path": "cooperatives", 1603 | "Fecha Actualizado": "10/19/2017", 1604 | "Status": "145", 1605 | "Fecha Obtenido": "10/7/2017" 1606 | }, 1607 | { 1608 | "Type": "Correos", 1609 | "path": "goverment.mail", 1610 | "Fecha Actualizado": "10/20/2017", 1611 | "Status": "92.42%", 1612 | "Fecha Obtenido": "10/7/2017" 1613 | }, 1614 | { 1615 | "Type": "Diesel (abastos en barriles)", 1616 | "path": "barrel.diesel", 1617 | "Fecha Actualizado": "10/21/2017", 1618 | "Status": "521,354", 1619 | "Fecha Obtenido": "10/7/2017" 1620 | }, 1621 | { 1622 | "Type": "Farmacias", 1623 | "path": "pharmacy", 1624 | "Fecha Actualizado": "10/22/2017", 1625 | "Status": "558", 1626 | "Fecha Obtenido": "10/7/2017" 1627 | }, 1628 | { 1629 | "Type": "Gasolina (abastos en barriles)", 1630 | "path": "barrel.gas", 1631 | "Fecha Actualizado": "10/23/2017", 1632 | "Status": "545,935", 1633 | "Fecha Obtenido": "10/7/2017" 1634 | }, 1635 | { 1636 | "Type": "Gasolineras", 1637 | "path": "gas", 1638 | "Fecha Actualizado": "10/24/2017", 1639 | "Status": "78.18%", 1640 | "Fecha Obtenido": "10/7/2017" 1641 | }, 1642 | { 1643 | "Type": "Hospitales Asistidos", 1644 | "path": "hospital", 1645 | "Fecha Actualizado": "10/25/2017", 1646 | "Status": "66", 1647 | "Fecha Obtenido": "10/7/2017" 1648 | }, 1649 | { 1650 | "Type": "Mascotas Refugiadas", 1651 | "path": "pet", 1652 | "Fecha Actualizado": "10/26/2017", 1653 | "Status": "154", 1654 | "Fecha Obtenido": "10/7/2017" 1655 | }, 1656 | { 1657 | "Type": "Puertos Abiertos", 1658 | "path": "port", 1659 | "Fecha Actualizado": "10/27/2017", 1660 | "Status": "75.00%", 1661 | "Fecha Obtenido": "10/7/2017" 1662 | }, 1663 | { 1664 | "Type": "Refugiados", 1665 | "path": "refugee", 1666 | "Fecha Actualizado": "10/28/2017", 1667 | "Status": "7,442", 1668 | "Fecha Obtenido": "10/7/2017" 1669 | }, 1670 | { 1671 | "Type": "Refugios", 1672 | "path": "shelter", 1673 | "Fecha Actualizado": "10/29/2017", 1674 | "Status": "116", 1675 | "Fecha Obtenido": "10/7/2017" 1676 | }, 1677 | { 1678 | "Type": "Rutas AMA", 1679 | "path": "ama", 1680 | "Fecha Actualizado": "10/30/2017", 1681 | "Status": "75.00%", 1682 | "Fecha Obtenido": "10/7/2017" 1683 | }, 1684 | { 1685 | "Type": "Sucursales Bancarias", 1686 | "path": "bank", 1687 | "Fecha Actualizado": "10/31/2017", 1688 | "Status": "57.19%", 1689 | "Fecha Obtenido": "10/7/2017" 1690 | }, 1691 | { 1692 | "Type": "Supermercados", 1693 | "path": "supermarket", 1694 | "Fecha Actualizado": "11/1/2017", 1695 | "Status": "77.41%", 1696 | "Fecha Obtenido": "10/7/2017" 1697 | }, 1698 | { 1699 | "Type": "Telecomunicaciones", 1700 | "path": "telecomunication", 1701 | "Fecha Actualizado": "11/2/2017", 1702 | "Status": "44.00%", 1703 | "Fecha Obtenido": "10/7/2017" 1704 | }, 1705 | { 1706 | "Type": "Torres celular", 1707 | "path": "tower", 1708 | "Fecha Actualizado": "11/3/2017", 1709 | "Status": "24.09%", 1710 | "Fecha Obtenido": "10/7/2017" 1711 | }, 1712 | { 1713 | "Type": "Vuelos Comerciales", 1714 | "path": "flight", 1715 | "Fecha Actualizado": "11/4/2017", 1716 | "Status": "100.00%", 1717 | "Fecha Obtenido": "10/7/2017" 1718 | } 1719 | ] 1720 | 1721 | for obj in objects: 1722 | if obj.get('path'): 1723 | doc = { 1724 | 'label': normalize_label(obj['Type']), 1725 | 'path': obj['path'], 1726 | 'value': normalize_value(obj['Status']), 1727 | 'last_updated_at': normalize_last_updated(obj['Fecha Actualizado']), 1728 | 'imported_at': datetime.utcnow(), 1729 | 'created_at': datetime.strptime(obj['Fecha Obtenido'], '%m/%d/%Y') 1730 | } 1731 | 1732 | print(doc) 1733 | 1734 | db.stats.insert_one(doc) 1735 | --------------------------------------------------------------------------------