├── .dockerignore ├── .gitignore ├── .python-version ├── Dockerfile ├── LICENSE ├── Pipfile ├── Pipfile.lock ├── README.md ├── app ├── __init__.py ├── cmds.py ├── config.py ├── jobs.py └── routes.py ├── devops ├── entrypoint.sh └── redis.env ├── docker-compose.yml ├── logs └── .keep ├── notebooks └── redis.ipynb ├── rq_config.py └── tmp └── .keep /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.vscode 2 | **/.idea 3 | **/.coverages 4 | **/.pytest_cache 5 | **/*.md 6 | **/__pycache__ 7 | **/node_modules 8 | **/build 9 | **/.git 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | #.python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # 132 | tmp/* 133 | **.ipynb 134 | **.sqlite3 135 | .vscode/* 136 | logs/* -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.6.8 -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6.8 2 | 3 | ENV home_dir=/opt/redis_data/ 4 | RUN mkdir -p ${home_dir} 5 | WORKDIR ${home_dir} 6 | 7 | COPY Pipfile* ./ 8 | RUN pip install --upgrade pip && pip install pipenv 9 | RUN pipenv install --system --dev 10 | COPY ./ ./ 11 | 12 | EXPOSE 5000 13 | EXPOSE 8888 14 | 15 | ENTRYPOINT ["devops/entrypoint.sh"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Dmitry Polyakovsky 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | pylint = "*" 8 | 9 | [packages] 10 | jupyter = "*" 11 | bokeh = "*" 12 | ipyparallel = "*" 13 | pandas = "*" 14 | pyarrow = "*" 15 | requests = "*" 16 | # 17 | redis = "*" 18 | flask = "*" 19 | rq-dashboard = "*" 20 | flask-rq2 = "*" 21 | 22 | [requires] 23 | python_version = "3.6.8" 24 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "74456e9fe5f5814644f8acb116302b016811fe722a1046083fa1616ae6cf4ed3" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.6.8" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "appnope": { 20 | "hashes": [ 21 | "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0", 22 | "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71" 23 | ], 24 | "markers": "platform_system == 'Darwin'", 25 | "version": "==0.1.0" 26 | }, 27 | "arrow": { 28 | "hashes": [ 29 | "sha256:01a16d8a93eddf86a29237f32ae36b29c27f047e79312eb4df5d55fd5a2b3183", 30 | "sha256:e1a318a4c0b787833ae46302c02488b6eeef413c6a13324b3261ad320f21ec1e" 31 | ], 32 | "version": "==0.15.4" 33 | }, 34 | "attrs": { 35 | "hashes": [ 36 | "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", 37 | "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" 38 | ], 39 | "version": "==19.3.0" 40 | }, 41 | "backcall": { 42 | "hashes": [ 43 | "sha256:38ecd85be2c1e78f77fd91700c76e14667dc21e2713b63876c0eb901196e01e4", 44 | "sha256:bbbf4b1e5cd2bdb08f915895b51081c041bac22394fdfcfdfbe9f14b77c08bf2" 45 | ], 46 | "version": "==0.1.0" 47 | }, 48 | "bleach": { 49 | "hashes": [ 50 | "sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16", 51 | "sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa" 52 | ], 53 | "version": "==3.1.0" 54 | }, 55 | "bokeh": { 56 | "hashes": [ 57 | "sha256:c60d38a41a777b8147ee4134e6142cea8026b5eebf48149e370c44689869dce7" 58 | ], 59 | "index": "pypi", 60 | "version": "==1.4.0" 61 | }, 62 | "certifi": { 63 | "hashes": [ 64 | "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", 65 | "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" 66 | ], 67 | "version": "==2019.11.28" 68 | }, 69 | "chardet": { 70 | "hashes": [ 71 | "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", 72 | "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" 73 | ], 74 | "version": "==3.0.4" 75 | }, 76 | "click": { 77 | "hashes": [ 78 | "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", 79 | "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" 80 | ], 81 | "version": "==7.0" 82 | }, 83 | "croniter": { 84 | "hashes": [ 85 | "sha256:0d905dbe6f131a910fd3dde792f0129788cd2cb3a8048c5f7aaa212670b0cef2", 86 | "sha256:538adeb3a7f7816c3cdec6db974c441620d764c25ff4ed0146ee7296b8a50590" 87 | ], 88 | "version": "==0.3.30" 89 | }, 90 | "decorator": { 91 | "hashes": [ 92 | "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce", 93 | "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d" 94 | ], 95 | "version": "==4.4.1" 96 | }, 97 | "defusedxml": { 98 | "hashes": [ 99 | "sha256:6687150770438374ab581bb7a1b327a847dd9c5749e396102de3fad4e8a3ef93", 100 | "sha256:f684034d135af4c6cbb949b8a4d2ed61634515257a67299e5f940fbaa34377f5" 101 | ], 102 | "version": "==0.6.0" 103 | }, 104 | "entrypoints": { 105 | "hashes": [ 106 | "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", 107 | "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" 108 | ], 109 | "version": "==0.3" 110 | }, 111 | "flask": { 112 | "hashes": [ 113 | "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", 114 | "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6" 115 | ], 116 | "index": "pypi", 117 | "version": "==1.1.1" 118 | }, 119 | "flask-rq2": { 120 | "hashes": [ 121 | "sha256:3ef6395065255447f8e1516ccca24858ba87da1d71a6975e0e3b55256bf04967", 122 | "sha256:abe1e52d3b98abe37e85830a614ba6af864516f1b6cf2229f352f8500eafc5fd" 123 | ], 124 | "index": "pypi", 125 | "version": "==18.3" 126 | }, 127 | "idna": { 128 | "hashes": [ 129 | "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", 130 | "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" 131 | ], 132 | "version": "==2.8" 133 | }, 134 | "importlib-metadata": { 135 | "hashes": [ 136 | "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45", 137 | "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f" 138 | ], 139 | "markers": "python_version < '3.8'", 140 | "version": "==1.3.0" 141 | }, 142 | "ipykernel": { 143 | "hashes": [ 144 | "sha256:1a7def9c986f1ee018c1138d16951932d4c9d4da01dad45f9d34e9899565a22f", 145 | "sha256:b368ad13edb71fa2db367a01e755a925d7f75ed5e09fbd3f06c85e7a8ef108a8" 146 | ], 147 | "version": "==5.1.3" 148 | }, 149 | "ipyparallel": { 150 | "hashes": [ 151 | "sha256:2acbffcbd6da955b47ec7befb320dcad1788cc146cfc7abfa6d1c74436d74d38", 152 | "sha256:76c7b028962b0ba762e4e45b450ee3a4353e7221526a8af812e817d7ef6ac065" 153 | ], 154 | "index": "pypi", 155 | "version": "==6.2.4" 156 | }, 157 | "ipython": { 158 | "hashes": [ 159 | "sha256:c66c7e27239855828a764b1e8fc72c24a6f4498a2637572094a78c5551fb9d51", 160 | "sha256:f186b01b36609e0c5d0de27c7ef8e80c990c70478f8c880863004b3489a9030e" 161 | ], 162 | "markers": "python_version >= '3.3'", 163 | "version": "==7.10.1" 164 | }, 165 | "ipython-genutils": { 166 | "hashes": [ 167 | "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8", 168 | "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8" 169 | ], 170 | "version": "==0.2.0" 171 | }, 172 | "ipywidgets": { 173 | "hashes": [ 174 | "sha256:13ffeca438e0c0f91ae583dc22f50379b9d6b28390ac7be8b757140e9a771516", 175 | "sha256:e945f6e02854a74994c596d9db83444a1850c01648f1574adf144fbbabe05c97" 176 | ], 177 | "version": "==7.5.1" 178 | }, 179 | "itsdangerous": { 180 | "hashes": [ 181 | "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", 182 | "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" 183 | ], 184 | "version": "==1.1.0" 185 | }, 186 | "jedi": { 187 | "hashes": [ 188 | "sha256:786b6c3d80e2f06fd77162a07fed81b8baa22dde5d62896a790a331d6ac21a27", 189 | "sha256:ba859c74fa3c966a22f2aeebe1b74ee27e2a462f56d3f5f7ca4a59af61bfe42e" 190 | ], 191 | "version": "==0.15.1" 192 | }, 193 | "jinja2": { 194 | "hashes": [ 195 | "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", 196 | "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" 197 | ], 198 | "version": "==2.10.3" 199 | }, 200 | "jsonschema": { 201 | "hashes": [ 202 | "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163", 203 | "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a" 204 | ], 205 | "version": "==3.2.0" 206 | }, 207 | "jupyter": { 208 | "hashes": [ 209 | "sha256:3e1f86076bbb7c8c207829390305a2b1fe836d471ed54be66a3b8c41e7f46cc7", 210 | "sha256:5b290f93b98ffbc21c0c7e749f054b3267782166d72fa5e3ed1ed4eaf34a2b78", 211 | "sha256:d9dc4b3318f310e34c82951ea5d6683f67bed7def4b259fafbfe4f1beb1d8e5f" 212 | ], 213 | "index": "pypi", 214 | "version": "==1.0.0" 215 | }, 216 | "jupyter-client": { 217 | "hashes": [ 218 | "sha256:60e6faec1031d63df57f1cc671ed673dced0ed420f4377ea33db37b1c188b910", 219 | "sha256:d0c077c9aaa4432ad485e7733e4d91e48f87b4f4bab7d283d42bb24cbbba0a0f" 220 | ], 221 | "version": "==5.3.4" 222 | }, 223 | "jupyter-console": { 224 | "hashes": [ 225 | "sha256:308ce876354924fb6c540b41d5d6d08acfc946984bf0c97777c1ddcb42e0b2f5", 226 | "sha256:cc80a97a5c389cbd30252ffb5ce7cefd4b66bde98219edd16bf5cb6f84bb3568" 227 | ], 228 | "version": "==6.0.0" 229 | }, 230 | "jupyter-core": { 231 | "hashes": [ 232 | "sha256:464769f7387d7a62a2403d067f1ddc616655b7f77f5d810c0dd62cb54bfd0fb9", 233 | "sha256:a183e0ec2e8f6adddf62b0a3fc6a2237e3e0056d381e536d3e7c7ecc3067e244" 234 | ], 235 | "version": "==4.6.1" 236 | }, 237 | "markupsafe": { 238 | "hashes": [ 239 | "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", 240 | "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", 241 | "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", 242 | "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", 243 | "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", 244 | "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", 245 | "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", 246 | "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", 247 | "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", 248 | "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", 249 | "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", 250 | "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", 251 | "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", 252 | "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", 253 | "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", 254 | "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", 255 | "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", 256 | "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", 257 | "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", 258 | "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", 259 | "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", 260 | "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", 261 | "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", 262 | "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", 263 | "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", 264 | "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", 265 | "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", 266 | "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" 267 | ], 268 | "version": "==1.1.1" 269 | }, 270 | "mistune": { 271 | "hashes": [ 272 | "sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e", 273 | "sha256:88a1051873018da288eee8538d476dffe1262495144b33ecb586c4ab266bb8d4" 274 | ], 275 | "version": "==0.8.4" 276 | }, 277 | "more-itertools": { 278 | "hashes": [ 279 | "sha256:b84b238cce0d9adad5ed87e745778d20a3f8487d0f0cb8b8a586816c7496458d", 280 | "sha256:c833ef592a0324bcc6a60e48440da07645063c453880c9477ceb22490aec1564" 281 | ], 282 | "version": "==8.0.2" 283 | }, 284 | "nbconvert": { 285 | "hashes": [ 286 | "sha256:21fb48e700b43e82ba0e3142421a659d7739b65568cc832a13976a77be16b523", 287 | "sha256:f0d6ec03875f96df45aa13e21fd9b8450c42d7e1830418cccc008c0df725fcee" 288 | ], 289 | "version": "==5.6.1" 290 | }, 291 | "nbformat": { 292 | "hashes": [ 293 | "sha256:b9a0dbdbd45bb034f4f8893cafd6f652ea08c8c1674ba83f2dc55d3955743b0b", 294 | "sha256:f7494ef0df60766b7cabe0a3651556345a963b74dbc16bc7c18479041170d402" 295 | ], 296 | "version": "==4.4.0" 297 | }, 298 | "notebook": { 299 | "hashes": [ 300 | "sha256:399a4411e171170173344761e7fd4491a3625659881f76ce47c50231ed714d9b", 301 | "sha256:f67d76a68b1074a91693e95dea903ea01fd02be7c9fac5a4b870b8475caed805" 302 | ], 303 | "version": "==6.0.2" 304 | }, 305 | "numpy": { 306 | "hashes": [ 307 | "sha256:0a7a1dd123aecc9f0076934288ceed7fd9a81ba3919f11a855a7887cbe82a02f", 308 | "sha256:0c0763787133dfeec19904c22c7e358b231c87ba3206b211652f8cbe1241deb6", 309 | "sha256:3d52298d0be333583739f1aec9026f3b09fdfe3ddf7c7028cb16d9d2af1cca7e", 310 | "sha256:43bb4b70585f1c2d153e45323a886839f98af8bfa810f7014b20be714c37c447", 311 | "sha256:475963c5b9e116c38ad7347e154e5651d05a2286d86455671f5b1eebba5feb76", 312 | "sha256:64874913367f18eb3013b16123c9fed113962e75d809fca5b78ebfbb73ed93ba", 313 | "sha256:683828e50c339fc9e68720396f2de14253992c495fdddef77a1e17de55f1decc", 314 | "sha256:6ca4000c4a6f95a78c33c7dadbb9495c10880be9c89316aa536eac359ab820ae", 315 | "sha256:75fd817b7061f6378e4659dd792c84c0b60533e867f83e0d1e52d5d8e53df88c", 316 | "sha256:7d81d784bdbed30137aca242ab307f3e65c8d93f4c7b7d8f322110b2e90177f9", 317 | "sha256:8d0af8d3664f142414fd5b15cabfd3b6cc3ef242a3c7a7493257025be5a6955f", 318 | "sha256:9679831005fb16c6df3dd35d17aa31dc0d4d7573d84f0b44cc481490a65c7725", 319 | "sha256:a8f67ebfae9f575d85fa859b54d3bdecaeece74e3274b0b5c5f804d7ca789fe1", 320 | "sha256:acbf5c52db4adb366c064d0b7c7899e3e778d89db585feadd23b06b587d64761", 321 | "sha256:ada4805ed51f5bcaa3a06d3dd94939351869c095e30a2b54264f5a5004b52170", 322 | "sha256:c7354e8f0eca5c110b7e978034cd86ed98a7a5ffcf69ca97535445a595e07b8e", 323 | "sha256:e2e9d8c87120ba2c591f60e32736b82b67f72c37ba88a4c23c81b5b8fa49c018", 324 | "sha256:e467c57121fe1b78a8f68dd9255fbb3bb3f4f7547c6b9e109f31d14569f490c3", 325 | "sha256:ede47b98de79565fcd7f2decb475e2dcc85ee4097743e551fe26cfc7eb3ff143", 326 | "sha256:f58913e9227400f1395c7b800503ebfdb0772f1c33ff8cb4d6451c06cabdf316", 327 | "sha256:fe39f5fd4103ec4ca3cb8600b19216cd1ff316b4990f4c0b6057ad982c0a34d5" 328 | ], 329 | "version": "==1.17.4" 330 | }, 331 | "packaging": { 332 | "hashes": [ 333 | "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", 334 | "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" 335 | ], 336 | "version": "==19.2" 337 | }, 338 | "pandas": { 339 | "hashes": [ 340 | "sha256:00dff3a8e337f5ed7ad295d98a31821d3d0fe7792da82d78d7fd79b89c03ea9d", 341 | "sha256:22361b1597c8c2ffd697aa9bf85423afa9e1fcfa6b1ea821054a244d5f24d75e", 342 | "sha256:255920e63850dc512ce356233081098554d641ba99c3767dde9e9f35630f994b", 343 | "sha256:26382aab9c119735908d94d2c5c08020a4a0a82969b7e5eefb92f902b3b30ad7", 344 | "sha256:33970f4cacdd9a0ddb8f21e151bfb9f178afb7c36eb7c25b9094c02876f385c2", 345 | "sha256:4545467a637e0e1393f7d05d61dace89689ad6d6f66f267f86fff737b702cce9", 346 | "sha256:52da74df8a9c9a103af0a72c9d5fdc8e0183a90884278db7f386b5692a2220a4", 347 | "sha256:61741f5aeb252f39c3031d11405305b6d10ce663c53bc3112705d7ad66c013d0", 348 | "sha256:6a3ac2c87e4e32a969921d1428525f09462770c349147aa8e9ab95f88c71ec71", 349 | "sha256:7458c48e3d15b8aaa7d575be60e1e4dd70348efcd9376656b72fecd55c59a4c3", 350 | "sha256:78bf638993219311377ce9836b3dc05f627a666d0dbc8cec37c0ff3c9ada673b", 351 | "sha256:8153705d6545fd9eb6dd2bc79301bff08825d2e2f716d5dced48daafc2d0b81f", 352 | "sha256:975c461accd14e89d71772e89108a050fa824c0b87a67d34cedf245f6681fc17", 353 | "sha256:9962957a27bfb70ab64103d0a7b42fa59c642fb4ed4cb75d0227b7bb9228535d", 354 | "sha256:adc3d3a3f9e59a38d923e90e20c4922fc62d1e5a03d083440468c6d8f3f1ae0a", 355 | "sha256:bbe3eb765a0b1e578833d243e2814b60c825b7fdbf4cdfe8e8aae8a08ed56ecf", 356 | "sha256:df8864824b1fe488cf778c3650ee59c3a0d8f42e53707de167ba6b4f7d35f133", 357 | "sha256:e45055c30a608076e31a9fcd780a956ed3b1fa20db61561b8d88b79259f526f7", 358 | "sha256:ee50c2142cdcf41995655d499a157d0a812fce55c97d9aad13bc1eef837ed36c" 359 | ], 360 | "index": "pypi", 361 | "version": "==0.25.3" 362 | }, 363 | "pandocfilters": { 364 | "hashes": [ 365 | "sha256:b3dd70e169bb5449e6bc6ff96aea89c5eea8c5f6ab5e207fc2f521a2cf4a0da9" 366 | ], 367 | "version": "==1.4.2" 368 | }, 369 | "parso": { 370 | "hashes": [ 371 | "sha256:63854233e1fadb5da97f2744b6b24346d2750b85965e7e399bec1620232797dc", 372 | "sha256:666b0ee4a7a1220f65d367617f2cd3ffddff3e205f3f16a0284df30e774c2a9c" 373 | ], 374 | "version": "==0.5.1" 375 | }, 376 | "pexpect": { 377 | "hashes": [ 378 | "sha256:2094eefdfcf37a1fdbfb9aa090862c1a4878e5c7e0e7e7088bdb511c558e5cd1", 379 | "sha256:9e2c1fd0e6ee3a49b28f95d4b33bc389c89b20af6a1255906e90ff1262ce62eb" 380 | ], 381 | "markers": "sys_platform != 'win32'", 382 | "version": "==4.7.0" 383 | }, 384 | "pickleshare": { 385 | "hashes": [ 386 | "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca", 387 | "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56" 388 | ], 389 | "version": "==0.7.5" 390 | }, 391 | "pillow": { 392 | "hashes": [ 393 | "sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031", 394 | "sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71", 395 | "sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c", 396 | "sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340", 397 | "sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa", 398 | "sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b", 399 | "sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573", 400 | "sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e", 401 | "sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab", 402 | "sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9", 403 | "sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e", 404 | "sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291", 405 | "sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12", 406 | "sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871", 407 | "sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281", 408 | "sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08", 409 | "sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41", 410 | "sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2", 411 | "sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5", 412 | "sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb", 413 | "sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547", 414 | "sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75", 415 | "sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9", 416 | "sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1", 417 | "sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a", 418 | "sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96", 419 | "sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132", 420 | "sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a", 421 | "sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5", 422 | "sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0" 423 | ], 424 | "version": "==6.2.1" 425 | }, 426 | "prometheus-client": { 427 | "hashes": [ 428 | "sha256:71cd24a2b3eb335cb800c7159f423df1bd4dcd5171b234be15e3f31ec9f622da" 429 | ], 430 | "version": "==0.7.1" 431 | }, 432 | "prompt-toolkit": { 433 | "hashes": [ 434 | "sha256:46642344ce457641f28fc9d1c9ca939b63dadf8df128b86f1b9860e59c73a5e4", 435 | "sha256:e7f8af9e3d70f514373bf41aa51bc33af12a6db3f71461ea47fea985defb2c31", 436 | "sha256:f15af68f66e664eaa559d4ac8a928111eebd5feda0c11738b5998045224829db" 437 | ], 438 | "version": "==2.0.10" 439 | }, 440 | "ptyprocess": { 441 | "hashes": [ 442 | "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0", 443 | "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f" 444 | ], 445 | "markers": "os_name != 'nt'", 446 | "version": "==0.6.0" 447 | }, 448 | "pyarrow": { 449 | "hashes": [ 450 | "sha256:030d67418b129eb14a1c1f1af06b1a48c8074005d704789725ea6f5addaf3b26", 451 | "sha256:13f921560bac5ad46b17513696e38fede0c0e92ba750c7b350c0b231815bb706", 452 | "sha256:14dbc00edd14133c15d62c8d6c566a82a7497b077f253fc0c2dad62c7f85beaa", 453 | "sha256:17cda6ba594acf5a72058dd2e5ca2586fe8781fc8d20bd750a3b7c66c8b274b2", 454 | "sha256:1f3934b2add6839844443c1ac0eba64e14b2b8253563574d45d6831851b11d47", 455 | "sha256:2964a3fe09fbe704160734d00bef7b023699dc6a603dc8eb889b095effc464db", 456 | "sha256:364806e26769ca20a79b1ead301c7ce28fd0534eb6d411d441053288d7e45817", 457 | "sha256:41cf5ed34012c43b4ceeeeb2534e3454c77e852bc9175d2e506b45bad132db49", 458 | "sha256:4f0276e258065c82dcb7edfc28c343ccad15da02b25e57e7c60ceb80e3f7268b", 459 | "sha256:4fa03d2bc725e948f361a8ce7de271e39d90130ee3a3375793ac241b452c5bfa", 460 | "sha256:5a07222b80ae36219c558cb8875e7e346f779d0862ae277c68899db879cf5cd7", 461 | "sha256:5f6026673ceaa037cb41fbe86ce7ea6483cfdc91e51dea929fbbf81883a73d96", 462 | "sha256:7ad074690ba38313067bf3bbda1258966d38e2037c035d08b9ffe3cce07747a5", 463 | "sha256:87a2324a6e41faff3a482dbfc54a1f51bbf2d7da39ee728ec73869e2ef892a97", 464 | "sha256:b508b860486f75bcfeab72b98b4d8caa3a1517e5b7a9b3adcd5bc4539bff8a1a", 465 | "sha256:bc7200f7a97aea7301f61cd616b33069d1098e6d9178db6a34ccd43ea9223f53", 466 | "sha256:c70f7d0032be960d8dbd32661a9de062af184f411400ea2f4a13883ca11b0b1f", 467 | "sha256:f5af4cd64c774693af560576a6b8039d165596b1921031ca5d739bd2e7e0554b" 468 | ], 469 | "index": "pypi", 470 | "version": "==0.15.1" 471 | }, 472 | "pygments": { 473 | "hashes": [ 474 | "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b", 475 | "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe" 476 | ], 477 | "version": "==2.5.2" 478 | }, 479 | "pyparsing": { 480 | "hashes": [ 481 | "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", 482 | "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" 483 | ], 484 | "version": "==2.4.5" 485 | }, 486 | "pyrsistent": { 487 | "hashes": [ 488 | "sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b" 489 | ], 490 | "version": "==0.15.6" 491 | }, 492 | "python-dateutil": { 493 | "hashes": [ 494 | "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", 495 | "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" 496 | ], 497 | "version": "==2.8.1" 498 | }, 499 | "pytz": { 500 | "hashes": [ 501 | "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", 502 | "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" 503 | ], 504 | "version": "==2019.3" 505 | }, 506 | "pyyaml": { 507 | "hashes": [ 508 | "sha256:0e7f69397d53155e55d10ff68fdfb2cf630a35e6daf65cf0bdeaf04f127c09dc", 509 | "sha256:2e9f0b7c5914367b0916c3c104a024bb68f269a486b9d04a2e8ac6f6597b7803", 510 | "sha256:35ace9b4147848cafac3db142795ee42deebe9d0dad885ce643928e88daebdcc", 511 | "sha256:38a4f0d114101c58c0f3a88aeaa44d63efd588845c5a2df5290b73db8f246d15", 512 | "sha256:483eb6a33b671408c8529106df3707270bfacb2447bf8ad856a4b4f57f6e3075", 513 | "sha256:4b6be5edb9f6bb73680f5bf4ee08ff25416d1400fbd4535fe0069b2994da07cd", 514 | "sha256:7f38e35c00e160db592091751d385cd7b3046d6d51f578b29943225178257b31", 515 | "sha256:8100c896ecb361794d8bfdb9c11fce618c7cf83d624d73d5ab38aef3bc82d43f", 516 | "sha256:c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c", 517 | "sha256:e4c015484ff0ff197564917b4b4246ca03f411b9bd7f16e02a2f586eb48b6d04", 518 | "sha256:ebc4ed52dcc93eeebeae5cf5deb2ae4347b3a81c3fa12b0b8c976544829396a4" 519 | ], 520 | "version": "==5.2" 521 | }, 522 | "pyzmq": { 523 | "hashes": [ 524 | "sha256:01b588911714a6696283de3904f564c550c9e12e8b4995e173f1011755e01086", 525 | "sha256:0573b9790aa26faff33fba40f25763657271d26f64bffb55a957a3d4165d6098", 526 | "sha256:0fa82b9fc3334478be95a5566f35f23109f763d1669bb762e3871a8fa2a4a037", 527 | "sha256:1e59b7b19396f26e360f41411a5d4603356d18871049cd7790f1a7d18f65fb2c", 528 | "sha256:2a294b4f44201bb21acc2c1a17ff87fbe57b82060b10ddb00ac03e57f3d7fcfa", 529 | "sha256:355b38d7dd6f884b8ee9771f59036bcd178d98539680c4f87e7ceb2c6fd057b6", 530 | "sha256:4b73d20aec63933bbda7957e30add233289d86d92a0bb9feb3f4746376f33527", 531 | "sha256:4ec47f2b50bdb97df58f1697470e5c58c3c5109289a623e30baf293481ff0166", 532 | "sha256:5541dc8cad3a8486d58bbed076cb113b65b5dd6b91eb94fb3e38a3d1d3022f20", 533 | "sha256:6fca7d11310430e751f9832257866a122edf9d7b635305c5d8c51f74a5174d3d", 534 | "sha256:7369656f89878455a5bcd5d56ca961884f5d096268f71c0750fc33d6732a25e5", 535 | "sha256:75d73ee7ca4b289a2a2dfe0e6bd8f854979fc13b3fe4ebc19381be3b04e37a4a", 536 | "sha256:80c928d5adcfa12346b08d31360988d843b54b94154575cccd628f1fe91446bc", 537 | "sha256:83ce18b133dc7e6789f64cb994e7376c5aa6b4aeced993048bf1d7f9a0fe6d3a", 538 | "sha256:8b8498ceee33a7023deb2f3db907ca41d6940321e282297327a9be41e3983792", 539 | "sha256:8c69a6cbfa94da29a34f6b16193e7c15f5d3220cb772d6d17425ff3faa063a6d", 540 | "sha256:8ff946b20d13a99dc5c21cb76f4b8b253eeddf3eceab4218df8825b0c65ab23d", 541 | "sha256:972d723a36ab6a60b7806faa5c18aa3c080b7d046c407e816a1d8673989e2485", 542 | "sha256:a6c9c42bbdba3f9c73aedbb7671815af1943ae8073e532c2b66efb72f39f4165", 543 | "sha256:aa3872f2ebfc5f9692ef8957fe69abe92d905a029c0608e45ebfcd451ad30ab5", 544 | "sha256:cf08435b14684f7f2ca2df32c9df38a79cdc17c20dc461927789216cb43d8363", 545 | "sha256:d30db4566177a6205ed1badb8dbbac3c043e91b12a2db5ef9171b318c5641b75", 546 | "sha256:d5ac84f38575a601ab20c1878818ffe0d09eb51d6cb8511b636da46d0fd8949a", 547 | "sha256:e37f22eb4bfbf69cd462c7000616e03b0cdc1b65f2d99334acad36ea0e4ddf6b", 548 | "sha256:e6549dd80de7b23b637f586217a4280facd14ac01e9410a037a13854a6977299", 549 | "sha256:ed6205ca0de035f252baa0fd26fdd2bc8a8f633f92f89ca866fd423ff26c6f25", 550 | "sha256:efdde21febb9b5d7a8e0b87ea2549d7e00fda1936459cfb27fb6fca0c36af6c1", 551 | "sha256:f4e72646bfe79ff3adbf1314906bbd2d67ef9ccc71a3a98b8b2ccbcca0ab7bec" 552 | ], 553 | "version": "==18.1.1" 554 | }, 555 | "qtconsole": { 556 | "hashes": [ 557 | "sha256:4de25b8895957d23ceacf2526b6f0a76da4e60e60115611930d387c853f3cb08", 558 | "sha256:654f423662e7dfe6a9b26fac8ec76aedcf742c339909ac49f1f0c1a1b744bcd1" 559 | ], 560 | "version": "==4.6.0" 561 | }, 562 | "redis": { 563 | "hashes": [ 564 | "sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62", 565 | "sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2" 566 | ], 567 | "index": "pypi", 568 | "version": "==3.3.11" 569 | }, 570 | "requests": { 571 | "hashes": [ 572 | "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", 573 | "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" 574 | ], 575 | "index": "pypi", 576 | "version": "==2.22.0" 577 | }, 578 | "rq": { 579 | "hashes": [ 580 | "sha256:2798d26a7b850e759f23f69695a389d676a9c08f2c14f96f0d34d9648c9d5616", 581 | "sha256:4f27c6a690d1bd02b9157e615d8819555b9b359c0c4ec8ff0013d160c31b40bb" 582 | ], 583 | "version": "==1.1.0" 584 | }, 585 | "rq-dashboard": { 586 | "hashes": [ 587 | "sha256:95163f0a8413e6be277d60eada0125f1c4b230577c3fa538a31a77a2cb556f8a" 588 | ], 589 | "index": "pypi", 590 | "version": "==0.6.1" 591 | }, 592 | "rq-scheduler": { 593 | "hashes": [ 594 | "sha256:06038a42d33d653f89d534ba3bb95694b9d82b39fdd17c6ac6d9bf77d1acdefb", 595 | "sha256:9f9f68d0a4749c83f023d903e148b81da2191229e25ac644a9ff9d6eac31bff4" 596 | ], 597 | "version": "==0.9.1" 598 | }, 599 | "send2trash": { 600 | "hashes": [ 601 | "sha256:60001cc07d707fe247c94f74ca6ac0d3255aabcb930529690897ca2a39db28b2", 602 | "sha256:f1691922577b6fa12821234aeb57599d887c4900b9ca537948d2dac34aea888b" 603 | ], 604 | "version": "==1.5.0" 605 | }, 606 | "six": { 607 | "hashes": [ 608 | "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", 609 | "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" 610 | ], 611 | "version": "==1.13.0" 612 | }, 613 | "terminado": { 614 | "hashes": [ 615 | "sha256:4804a774f802306a7d9af7322193c5390f1da0abb429e082a10ef1d46e6fb2c2", 616 | "sha256:a43dcb3e353bc680dd0783b1d9c3fc28d529f190bc54ba9a229f72fe6e7a54d7" 617 | ], 618 | "version": "==0.8.3" 619 | }, 620 | "testpath": { 621 | "hashes": [ 622 | "sha256:60e0a3261c149755f4399a1fff7d37523179a70fdc3abdf78de9fc2604aeec7e", 623 | "sha256:bfcf9411ef4bf3db7579063e0546938b1edda3d69f4e1fb8756991f5951f85d4" 624 | ], 625 | "version": "==0.4.4" 626 | }, 627 | "tornado": { 628 | "hashes": [ 629 | "sha256:349884248c36801afa19e342a77cc4458caca694b0eda633f5878e458a44cb2c", 630 | "sha256:398e0d35e086ba38a0427c3b37f4337327231942e731edaa6e9fd1865bbd6f60", 631 | "sha256:4e73ef678b1a859f0cb29e1d895526a20ea64b5ffd510a2307b5998c7df24281", 632 | "sha256:559bce3d31484b665259f50cd94c5c28b961b09315ccd838f284687245f416e5", 633 | "sha256:abbe53a39734ef4aba061fca54e30c6b4639d3e1f59653f0da37a0003de148c7", 634 | "sha256:c845db36ba616912074c5b1ee897f8e0124df269468f25e4fe21fe72f6edd7a9", 635 | "sha256:c9399267c926a4e7c418baa5cbe91c7d1cf362d505a1ef898fde44a07c9dd8a5" 636 | ], 637 | "version": "==6.0.3" 638 | }, 639 | "traitlets": { 640 | "hashes": [ 641 | "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44", 642 | "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7" 643 | ], 644 | "version": "==4.3.3" 645 | }, 646 | "urllib3": { 647 | "hashes": [ 648 | "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", 649 | "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" 650 | ], 651 | "version": "==1.25.7" 652 | }, 653 | "wcwidth": { 654 | "hashes": [ 655 | "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", 656 | "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" 657 | ], 658 | "version": "==0.1.7" 659 | }, 660 | "webencodings": { 661 | "hashes": [ 662 | "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", 663 | "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" 664 | ], 665 | "version": "==0.5.1" 666 | }, 667 | "werkzeug": { 668 | "hashes": [ 669 | "sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7", 670 | "sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4" 671 | ], 672 | "version": "==0.16.0" 673 | }, 674 | "widgetsnbextension": { 675 | "hashes": [ 676 | "sha256:079f87d87270bce047512400efd70238820751a11d2d8cb137a5a5bdbaf255c7", 677 | "sha256:bd314f8ceb488571a5ffea6cc5b9fc6cba0adaf88a9d2386b93a489751938bcd" 678 | ], 679 | "version": "==3.5.1" 680 | }, 681 | "zipp": { 682 | "hashes": [ 683 | "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", 684 | "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" 685 | ], 686 | "version": "==0.6.0" 687 | } 688 | }, 689 | "develop": { 690 | "astroid": { 691 | "hashes": [ 692 | "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", 693 | "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42" 694 | ], 695 | "version": "==2.3.3" 696 | }, 697 | "isort": { 698 | "hashes": [ 699 | "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", 700 | "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" 701 | ], 702 | "version": "==4.3.21" 703 | }, 704 | "lazy-object-proxy": { 705 | "hashes": [ 706 | "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", 707 | "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", 708 | "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", 709 | "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", 710 | "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", 711 | "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", 712 | "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", 713 | "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", 714 | "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", 715 | "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", 716 | "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", 717 | "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", 718 | "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", 719 | "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", 720 | "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", 721 | "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", 722 | "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", 723 | "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", 724 | "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", 725 | "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", 726 | "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" 727 | ], 728 | "version": "==1.4.3" 729 | }, 730 | "mccabe": { 731 | "hashes": [ 732 | "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", 733 | "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" 734 | ], 735 | "version": "==0.6.1" 736 | }, 737 | "pylint": { 738 | "hashes": [ 739 | "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd", 740 | "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4" 741 | ], 742 | "index": "pypi", 743 | "version": "==2.4.4" 744 | }, 745 | "six": { 746 | "hashes": [ 747 | "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", 748 | "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" 749 | ], 750 | "version": "==1.13.0" 751 | }, 752 | "typed-ast": { 753 | "hashes": [ 754 | "sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161", 755 | "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", 756 | "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", 757 | "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", 758 | "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", 759 | "sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47", 760 | "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", 761 | "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", 762 | "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", 763 | "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", 764 | "sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2", 765 | "sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e", 766 | "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", 767 | "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", 768 | "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", 769 | "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", 770 | "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", 771 | "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", 772 | "sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66", 773 | "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" 774 | ], 775 | "markers": "implementation_name == 'cpython' and python_version < '3.8'", 776 | "version": "==1.4.0" 777 | }, 778 | "wrapt": { 779 | "hashes": [ 780 | "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" 781 | ], 782 | "version": "==1.11.2" 783 | } 784 | } 785 | } 786 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # using Redis for data science and data engineering 2 | 3 | * Install Docker and Docker Compose https://docs.docker.com/compose/install/ 4 | * Clone this repo 5 | * Create Github token via https://github.com/settings/tokens. 6 | * In `~/.bash_profile` set `export GITHUB_TOKEN="token-here"`. Need it to deal with Github rate limiting https://developer.github.com/v3/#rate-limiting 7 | * `docker-compose up --build -d --scale worker=2` 8 | * Browse to http://localhost:5000/github to start data process 9 | * Browse to http://localhost:5000/rq/ to view jobs running 10 | * Browse to http://localhost:8888/ to use Jupyter Notebooks -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from flask import Flask 3 | from flask_rq2 import RQ 4 | import rq_dashboard 5 | 6 | APP = Flask(__name__) 7 | APP.config.from_pyfile('config.py') 8 | 9 | #APP.config.from_object(rq_dashboard.default_settings) 10 | APP.register_blueprint(rq_dashboard.blueprint, url_prefix='/rq') 11 | RQ_CLIENT = RQ(APP) 12 | 13 | from app import routes, cmds 14 | # logging.info('initialized app') 15 | -------------------------------------------------------------------------------- /app/cmds.py: -------------------------------------------------------------------------------- 1 | import click 2 | from flask.cli import AppGroup 3 | from . import APP, jobs 4 | 5 | 6 | AGD = AppGroup('demo') 7 | APP.cli.add_command(AGD) 8 | 9 | 10 | @AGD.command() 11 | def github(): 12 | ''' queue github jobs ''' 13 | job = jobs.github_users.queue() 14 | click.echo(click.style(job.id, bold=True, fg='blue')) 15 | -------------------------------------------------------------------------------- /app/config.py: -------------------------------------------------------------------------------- 1 | ''' base config ''' 2 | import os 3 | from logging.config import dictConfig 4 | import redis 5 | 6 | HOME_DIR = os.environ.get('home_dir') 7 | LOGS_DIR = f'{HOME_DIR}logs/' 8 | APP_ENV = os.environ.get("APP_ENV") 9 | SECRET_KEY = 'foobar' 10 | 11 | REDIS_HOST = os.environ.get('REDIS_HOST') 12 | REDIS_DB = 0 13 | 14 | RQ_DASHBOARD_REDIS_URL = f'redis://{REDIS_HOST}:6379/1' 15 | RQ_REDIS_URL = f'redis://{REDIS_HOST}:6379/1' 16 | RQ_QUEUES = ['high', 'default', 'low'] 17 | 18 | MAX_GITHUB_REQUESTS = 10 19 | GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN') 20 | 21 | dictConfig({ 22 | 'version': 1, 23 | 'formatters': {'default': { 24 | 'format': '{timestamp:%(asctime)s, level:%(levelname)s, module:%(module)s, %(message)s}', 25 | }}, 26 | 'handlers': { 27 | 'file': { 28 | 'level': 'INFO', 29 | 'class': 'logging.handlers.TimedRotatingFileHandler', 30 | 'formatter': 'default', 31 | 'filename': f'{LOGS_DIR}redis-data-{APP_ENV}.log', 32 | 'when': 'D', 33 | 'interval': 1, 34 | 'backupCount': 7 35 | } 36 | }, 37 | 'root': { 38 | 'level': 'INFO', 39 | 'handlers': ['file'] 40 | } 41 | }) 42 | -------------------------------------------------------------------------------- /app/jobs.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import pandas as pd 3 | import requests 4 | import redis 5 | from . import APP, RQ_CLIENT 6 | 7 | 8 | REDIS_CLIENT = redis.Redis(host=APP.config.get('REDIS_HOST'), db=0) 9 | GH_HEADERS = {'Authorization': 'token ' + APP.config.get('GITHUB_TOKEN', '')} 10 | 11 | 12 | # https://stackoverflow.com/questions/17622439/how-to-use-github-api-token-in-python-for-requesting 13 | # https://github.com/pandas-dev/pandas/issues/10526 14 | @RQ_CLIENT.job() 15 | def github_users(since=0, counter=0): 16 | ''' get users data from github api ''' 17 | req = requests.get(f'https://api.github.com/users?since={since}', headers=GH_HEADERS) 18 | df = pd.DataFrame(req.json()) 19 | for _, v in df.iterrows(): 20 | github_each_user.queue(v['login']) 21 | since = v['id'] 22 | # queue next job request 23 | if counter+1 < 10: 24 | github_users.queue(since=since, counter=counter+1) 25 | 26 | 27 | @RQ_CLIENT.job() 28 | def github_each_user(login): 29 | ''' get data for specific user ''' 30 | req = requests.get(f'https://api.github.com/users/{login}', headers=GH_HEADERS) 31 | ds = pd.Series(req.json()).to_dict() 32 | keys = ['public_repos', 'public_gists', 'followers', 'following'] 33 | ds2 = {key: ds[key] for key in keys} 34 | logging.info(ds2) 35 | REDIS_CLIENT.hmset(login, ds2) 36 | -------------------------------------------------------------------------------- /app/routes.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from flask import jsonify, request 3 | from app import APP, jobs 4 | 5 | 6 | @APP.route('/') 7 | def root(): 8 | return 'root' 9 | 10 | 11 | @APP.route('/github') 12 | def github(): 13 | job = jobs.github_users.queue() 14 | logging.info(job) 15 | return jsonify({'job_id': job.id}) 16 | -------------------------------------------------------------------------------- /devops/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # https://docs.docker.com/config/containers/multi-service_container/ 3 | set -m 4 | 5 | # https://ryanstutorials.net/bash-scripting-tutorial/bash-if-statements.php 6 | if [ $CONTAINER_TYPE = 'web' ] 7 | then 8 | flask run -h 0.0.0.0 -p 5000 & 9 | jupyter notebook --ip=0.0.0.0 --no-browser --allow-root --NotebookApp.token='' --NotebookApp.password='' & 10 | elif [ $CONTAINER_TYPE = 'worker' ] 11 | then 12 | rq worker -c rq_config & 13 | fi 14 | 15 | fg %1 -------------------------------------------------------------------------------- /devops/redis.env: -------------------------------------------------------------------------------- 1 | # https://docs.docker.com/compose/env-file/ 2 | REDIS_HOST=redis_data_redis_1 3 | FLASK_APP=app/__init__.py 4 | FLASK_ENV=development 5 | FLASK_DEBUG=1 6 | APP_ENV=dev 7 | GITHUB_TOKEN=${GITHUB_TOKEN} 8 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | 5 | web: 6 | build: 7 | context: . 8 | dockerfile: Dockerfile 9 | volumes: 10 | - ./:/opt/redis_data 11 | ports: 12 | - target: 5000 13 | published: 5000 14 | - target: 8888 15 | published: 8888 16 | env_file: 17 | - devops/redis.env 18 | environment: 19 | CONTAINER_TYPE: web 20 | GITHUB_TOKEN: ${GITHUB_TOKEN} 21 | 22 | worker: 23 | build: 24 | context: . 25 | dockerfile: Dockerfile 26 | volumes: 27 | - ./:/opt/redis_data 28 | env_file: 29 | - devops/redis.env 30 | environment: 31 | CONTAINER_TYPE: worker 32 | GITHUB_TOKEN: ${GITHUB_TOKEN} 33 | 34 | redis: 35 | image: redis:5.0.6-alpine 36 | ports: 37 | - target: 6379 38 | published: 6379 39 | expose: 40 | - 6379 41 | -------------------------------------------------------------------------------- /logs/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dmitrypol/redis_data/6ea52a579e0a5cea564b4ee9f5ca895b5d867a7c/logs/.keep -------------------------------------------------------------------------------- /notebooks/redis.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 23, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "import pandas as pd\n", 11 | "import redis\n", 12 | "RC = redis.Redis(host=os.environ.get('REDIS_HOST'), charset='utf-8', decode_responses=True)" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 27, 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "df = pd.DataFrame()\n", 22 | "for key in RC.keys():\n", 23 | " value = RC.hgetall(key)\n", 24 | " # print(value)\n", 25 | " value['login'] = key\n", 26 | " df = df.append(value, ignore_index=True)\n", 27 | " for k in ['public_repos', 'public_gists', 'followers', 'following']:\n", 28 | " df[k] = df[k].astype(int)\n", 29 | "# print(df)\n", 30 | "# df.sum()" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [] 39 | } 40 | ], 41 | "metadata": { 42 | "kernelspec": { 43 | "display_name": "Python 3", 44 | "language": "python", 45 | "name": "python3" 46 | }, 47 | "language_info": { 48 | "codemirror_mode": { 49 | "name": "ipython", 50 | "version": 3 51 | }, 52 | "file_extension": ".py", 53 | "mimetype": "text/x-python", 54 | "name": "python", 55 | "nbconvert_exporter": "python", 56 | "pygments_lexer": "ipython3", 57 | "version": "3.6.8" 58 | } 59 | }, 60 | "nbformat": 4, 61 | "nbformat_minor": 2 62 | } 63 | -------------------------------------------------------------------------------- /rq_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | REDIS_URL = f"redis://{os.environ.get('REDIS_HOST')}:6379/1" 3 | -------------------------------------------------------------------------------- /tmp/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dmitrypol/redis_data/6ea52a579e0a5cea564b4ee9f5ca895b5d867a7c/tmp/.keep --------------------------------------------------------------------------------