├── .gitignore ├── .mitmproxy ├── mitmproxy-ca-cert.cer ├── mitmproxy-ca-cert.p12 ├── mitmproxy-ca-cert.pem ├── mitmproxy-ca.p12 ├── mitmproxy-ca.pem └── mitmproxy-dhparam.pem ├── LICENSE ├── Makefile ├── Pipfile ├── Pipfile.lock ├── README.md ├── app ├── driver │ ├── __init__.py │ ├── base.py │ └── douyin.py ├── event.py ├── models │ ├── __init__.py │ └── douyin.py ├── service │ ├── __init__.py │ └── redis_service.py ├── settings.py ├── tools │ ├── __init__.py │ └── simple_data_import.py └── utils │ ├── __init__.py │ └── decorator.py ├── conf └── systemd │ └── app-crawler.service ├── crawler.py ├── docker-compose.yaml ├── dy.py └── manage.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | .idea/ 106 | -------------------------------------------------------------------------------- /.mitmproxy/mitmproxy-ca-cert.cer: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDoTCCAomgAwIBAgIGDk50r5i8MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM 3 | CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTE5MTEwNDAzNTk1MFoX 4 | DTIyMTEwNTAzNTk1MFowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt 5 | aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5AyTiVIbn 6 | 1OZNDYN1JfxYEa0kjFPsHdWLpoc5+GYhdnfKLZGt2kq8jWSx0Oy+bY1rlcdgNF8/ 7 | 5ExMvkNpYAh+tfZ46FMY8/v6XVmFHSobKp1pbudU3aEsw2mdHRAeQBaRz2V5U3eZ 8 | oberHhX3BDdmMoZ+220Ty5idjDgSWZGSQlgJR/8CvR1T0/A25uqdhXApLIbFeNvb 9 | TXdcWBxxAU3LbiVwWG0OSiKOkUWb27KBX/byCdpzoFCDMLmak3mX8HNFWI6dZYBO 10 | xN7etonhrgN78kUl/QC1TBZAB5QsA2b8zmLY2aQbGXYXYCoDtGbZot5v4Efr/bWp 11 | F3lssTJ/d4rTAgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC 12 | AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD 13 | BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG 14 | CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC 15 | AQYwHQYDVR0OBBYEFL/CPG+rHmYgbzds2aysvWZCt6klMA0GCSqGSIb3DQEBCwUA 16 | A4IBAQAZp0VgQUZjKHwnlcdH6LOgwv4X4nKyArH2166sONz5s17+OkZPA32HgNSc 17 | 8AzKu9UepwSlHsIq60COUKqTBjmQu4DigDrZ6PkfFyNpaSIUC4KiyybhZrO/c0Os 18 | y6FkzQ6505ImPIP3lUB4wfoO/3blPahP2GBK+57hUZObHH/17loArPrh6kniEey6 19 | bGM3Aho/eE7nMdAtxeo1EMzKVY8/bCnyqbbalStnC3uBJvtqBCHiPbPr9A5M0i4o 20 | TpYFmPaF0q93pB1Bb1NK9qObTlJ79DBnqmc7yMrZZbrQ4772uO6C1o3CfWtV6ctV 21 | Z3R9MeANVYHP9P968pJyy6t4Ce2C 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /.mitmproxy/mitmproxy-ca-cert.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maguowei/app-crawler/5a8211671efc86d01f72c9bfdb4543aff135c5d7/.mitmproxy/mitmproxy-ca-cert.p12 -------------------------------------------------------------------------------- /.mitmproxy/mitmproxy-ca-cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDoTCCAomgAwIBAgIGDk50r5i8MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM 3 | CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTE5MTEwNDAzNTk1MFoX 4 | DTIyMTEwNTAzNTk1MFowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt 5 | aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5AyTiVIbn 6 | 1OZNDYN1JfxYEa0kjFPsHdWLpoc5+GYhdnfKLZGt2kq8jWSx0Oy+bY1rlcdgNF8/ 7 | 5ExMvkNpYAh+tfZ46FMY8/v6XVmFHSobKp1pbudU3aEsw2mdHRAeQBaRz2V5U3eZ 8 | oberHhX3BDdmMoZ+220Ty5idjDgSWZGSQlgJR/8CvR1T0/A25uqdhXApLIbFeNvb 9 | TXdcWBxxAU3LbiVwWG0OSiKOkUWb27KBX/byCdpzoFCDMLmak3mX8HNFWI6dZYBO 10 | xN7etonhrgN78kUl/QC1TBZAB5QsA2b8zmLY2aQbGXYXYCoDtGbZot5v4Efr/bWp 11 | F3lssTJ/d4rTAgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC 12 | AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD 13 | BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG 14 | CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC 15 | AQYwHQYDVR0OBBYEFL/CPG+rHmYgbzds2aysvWZCt6klMA0GCSqGSIb3DQEBCwUA 16 | A4IBAQAZp0VgQUZjKHwnlcdH6LOgwv4X4nKyArH2166sONz5s17+OkZPA32HgNSc 17 | 8AzKu9UepwSlHsIq60COUKqTBjmQu4DigDrZ6PkfFyNpaSIUC4KiyybhZrO/c0Os 18 | y6FkzQ6505ImPIP3lUB4wfoO/3blPahP2GBK+57hUZObHH/17loArPrh6kniEey6 19 | bGM3Aho/eE7nMdAtxeo1EMzKVY8/bCnyqbbalStnC3uBJvtqBCHiPbPr9A5M0i4o 20 | TpYFmPaF0q93pB1Bb1NK9qObTlJ79DBnqmc7yMrZZbrQ4772uO6C1o3CfWtV6ctV 21 | Z3R9MeANVYHP9P968pJyy6t4Ce2C 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /.mitmproxy/mitmproxy-ca.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maguowei/app-crawler/5a8211671efc86d01f72c9bfdb4543aff135c5d7/.mitmproxy/mitmproxy-ca.p12 -------------------------------------------------------------------------------- /.mitmproxy/mitmproxy-ca.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC5AyTiVIbn1OZN 3 | DYN1JfxYEa0kjFPsHdWLpoc5+GYhdnfKLZGt2kq8jWSx0Oy+bY1rlcdgNF8/5ExM 4 | vkNpYAh+tfZ46FMY8/v6XVmFHSobKp1pbudU3aEsw2mdHRAeQBaRz2V5U3eZober 5 | HhX3BDdmMoZ+220Ty5idjDgSWZGSQlgJR/8CvR1T0/A25uqdhXApLIbFeNvbTXdc 6 | WBxxAU3LbiVwWG0OSiKOkUWb27KBX/byCdpzoFCDMLmak3mX8HNFWI6dZYBOxN7e 7 | tonhrgN78kUl/QC1TBZAB5QsA2b8zmLY2aQbGXYXYCoDtGbZot5v4Efr/bWpF3ls 8 | sTJ/d4rTAgMBAAECggEADUeMfjJS/JDrdtxhcycm1atvIhBwdaIukbeCPUb3Da/T 9 | KU/cBP6GHsKHV7wHQiPa4r39cYpQy+U3FMsTS5z4/x9QcZ07Wa8phu7sXgGZ5BlK 10 | 8yZitFS0HOJp0j0jywgOlqisP+kdzIkvBblDGlfTIiK+RJy7V5Blg+lsnOSQ7k51 11 | SEAiW6h0rSDm6pwosSoInhAnrI+ylNItR7TMySFENuB6p1Fbud3d/ft7ntRPCMNc 12 | HInBeltQcZOGTHfJTC1JRwoIBzPeDDFw6dRC4HlJtcHj5Fp9vYdxpmdVD1JiCElS 13 | o0WVCTmCroXVRmTnPGP/kAdGJ3B2YQIO1xZroVyjAQKBgQD2Oj8z+p2F4nnTWy+I 14 | wo5ECdVE66Xr3cW4TkMI1GzqYcme1OHOy4EL1dTO2280z+jNIV2XLeXO4p6T5sUo 15 | i90EPVEKHAjr0McQ1N0A7j2WRbLcDPK60QnSCiSpXjhHALUglNk3SYjGb+Rpf4NF 16 | d/5NHJMd1TeF3XbJDBhI5epAXQKBgQDAWu4X4val8QxlX+cX/Ma/tYC9rQO++4eh 17 | yE8sGvGz8vcp4jYEZfbPi8/37pPKenphjG9rGl2eh/lWdvUvvjiUPPcGLf5A5aNn 18 | jC4tQ8G5BNq/1TWyTHdtQLegbZso+3sPE6nE1EGru+sASUxAqG7NMBl0LC7LlE+3 19 | t2LHN5QE7wKBgE6RZ3l+jqUaDyWSAyjwa9TqDlNfniIVMfINhvZaUia37U3xP9xs 20 | s1DDepRHEMUZUn+iFesXiizpwxAOovq9Jlkj96fyKiaVB/Am/B4/5wb8VoZA23Mh 21 | ILEU633fwrVlVGaBGI67uBkZuxiux4RzNYIFjCwIzR65/B1K7rTUKFtBAoGBAJdw 22 | v5UkxoVdIyJI//sdsyhJ8MLkjVd2VpFj0BlsWTEQr9FU3KxXzpoWfDS5q7e099cH 23 | Js1O47glW4PcgisFeywSV3WRDmTvptemQNC4ULOnA8YWSYFHvJrSYf+3a3o3i+oR 24 | 1A44Aj//4gPGsXcSZLG9Fb0l7+2tpZmBkw4/TpUXAoGAM5iOD9X8mwBnr775RytY 25 | FelcMwzWU/jHVJbi8WNVTH4x6U3EPNxylD7b/V8xsMA50VCiVqVbq5X5zj5RLym9 26 | 8LLBC/F7l2epWt1CN8kS3lXSZ0Gf9MgYAsyHRdAxmjxa6J0Yf6Vr4W+o+A/zEXV9 27 | KndgthCj1RkdWnFvCmC3Em4= 28 | -----END PRIVATE KEY----- 29 | -----BEGIN CERTIFICATE----- 30 | MIIDoTCCAomgAwIBAgIGDk50r5i8MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM 31 | CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTE5MTEwNDAzNTk1MFoX 32 | DTIyMTEwNTAzNTk1MFowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt 33 | aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5AyTiVIbn 34 | 1OZNDYN1JfxYEa0kjFPsHdWLpoc5+GYhdnfKLZGt2kq8jWSx0Oy+bY1rlcdgNF8/ 35 | 5ExMvkNpYAh+tfZ46FMY8/v6XVmFHSobKp1pbudU3aEsw2mdHRAeQBaRz2V5U3eZ 36 | oberHhX3BDdmMoZ+220Ty5idjDgSWZGSQlgJR/8CvR1T0/A25uqdhXApLIbFeNvb 37 | TXdcWBxxAU3LbiVwWG0OSiKOkUWb27KBX/byCdpzoFCDMLmak3mX8HNFWI6dZYBO 38 | xN7etonhrgN78kUl/QC1TBZAB5QsA2b8zmLY2aQbGXYXYCoDtGbZot5v4Efr/bWp 39 | F3lssTJ/d4rTAgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC 40 | AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD 41 | BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG 42 | CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC 43 | AQYwHQYDVR0OBBYEFL/CPG+rHmYgbzds2aysvWZCt6klMA0GCSqGSIb3DQEBCwUA 44 | A4IBAQAZp0VgQUZjKHwnlcdH6LOgwv4X4nKyArH2166sONz5s17+OkZPA32HgNSc 45 | 8AzKu9UepwSlHsIq60COUKqTBjmQu4DigDrZ6PkfFyNpaSIUC4KiyybhZrO/c0Os 46 | y6FkzQ6505ImPIP3lUB4wfoO/3blPahP2GBK+57hUZObHH/17loArPrh6kniEey6 47 | bGM3Aho/eE7nMdAtxeo1EMzKVY8/bCnyqbbalStnC3uBJvtqBCHiPbPr9A5M0i4o 48 | TpYFmPaF0q93pB1Bb1NK9qObTlJ79DBnqmc7yMrZZbrQ4772uO6C1o3CfWtV6ctV 49 | Z3R9MeANVYHP9P968pJyy6t4Ce2C 50 | -----END CERTIFICATE----- 51 | -------------------------------------------------------------------------------- /.mitmproxy/mitmproxy-dhparam.pem: -------------------------------------------------------------------------------- 1 | 2 | -----BEGIN DH PARAMETERS----- 3 | MIICCAKCAgEAyT6LzpwVFS3gryIo29J5icvgxCnCebcdSe/NHMkD8dKJf8suFCg3 4 | O2+dguLakSVif/t6dhImxInJk230HmfC8q93hdcg/j8rLGJYDKu3ik6H//BAHKIv 5 | j5O9yjU3rXCfmVJQic2Nne39sg3CreAepEts2TvYHhVv3TEAzEqCtOuTjgDv0ntJ 6 | Gwpj+BJBRQGG9NvprX1YGJ7WOFBP/hWU7d6tgvE6Xa7T/u9QIKpYHMIkcN/l3ZFB 7 | chZEqVlyrcngtSXCROTPcDOQ6Q8QzhaBJS+Z6rcsd7X+haiQqvoFcmaJ08Ks6LQC 8 | ZIL2EtYJw8V8z7C0igVEBIADZBI6OTbuuhDwRw//zU1uq52Oc48CIZlGxTYG/Evq 9 | o9EWAXUYVzWkDSTeBH1r4z/qLPE2cnhtMxbFxuvK53jGB0emy2y1Ei6IhKshJ5qX 10 | IB/aE7SSHyQ3MDHHkCmQJCsOd4Mo26YX61NZ+n501XjqpCBQ2+DfZCBh8Va2wDyv 11 | A2Ryg9SUz8j0AXViRNMJgJrr446yro/FuJZwnQcO3WQnXeqSBnURqKjmqkeFP+d8 12 | 6mk2tqJaY507lRNqtGlLnj7f5RNoBFJDCLBNurVgfvq9TCVWKDIFD4vZRjCrnl6I 13 | rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI= 14 | -----END DH PARAMETERS----- 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 maguowei 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | run-mitmproxy: 2 | mitmdump -s manage.py 3 | 4 | up: 5 | docker-compose up -d 6 | 7 | down: 8 | docker-compose down 9 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | 8 | [packages] 9 | uiautomator2 = {index = "https://mirrors.aliyun.com/pypi/simple/",version = "*"} 10 | mitmproxy = {index = "https://mirrors.aliyun.com/pypi/simple/",version = "*"} 11 | redis = {index = "https://mirrors.aliyun.com/pypi/simple/",version = "*"} 12 | ipython = {index = "https://mirrors.aliyun.com/pypi/simple/",version = "*"} 13 | weditor = {index = "https://mirrors.aliyun.com/pypi/simple/",version = "*"} 14 | fire = {index = "https://mirrors.aliyun.com/pypi/simple/",version = "*"} 15 | sqlalchemy = {index = "https://mirrors.aliyun.com/pypi/simple/",version = "*"} 16 | pymysql = {index = "https://mirrors.aliyun.com/pypi/simple/",version = "*"} 17 | 18 | [requires] 19 | python_version = "3.7" 20 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "78f7db355a566120bc0c910ce9416eab811cc6f77ce346b3ba54234d887b7732" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.7" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "adbutils": { 20 | "hashes": [ 21 | "sha256:0ddb709ee3367d1d2ad21889d8326728c6c259cdcec7a64293b78bbec495ce3a" 22 | ], 23 | "version": "==0.6.4" 24 | }, 25 | "apkutils2": { 26 | "hashes": [ 27 | "sha256:c5ae8f86d3ebee6a59fc014d88507741d7f3f9ab183bab34b44d011fe878660b" 28 | ], 29 | "version": "==1.0.0" 30 | }, 31 | "appnope": { 32 | "hashes": [ 33 | "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0", 34 | "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71" 35 | ], 36 | "markers": "sys_platform == 'darwin'", 37 | "version": "==0.1.0" 38 | }, 39 | "asn1crypto": { 40 | "hashes": [ 41 | "sha256:7bb1cc02a5620b3d72da4ba070bda2f44f0e61b44dee910a302eddff802b6fb5", 42 | "sha256:87620880a477123e01177a1f73d0f327210b43a3cdbd714efcd2fa49a8d7b384" 43 | ], 44 | "version": "==1.2.0" 45 | }, 46 | "backcall": { 47 | "hashes": [ 48 | "sha256:38ecd85be2c1e78f77fd91700c76e14667dc21e2713b63876c0eb901196e01e4", 49 | "sha256:bbbf4b1e5cd2bdb08f915895b51081c041bac22394fdfcfdfbe9f14b77c08bf2" 50 | ], 51 | "version": "==0.1.0" 52 | }, 53 | "blinker": { 54 | "hashes": [ 55 | "sha256:471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6" 56 | ], 57 | "version": "==1.4" 58 | }, 59 | "brotlipy": { 60 | "hashes": [ 61 | "sha256:07194f4768eb62a4f4ea76b6d0df6ade185e24ebd85877c351daa0a069f1111a", 62 | "sha256:091b299bf36dd6ef7a06570dbc98c0f80a504a56c5b797f31934d2ad01ae7d17", 63 | "sha256:09ec3e125d16749b31c74f021aba809541b3564e5359f8c265cbae442810b41a", 64 | "sha256:0be698678a114addcf87a4b9496c552c68a2c99bf93cf8e08f5738b392e82057", 65 | "sha256:0fa6088a9a87645d43d7e21e32b4a6bf8f7c3939015a50158c10972aa7f425b7", 66 | "sha256:1379347337dc3d20b2d61456d44ccce13e0625db2611c368023b4194d5e2477f", 67 | "sha256:1ea4e578241504b58f2456a6c69952c88866c794648bdc74baee74839da61d44", 68 | "sha256:2699945a0a992c04fc7dc7fa2f1d0575a2c8b4b769f2874a08e8eae46bef36ae", 69 | "sha256:2a80319ae13ea8dd60ecdc4f5ccf6da3ae64787765923256b62c598c5bba4121", 70 | "sha256:2e5c64522364a9ebcdf47c5744a5ddeb3f934742d31e61ebfbbc095460b47162", 71 | "sha256:36def0b859beaf21910157b4c33eb3b06d8ce459c942102f16988cca6ea164df", 72 | "sha256:3a3e56ced8b15fbbd363380344f70f3b438e0fd1fcf27b7526b6172ea950e867", 73 | "sha256:3c1d5e2cf945a46975bdb11a19257fa057b67591eb232f393d260e7246d9e571", 74 | "sha256:4e4638b49835d567d447a2cfacec109f9a777f219f071312268b351b6839436d", 75 | "sha256:50ca336374131cfad20612f26cc43c637ac0bfd2be3361495e99270883b52962", 76 | "sha256:5de6f7d010b7558f72f4b061a07395c5c3fd57f0285c5af7f126a677b976a868", 77 | "sha256:637847560d671657f993313ecc6c6c6666a936b7a925779fd044065c7bc035b9", 78 | "sha256:653faef61241bf8bf99d73ca7ec4baa63401ba7b2a2aa88958394869379d67c7", 79 | "sha256:786afc8c9bd67de8d31f46e408a3386331e126829114e4db034f91eacb05396d", 80 | "sha256:79aaf217072840f3e9a3b641cccc51f7fc23037496bd71e26211856b93f4b4cb", 81 | "sha256:7e31f7adcc5851ca06134705fcf3478210da45d35ad75ec181e1ce9ce345bb38", 82 | "sha256:8b39abc3256c978f575df5cd7893153277216474f303e26f0e43ba3d3969ef96", 83 | "sha256:9448227b0df082e574c45c983fa5cd4bda7bfb11ea6b59def0940c1647be0c3c", 84 | "sha256:96bc59ff9b5b5552843dc67999486a220e07a0522dddd3935da05dc194fa485c", 85 | "sha256:a07647886e24e2fb2d68ca8bf3ada398eb56fd8eac46c733d4d95c64d17f743b", 86 | "sha256:af65d2699cb9f13b26ec3ba09e75e80d31ff422c03675fcb36ee4dabe588fdc2", 87 | "sha256:b4c98b0d2c9c7020a524ca5bbff42027db1004c6571f8bc7b747f2b843128e7a", 88 | "sha256:c6cc0036b1304dd0073eec416cb2f6b9e37ac8296afd9e481cac3b1f07f9db25", 89 | "sha256:d2c1c724c4ac375feb2110f1af98ecdc0e5a8ea79d068efb5891f621a5b235cb", 90 | "sha256:dc6c5ee0df9732a44d08edab32f8a616b769cc5a4155a12d2d010d248eb3fb07", 91 | "sha256:fd1d1c64214af5d90014d82cee5d8141b13d44c92ada7a0c0ec0679c6f15a471" 92 | ], 93 | "version": "==0.7.0" 94 | }, 95 | "certifi": { 96 | "hashes": [ 97 | "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", 98 | "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" 99 | ], 100 | "version": "==2019.11.28" 101 | }, 102 | "cffi": { 103 | "hashes": [ 104 | "sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42", 105 | "sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04", 106 | "sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5", 107 | "sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54", 108 | "sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba", 109 | "sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57", 110 | "sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396", 111 | "sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12", 112 | "sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97", 113 | "sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43", 114 | "sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db", 115 | "sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3", 116 | "sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b", 117 | "sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579", 118 | "sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346", 119 | "sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159", 120 | "sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652", 121 | "sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e", 122 | "sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a", 123 | "sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506", 124 | "sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f", 125 | "sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d", 126 | "sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c", 127 | "sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20", 128 | "sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858", 129 | "sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc", 130 | "sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a", 131 | "sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3", 132 | "sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e", 133 | "sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410", 134 | "sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25", 135 | "sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b", 136 | "sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d" 137 | ], 138 | "version": "==1.13.2" 139 | }, 140 | "chardet": { 141 | "hashes": [ 142 | "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", 143 | "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" 144 | ], 145 | "version": "==3.0.4" 146 | }, 147 | "cigam": { 148 | "hashes": [ 149 | "sha256:8fcf65d7361f0372c53780e861307abd1f11a94b6204fa653ba3f38277822783" 150 | ], 151 | "version": "==0.0.3" 152 | }, 153 | "click": { 154 | "hashes": [ 155 | "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", 156 | "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" 157 | ], 158 | "version": "==6.7" 159 | }, 160 | "cryptography": { 161 | "hashes": [ 162 | "sha256:02602e1672b62e803e08617ec286041cc453e8d43f093a5f4162095506bc0beb", 163 | "sha256:10b48e848e1edb93c1d3b797c83c72b4c387ab0eb4330aaa26da8049a6cbede0", 164 | "sha256:17db09db9d7c5de130023657be42689d1a5f60502a14f6f745f6f65a6b8195c0", 165 | "sha256:227da3a896df1106b1a69b1e319dce218fa04395e8cc78be7e31ca94c21254bc", 166 | "sha256:2cbaa03ac677db6c821dac3f4cdfd1461a32d0615847eedbb0df54bb7802e1f7", 167 | "sha256:31db8febfc768e4b4bd826750a70c79c99ea423f4697d1dab764eb9f9f849519", 168 | "sha256:4a510d268e55e2e067715d728e4ca6cd26a8e9f1f3d174faf88e6f2cb6b6c395", 169 | "sha256:6a88d9004310a198c474d8a822ee96a6dd6c01efe66facdf17cb692512ae5bc0", 170 | "sha256:76936ec70a9b72eb8c58314c38c55a0336a2b36de0c7ee8fb874a4547cadbd39", 171 | "sha256:7e3b4aecc4040928efa8a7cdaf074e868af32c58ffc9bb77e7bf2c1a16783286", 172 | "sha256:8168bcb08403ef144ff1fb880d416f49e2728101d02aaadfe9645883222c0aa5", 173 | "sha256:8229ceb79a1792823d87779959184a1bf95768e9248c93ae9f97c7a2f60376a1", 174 | "sha256:8a19e9f2fe69f6a44a5c156968d9fc8df56d09798d0c6a34ccc373bb186cee86", 175 | "sha256:8d10113ca826a4c29d5b85b2c4e045ffa8bad74fb525ee0eceb1d38d4c70dfd6", 176 | "sha256:be495b8ec5a939a7605274b6e59fbc35e76f5ad814ae010eb679529671c9e119", 177 | "sha256:dc2d3f3b1548f4d11786616cf0f4415e25b0fbecb8a1d2cd8c07568f13fdde38", 178 | "sha256:e4aecdd9d5a3d06c337894c9a6e2961898d3f64fe54ca920a72234a3de0f9cb3", 179 | "sha256:e79ab4485b99eacb2166f3212218dd858258f374855e1568f728462b0e6ee0d9", 180 | "sha256:f995d3667301e1754c57b04e0bae6f0fa9d710697a9f8d6712e8cca02550910f" 181 | ], 182 | "version": "==2.3.1" 183 | }, 184 | "decorator": { 185 | "hashes": [ 186 | "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce", 187 | "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d" 188 | ], 189 | "version": "==4.4.1" 190 | }, 191 | "deprecated": { 192 | "hashes": [ 193 | "sha256:408038ab5fdeca67554e8f6742d1521cd3cd0ee0ff9d47f29318a4f4da31c308", 194 | "sha256:8b6a5aa50e482d8244a62e5582b96c372e87e3a28e8b49c316e46b95c76a611d" 195 | ], 196 | "version": "==1.2.7" 197 | }, 198 | "deprecation": { 199 | "hashes": [ 200 | "sha256:c0392f676a6146f0238db5744d73e786a43510d54033f80994ef2f4c9df192ed", 201 | "sha256:dc9b4f252b7aca8165ce2764a71da92a653b5ffbf7a389461d7a640f6536ecb2" 202 | ], 203 | "version": "==2.0.7" 204 | }, 205 | "facebook-wda": { 206 | "hashes": [ 207 | "sha256:2c8ac936c497242db8624f920d7316f8938ed8d6c9e0460558c60faf561df51a", 208 | "sha256:48ef31cd8e6e0ea383f1b4eb0af17d7665a144f3a1b532982f15b081ac4733b2", 209 | "sha256:abd3cdc35bb8a83fac7506821effa1bc25465ef0a3216d86615dfa4f31266dd6" 210 | ], 211 | "version": "==0.4.1" 212 | }, 213 | "fire": { 214 | "hashes": [ 215 | "sha256:6865fefc6981a713d2ce56a2a2c92c56c729269f74a6cddd6f4b94d16ae084c9" 216 | ], 217 | "version": "==0.2.1" 218 | }, 219 | "h11": { 220 | "hashes": [ 221 | "sha256:1c0fbb1cba6f809fe3e6b27f8f6d517ca171f848922708871403636143d530d9", 222 | "sha256:af77d5d82fa027c032650fb8afdef3cd0a3735ba01480bee908cddad9be1bdce" 223 | ], 224 | "version": "==0.7.0" 225 | }, 226 | "h2": { 227 | "hashes": [ 228 | "sha256:ac377fcf586314ef3177bfd90c12c7826ab0840edeb03f0f24f511858326049e", 229 | "sha256:b8a32bd282594424c0ac55845377eea13fa54fe4a8db012f3a198ed923dc3ab4" 230 | ], 231 | "version": "==3.1.1" 232 | }, 233 | "hpack": { 234 | "hashes": [ 235 | "sha256:0edd79eda27a53ba5be2dfabf3b15780928a0dff6eb0c60a3d6767720e970c89", 236 | "sha256:8eec9c1f4bfae3408a3f30500261f7e6a65912dc138526ea054f9ad98892e9d2" 237 | ], 238 | "version": "==3.0.0" 239 | }, 240 | "humanize": { 241 | "hashes": [ 242 | "sha256:a43f57115831ac7c70de098e6ac46ac13be00d69abbf60bdcac251344785bb19" 243 | ], 244 | "version": "==0.5.1" 245 | }, 246 | "hyperframe": { 247 | "hashes": [ 248 | "sha256:5187962cb16dcc078f23cb5a4b110098d546c3f41ff2d4038a9896893bbd0b40", 249 | "sha256:a9f5c17f2cc3c719b917c4f33ed1c61bd1f8dfac4b1bd23b7c80b3400971b41f" 250 | ], 251 | "version": "==5.2.0" 252 | }, 253 | "idna": { 254 | "hashes": [ 255 | "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", 256 | "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" 257 | ], 258 | "version": "==2.8" 259 | }, 260 | "ipython": { 261 | "hashes": [ 262 | "sha256:c66c7e27239855828a764b1e8fc72c24a6f4498a2637572094a78c5551fb9d51", 263 | "sha256:f186b01b36609e0c5d0de27c7ef8e80c990c70478f8c880863004b3489a9030e" 264 | ], 265 | "version": "==7.10.1" 266 | }, 267 | "ipython-genutils": { 268 | "hashes": [ 269 | "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8", 270 | "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8" 271 | ], 272 | "version": "==0.2.0" 273 | }, 274 | "jedi": { 275 | "hashes": [ 276 | "sha256:786b6c3d80e2f06fd77162a07fed81b8baa22dde5d62896a790a331d6ac21a27", 277 | "sha256:ba859c74fa3c966a22f2aeebe1b74ee27e2a462f56d3f5f7ca4a59af61bfe42e" 278 | ], 279 | "version": "==0.15.1" 280 | }, 281 | "kaitaistruct": { 282 | "hashes": [ 283 | "sha256:d1d17c7f6839b3d28fc22b21295f787974786c2201e8788975e72e2a1d109ff5" 284 | ], 285 | "version": "==0.8" 286 | }, 287 | "ldap3": { 288 | "hashes": [ 289 | "sha256:3f67c83185b1f0df8fdf6b52fa42c55bc9e9b7120c8b7fec60f0d6003c536d18", 290 | "sha256:dd9be8ea27773c4ffc18ede0b95c3ca1eb12513a184590b9f8ae423db3f71eb9" 291 | ], 292 | "version": "==2.5.2" 293 | }, 294 | "logzero": { 295 | "hashes": [ 296 | "sha256:34fa1e2e436dfa9f37e5ff8750e932bafe0c5abbb42e1f669e4cf5ce1f179142", 297 | "sha256:818072e4fcb53a3f6fb4114a92f920e1135fe6f47bffd9dc2b6c4d10eedacf27" 298 | ], 299 | "version": "==1.5.0" 300 | }, 301 | "lxml": { 302 | "hashes": [ 303 | "sha256:00ac0d64949fef6b3693813fe636a2d56d97a5a49b5bbb86e4cc4cc50ebc9ea2", 304 | "sha256:0571e607558665ed42e450d7bf0e2941d542c18e117b1ebbf0ba72f287ad841c", 305 | "sha256:0e3f04a7615fdac0be5e18b2406529521d6dbdb0167d2a690ee328bef7807487", 306 | "sha256:13cf89be53348d1c17b453867da68704802966c433b2bb4fa1f970daadd2ef70", 307 | "sha256:217262fcf6a4c2e1c7cb1efa08bd9ebc432502abc6c255c4abab611e8be0d14d", 308 | "sha256:223e544828f1955daaf4cefbb4853bc416b2ec3fd56d4f4204a8b17007c21250", 309 | "sha256:277cb61fede2f95b9c61912fefb3d43fbd5f18bf18a14fae4911b67984486f5d", 310 | "sha256:3213f753e8ae86c396e0e066866e64c6b04618e85c723b32ecb0909885211f74", 311 | "sha256:4690984a4dee1033da0af6df0b7a6bde83f74e1c0c870623797cec77964de34d", 312 | "sha256:4fcc472ef87f45c429d3b923b925704aa581f875d65bac80f8ab0c3296a63f78", 313 | "sha256:61409bd745a265a742f2693e4600e4dbd45cc1daebe1d5fad6fcb22912d44145", 314 | "sha256:678f1963f755c5d9f5f6968dded7b245dd1ece8cf53c1aa9d80e6734a8c7f41d", 315 | "sha256:6c6d03549d4e2734133badb9ab1c05d9f0ef4bcd31d83e5d2b4747c85cfa21da", 316 | "sha256:6e74d5f4d6ecd6942375c52ffcd35f4318a61a02328f6f1bd79fcb4ffedf969e", 317 | "sha256:7b4fc7b1ecc987ca7aaf3f4f0e71bbfbd81aaabf87002558f5bc95da3a865bcd", 318 | "sha256:7ed386a40e172ddf44c061ad74881d8622f791d9af0b6f5be20023029129bc85", 319 | "sha256:8f54f0924d12c47a382c600c880770b5ebfc96c9fd94cf6f6bdc21caf6163ea7", 320 | "sha256:ad9b81351fdc236bda538efa6879315448411a81186c836d4b80d6ca8217cdb9", 321 | "sha256:bbd00e21ea17f7bcc58dccd13869d68441b32899e89cf6cfa90d624a9198ce85", 322 | "sha256:c3c289762cc09735e2a8f8a49571d0e8b4f57ea831ea11558247b5bdea0ac4db", 323 | "sha256:cf4650942de5e5685ad308e22bcafbccfe37c54aa7c0e30cd620c2ee5c93d336", 324 | "sha256:cfcbc33c9c59c93776aa41ab02e55c288a042211708b72fdb518221cc803abc8", 325 | "sha256:e301055deadfedbd80cf94f2f65ff23126b232b0d1fea28f332ce58137bcdb18", 326 | "sha256:ebbfe24df7f7b5c6c7620702496b6419f6a9aa2fd7f005eb731cc80d7b4692b9", 327 | "sha256:eff69ddbf3ad86375c344339371168640951c302450c5d3e9936e98d6459db06", 328 | "sha256:f6ed60a62c5f1c44e789d2cf14009423cb1646b44a43e40a9cf6a21f077678a1" 329 | ], 330 | "version": "==4.4.2" 331 | }, 332 | "mitmproxy": { 333 | "hashes": [ 334 | "sha256:e74869c7bf4e5b988fbe3a3d0039f430d1e1eeb5927abf2097183a711bf5b312" 335 | ], 336 | "version": "==4.0.4" 337 | }, 338 | "packaging": { 339 | "hashes": [ 340 | "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", 341 | "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" 342 | ], 343 | "version": "==19.2" 344 | }, 345 | "parso": { 346 | "hashes": [ 347 | "sha256:63854233e1fadb5da97f2744b6b24346d2750b85965e7e399bec1620232797dc", 348 | "sha256:666b0ee4a7a1220f65d367617f2cd3ffddff3e205f3f16a0284df30e774c2a9c" 349 | ], 350 | "version": "==0.5.1" 351 | }, 352 | "passlib": { 353 | "hashes": [ 354 | "sha256:68c35c98a7968850e17f1b6892720764cc7eed0ef2b7cb3116a89a28e43fe177", 355 | "sha256:8d666cef936198bc2ab47ee9b0410c94adf2ba798e5a84bf220be079ae7ab6a8" 356 | ], 357 | "version": "==1.7.2" 358 | }, 359 | "pexpect": { 360 | "hashes": [ 361 | "sha256:2094eefdfcf37a1fdbfb9aa090862c1a4878e5c7e0e7e7088bdb511c558e5cd1", 362 | "sha256:9e2c1fd0e6ee3a49b28f95d4b33bc389c89b20af6a1255906e90ff1262ce62eb" 363 | ], 364 | "markers": "sys_platform != 'win32'", 365 | "version": "==4.7.0" 366 | }, 367 | "pickleshare": { 368 | "hashes": [ 369 | "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca", 370 | "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56" 371 | ], 372 | "version": "==0.7.5" 373 | }, 374 | "pillow": { 375 | "hashes": [ 376 | "sha256:047d9473cf68af50ac85f8ee5d5f21a60f849bc17d348da7fc85711287a75031", 377 | "sha256:0f66dc6c8a3cc319561a633b6aa82c44107f12594643efa37210d8c924fc1c71", 378 | "sha256:12c9169c4e8fe0a7329e8658c7e488001f6b4c8e88740e76292c2b857af2e94c", 379 | "sha256:248cffc168896982f125f5c13e9317c059f74fffdb4152893339f3be62a01340", 380 | "sha256:27faf0552bf8c260a5cee21a76e031acaea68babb64daf7e8f2e2540745082aa", 381 | "sha256:285edafad9bc60d96978ed24d77cdc0b91dace88e5da8c548ba5937c425bca8b", 382 | "sha256:384b12c9aa8ef95558abdcb50aada56d74bc7cc131dd62d28c2d0e4d3aadd573", 383 | "sha256:38950b3a707f6cef09cd3cbb142474357ad1a985ceb44d921bdf7b4647b3e13e", 384 | "sha256:4aad1b88933fd6dc2846552b89ad0c74ddbba2f0884e2c162aa368374bf5abab", 385 | "sha256:4ac6148008c169603070c092e81f88738f1a0c511e07bd2bb0f9ef542d375da9", 386 | "sha256:4deb1d2a45861ae6f0b12ea0a786a03d19d29edcc7e05775b85ec2877cb54c5e", 387 | "sha256:59aa2c124df72cc75ed72c8d6005c442d4685691a30c55321e00ed915ad1a291", 388 | "sha256:5a47d2123a9ec86660fe0e8d0ebf0aa6bc6a17edc63f338b73ea20ba11713f12", 389 | "sha256:5cc901c2ab9409b4b7ac7b5bcc3e86ac14548627062463da0af3b6b7c555a871", 390 | "sha256:6c1db03e8dff7b9f955a0fb9907eb9ca5da75b5ce056c0c93d33100a35050281", 391 | "sha256:7ce80c0a65a6ea90ef9c1f63c8593fcd2929448613fc8da0adf3e6bfad669d08", 392 | "sha256:809c19241c14433c5d6135e1b6c72da4e3b56d5c865ad5736ab99af8896b8f41", 393 | "sha256:83792cb4e0b5af480588601467c0764242b9a483caea71ef12d22a0d0d6bdce2", 394 | "sha256:846fa202bd7ee0f6215c897a1d33238ef071b50766339186687bd9b7a6d26ac5", 395 | "sha256:9f5529fc02009f96ba95bea48870173426879dc19eec49ca8e08cd63ecd82ddb", 396 | "sha256:a423c2ea001c6265ed28700df056f75e26215fd28c001e93ef4380b0f05f9547", 397 | "sha256:ac4428094b42907aba5879c7c000d01c8278d451a3b7cccd2103e21f6397ea75", 398 | "sha256:b1ae48d87f10d1384e5beecd169c77502fcc04a2c00a4c02b85f0a94b419e5f9", 399 | "sha256:bf4e972a88f8841d8fdc6db1a75e0f8d763e66e3754b03006cbc3854d89f1cb1", 400 | "sha256:c6414f6aad598364aaf81068cabb077894eb88fed99c6a65e6e8217bab62ae7a", 401 | "sha256:c710fcb7ee32f67baf25aa9ffede4795fd5d93b163ce95fdc724383e38c9df96", 402 | "sha256:c7be4b8a09852291c3c48d3c25d1b876d2494a0a674980089ac9d5e0d78bd132", 403 | "sha256:c9e5ffb910b14f090ac9c38599063e354887a5f6d7e6d26795e916b4514f2c1a", 404 | "sha256:e0697b826da6c2472bb6488db4c0a7fa8af0d52fa08833ceb3681358914b14e5", 405 | "sha256:e9a3edd5f714229d41057d56ac0f39ad9bdba6767e8c888c951869f0bdd129b0" 406 | ], 407 | "version": "==6.2.1" 408 | }, 409 | "progress": { 410 | "hashes": [ 411 | "sha256:69ecedd1d1bbe71bf6313d88d1e6c4d2957b7f1d4f71312c211257f7dae64372" 412 | ], 413 | "version": "==1.5" 414 | }, 415 | "prompt-toolkit": { 416 | "hashes": [ 417 | "sha256:0278d2f51b5ceba6ea8da39f76d15684e84c996b325475f6e5720edc584326a7", 418 | "sha256:63daee79aa8366c8f1c637f1a4876b890da5fc92a19ebd2f7080ebacb901e990" 419 | ], 420 | "version": "==3.0.2" 421 | }, 422 | "ptyprocess": { 423 | "hashes": [ 424 | "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0", 425 | "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f" 426 | ], 427 | "version": "==0.6.0" 428 | }, 429 | "py": { 430 | "hashes": [ 431 | "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", 432 | "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" 433 | ], 434 | "version": "==1.8.0" 435 | }, 436 | "pyasn1": { 437 | "hashes": [ 438 | "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", 439 | "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba" 440 | ], 441 | "version": "==0.4.8" 442 | }, 443 | "pycparser": { 444 | "hashes": [ 445 | "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" 446 | ], 447 | "version": "==2.19" 448 | }, 449 | "pyelftools": { 450 | "hashes": [ 451 | "sha256:86ac6cee19f6c945e8dedf78c6ee74f1112bd14da5a658d8c9d4103aed5756a2", 452 | "sha256:cc0ea0de82b240a73ef4056fce44acbb4727dca7d66759371aff2bad457ed711" 453 | ], 454 | "version": "==0.26" 455 | }, 456 | "pygments": { 457 | "hashes": [ 458 | "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b", 459 | "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe" 460 | ], 461 | "version": "==2.5.2" 462 | }, 463 | "pymysql": { 464 | "hashes": [ 465 | "sha256:3943fbbbc1e902f41daf7f9165519f140c4451c179380677e6a848587042561a", 466 | "sha256:d8c059dcd81dedb85a9f034d5e22dcb4442c0b201908bede99e306d65ea7c8e7" 467 | ], 468 | "version": "==0.9.3" 469 | }, 470 | "pyopenssl": { 471 | "hashes": [ 472 | "sha256:26ff56a6b5ecaf3a2a59f132681e2a80afcc76b4f902f612f518f92c2a1bf854", 473 | "sha256:6488f1423b00f73b7ad5167885312bb0ce410d3312eb212393795b53c8caa580" 474 | ], 475 | "version": "==18.0.0" 476 | }, 477 | "pyparsing": { 478 | "hashes": [ 479 | "sha256:bc6c7146b91af3f567cf6daeaec360bc07d45ffec4cf5353f4d7a208ce7ca30a", 480 | "sha256:d29593d8ebe7b57d6967b62494f8c72b03ac0262b1eed63826c6f788b3606401" 481 | ], 482 | "version": "==2.2.2" 483 | }, 484 | "pyperclip": { 485 | "hashes": [ 486 | "sha256:406bc020d4b8e60d8673876271b815befc4c02fd8d919e4aacc667d69fab99ea" 487 | ], 488 | "version": "==1.6.5" 489 | }, 490 | "redis": { 491 | "hashes": [ 492 | "sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62", 493 | "sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2" 494 | ], 495 | "version": "==3.3.11" 496 | }, 497 | "requests": { 498 | "hashes": [ 499 | "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", 500 | "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" 501 | ], 502 | "version": "==2.22.0" 503 | }, 504 | "retry": { 505 | "hashes": [ 506 | "sha256:ccddf89761fa2c726ab29391837d4327f819ea14d244c232a1d24c67a2f98606", 507 | "sha256:f8bfa8b99b69c4506d6f5bd3b0aabf77f98cdb17f3c9fc3f5ca820033336fba4" 508 | ], 509 | "version": "==0.9.2" 510 | }, 511 | "ruamel.yaml": { 512 | "hashes": [ 513 | "sha256:08aaaa74ff66565024ecabf9ba2db212712382a21c0458f9a91c623a1fa83b34", 514 | "sha256:23f2efb872d2ebe3d5428b4f1a8f30cbf59f56e780c4981c155411ee65572673", 515 | "sha256:38718e69270141c403b5fc539f774ed394568f8a5195b507991f5b690356facb", 516 | "sha256:44da2be1153e173f90ad8775d4ac4237a3c06cfbb9660c1c1980271621833faa", 517 | "sha256:4b1674a936cdae9735578d4fd64bcbc6cfbb77a1a8f7037a50c6e3874ba4c9d8", 518 | "sha256:51d49c870aca850e652e2cd1c9bea9b52b77d13ad52b0556de496c1d264ea65f", 519 | "sha256:63dc8c6147a4cf77efadf2ae0f34e89e03de79289298bb941b7ae333d5d4020b", 520 | "sha256:6672798c6b52a976a7b24e20665055852388c83198d88029d3c76e2197ac221a", 521 | "sha256:6b6025f9b6a557e15e9fdfda4d9af0b57cd8d59ff98e23a0097ab2d7c0540f07", 522 | "sha256:7b750252e3d1ec5b53d03be508796c04a907060900c7d207280b7456650ebbfc", 523 | "sha256:847177699994f9c31adf78d1ef1ff8f069ef0241e744a3ee8b30fbdaa914cc1e", 524 | "sha256:8e42f3067a59e819935a2926e247170ed93c8f0b2ab64526f888e026854db2e4", 525 | "sha256:922d9e483c05d9000256640026f277fcc0c2e1e9271d05acada8e6cfb4c8b721", 526 | "sha256:92a8ca79f9173cca29ca9663b49d9c936aefc4c8a76f39318b0218c8f3626438", 527 | "sha256:ab8eeca4de4decf0d0a42cb6949d354da9fc70a2d9201f0dd55186c599b2e3a5", 528 | "sha256:bd4b60b649f4a81086f70cd56eff4722018ef36a28094c396f1a53bf450bd579", 529 | "sha256:fc6471ef15b69e454cca82433ac5f84929d9f3e2d72b9e54b06850b6b7133cc0", 530 | "sha256:ffc89770339191acbe5a15041950b5ad9daec7d659619b0ed9dad8c9c80c26f3" 531 | ], 532 | "version": "==0.15.100" 533 | }, 534 | "six": { 535 | "hashes": [ 536 | "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", 537 | "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" 538 | ], 539 | "version": "==1.13.0" 540 | }, 541 | "sortedcontainers": { 542 | "hashes": [ 543 | "sha256:220bb2e3e1886297fd7cdd6d164cb5cf237be1cfae1a3a3e526d149c52816682", 544 | "sha256:b74f2756fb5e23512572cc76f0fe0832fd86310f77dfee54335a35fb33f6b950" 545 | ], 546 | "version": "==2.0.5" 547 | }, 548 | "sqlalchemy": { 549 | "hashes": [ 550 | "sha256:afa5541e9dea8ad0014251bc9d56171ca3d8b130c9627c6cb3681cff30be3f8a" 551 | ], 552 | "version": "==1.3.11" 553 | }, 554 | "termcolor": { 555 | "hashes": [ 556 | "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b" 557 | ], 558 | "version": "==1.1.0" 559 | }, 560 | "tornado": { 561 | "hashes": [ 562 | "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d", 563 | "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409", 564 | "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f", 565 | "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f", 566 | "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5", 567 | "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb", 568 | "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" 569 | ], 570 | "version": "==5.1.1" 571 | }, 572 | "traitlets": { 573 | "hashes": [ 574 | "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44", 575 | "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7" 576 | ], 577 | "version": "==4.3.3" 578 | }, 579 | "uiautomator2": { 580 | "hashes": [ 581 | "sha256:e597014c5c34b3bff35e561b8135572c4205ba4d42d633bf2637f1b07cfb21d0" 582 | ], 583 | "version": "==2.3.2" 584 | }, 585 | "urllib3": { 586 | "hashes": [ 587 | "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", 588 | "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" 589 | ], 590 | "version": "==1.25.7" 591 | }, 592 | "urwid": { 593 | "hashes": [ 594 | "sha256:644d3e3900867161a2fc9287a9762753d66bd194754679adb26aede559bcccbc" 595 | ], 596 | "version": "==2.0.1" 597 | }, 598 | "wcwidth": { 599 | "hashes": [ 600 | "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", 601 | "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" 602 | ], 603 | "version": "==0.1.7" 604 | }, 605 | "weditor": { 606 | "hashes": [ 607 | "sha256:4a126abce472ec1b9c795eb9c83a02757af84bb1d6d468d3487619b9f16faaff", 608 | "sha256:4ba83bffbb7a9cb580adb60ecb191bfdc0a0a9e0cb6eaaf3f870285129609abd" 609 | ], 610 | "version": "==0.4.2" 611 | }, 612 | "whichcraft": { 613 | "hashes": [ 614 | "sha256:acdbb91b63d6a15efbd6430d1d7b2d36e44a71697e93e19b7ded477afd9fce87", 615 | "sha256:deda9266fbb22b8c64fd3ee45c050d61139cd87419765f588e37c8d23e236dd9" 616 | ], 617 | "version": "==0.6.1" 618 | }, 619 | "wrapt": { 620 | "hashes": [ 621 | "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" 622 | ], 623 | "version": "==1.11.2" 624 | }, 625 | "wsproto": { 626 | "hashes": [ 627 | "sha256:02f214f6bb43cda62a511e2e8f1d5fa4703ed83d376d18d042bd2bbf2e995824", 628 | "sha256:d2a7f718ab3144ec956a3267d57b5c172f0668827f5803e7d670837b0125b9fa" 629 | ], 630 | "version": "==0.11.0" 631 | }, 632 | "xmltodict": { 633 | "hashes": [ 634 | "sha256:50d8c638ed7ecb88d90561beedbf720c9b4e851a9fa6c47ebd64e99d166d8a21", 635 | "sha256:8bbcb45cc982f48b2ca8fe7e7827c5d792f217ecf1792626f808bf41c3b86051" 636 | ], 637 | "version": "==0.12.0" 638 | } 639 | }, 640 | "develop": {} 641 | } 642 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # app crawler 2 | 3 | crawling app by uiautomator2 & mitmproxy 4 | 5 | 使用 URL_SCHEMA 跳转实现抖音用户视频和视频评论的抓取 6 | 7 | ```python3 8 | URL_SCHEMA_MAP = { 9 | 'home': "snssdk1128://feed?refer=web", 10 | 'user': 'snssdk1128://user/profile/{uid}?refer=web', 11 | 'detail': 'snssdk1128://aweme/detail/{aweme_id}?refer=web', 12 | 'challenge': 'snssdk1128://challenge/detail/{challenge_id}?refer=web', 13 | 'music': 'snssdk1128://music/detail/{music_id}?refer=web', 14 | 'live': 'snssdk1128://live?room_id={room_id}&user_id={user_id}&from=webview&refer=web', 15 | 'poi":': 'snssdk1128://poi/?id={poi_id}', 16 | 'webview': 'snssdk1128://webview?url={url}&from=webview&refer=web', 17 | 'webview_fullscreen': 'snssdk1128://webview?url={url}&from=webview&hide_nav_bar=1&refer=web', 18 | 'poidetail': 'snssdk1128://poi/detail?id={id}&from=webview&refer=web', 19 | 'forward': 'snssdk1128://forward/detail/{id}', 20 | 'billboard_word': 'snssdk1128://search/trending', 21 | 'billboard_video': "snssdk1128://search/trending?type=1", 22 | 'billboard_music': "snssdk1128://search/trending?type=2", 23 | 'billboard_positive': "snssdk1128://search/trending?type=3", 24 | 'billboard_star': "snssdk1128://search/trending?type=4", 25 | } 26 | ``` 27 | 28 | - [uiautomator2](https://github.com/openatx/uiautomator2) 29 | - [浅谈自动化测试工具 python-uiautomator2](https://testerhome.com/topics/11357) 30 | 31 | ## 依赖安装 32 | 33 | 下载 Android platform-tools 并解压获取 `adb` 34 | - https://developer.android.com/studio/releases/platform-tools?hl=zh-Cn 35 | 36 | ```bash 37 | # 列出连接的设备(设备需开启`开发者选项`） 38 | adb devices 39 | ``` 40 | 41 | - [Android 调试桥 (adb)](https://developer.android.com/studio/command-line/adb?hl=zh-Cn) 42 | 43 | ```bash 44 | pipenv install 45 | pipenv shell 46 | uiautomator2 init 47 | ``` 48 | 49 | ## 抖音安装 50 | 51 | - 使用豌豆荚安装旧版抖音APP(v7.5.0以下版本仍然信任用户CA证书) 52 | 53 | ## weditor 54 | 55 | 使用web界面查看和定位元素 56 | ```bash 57 | python -m weditor 58 | 59 | ``` 60 | 61 | ## mitmproxy 62 | 63 | ### 安装和信任证书 64 | - https://docs.mitmproxy.org/stable/concepts-certificates/ 65 | 66 | ### 使用 67 | 68 | ```bash 69 | cp .env.tpl .env 70 | cp -r .mitmproxy ~/.mitmproxy 71 | make run-mitmproxy 72 | 73 | # 数据库启动 74 | make up 75 | 76 | # 导入测试数据 77 | ./app/tools/simple_data_import.py 78 | 79 | # 指定设备抓取用户信息和视频列表 80 | ./dy.py crawler_users --max_num=200 --device_serial=xxxxx 81 | 82 | # 指定设备抓取 83 | ./dy.py crawler_comments --device_serial=xxxxxxx 84 | 85 | # 指定设备抓取用户粉丝 86 | ./dy.py crawler_follower --device_serial=72bf965 87 | 88 | # 多设备 抓取用户粉丝 89 | ./crawler.py crawler_follower --max_num=200 90 | 91 | # 多设备 抓取用户信息、评论 92 | ./crawler.py run 93 | 94 | ``` 95 | 96 | ### 部署机器进程管理 97 | 98 | ```bash 99 | sudo cp frp/systemd/app-crawler.service /etc/systemd/system/ 100 | sudo systemctl daemon-reload 101 | sudo systemctl start app-crawler.service 102 | 103 | # 重启服务进程 104 | sudo systemctl restart app-crawler.service 105 | 106 | # 开机自启动 107 | sudo systemctl enable app-crawler.service 108 | ``` 109 | 110 | 111 | ## 常见问题 112 | 113 | 1. 找不到设备 114 | 115 | ```bash 116 | adb kill-server 117 | adb start-server 118 | ``` 119 | 还是不行，重启手机试试 120 | 121 | 2. `adb devices` 出现 `no permissions (user in plugdev group; are your udev rules wrong?)` 122 | - [adb-devices-no-permissions-user-in-plugdev-group-are-your-udev-rules-wrong](https://stackoverflow.com/questions/53887322/adb-devices-no-permissions-user-in-plugdev-group-are-your-udev-rules-wrong) 123 | 124 | 3. `weditor` 打开时出现 `adbutils.errors.AdbError: device not found` 125 | 更换设备会出现，需要清理 Chrome 的 LocalStorage 126 | - [openatx/weditor/issues/57](https://github.com/openatx/weditor/issues/57) 127 | 128 | 4. 测试机型 129 | 130 | - Xiaomi Mi 6 131 | - Redmi Note 8 132 | -------------------------------------------------------------------------------- /app/driver/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maguowei/app-crawler/5a8211671efc86d01f72c9bfdb4543aff135c5d7/app/driver/__init__.py -------------------------------------------------------------------------------- /app/driver/base.py: -------------------------------------------------------------------------------- 1 | import time 2 | import sys 3 | import logging 4 | import uiautomator2 as u2 5 | 6 | FORMAT = "[%(process)d-%(filename)s:%(lineno)s - %(funcName)20s %(levelname)s] %(message)s" 7 | 8 | 9 | class BaseDriver: 10 | def __init__(self, device_serial=None): 11 | self.device_serial = str(device_serial) 12 | if self.device_serial: 13 | self.device = u2.connect(self.device_serial) 14 | else: 15 | self.device = u2.connect() 16 | self.session = None 17 | self.pkg_name = None 18 | self.activity = None 19 | self.size = self.device.window_size() 20 | self.width, self.height = self.size 21 | 22 | self.popup_list = [] 23 | self.break_list = [] 24 | 25 | # set logger 26 | self.logger = logging.getLogger(__name__) 27 | logging.basicConfig(format=FORMAT) 28 | self.logger.setLevel(logging.DEBUG) 29 | 30 | def app_start(self): 31 | self.device.app_start(self.pkg_name, self.activity) 32 | self.session = self.device.session(self.pkg_name, attach=True) 33 | 34 | def app_close(self): 35 | self.device.app_stop(self.pkg_name) 36 | 37 | def open_schema(self, url_schema): 38 | self.device.shell(f'am start -W -a android.intent.action.VIEW -d {url_schema} {self.pkg_name}') 39 | 40 | def size(self): 41 | self.device.window_size() 42 | self.device.push_url() 43 | 44 | def get_source(self): 45 | return self.device.dump_hierarchy() 46 | 47 | def get_current_package(self): 48 | pkg_name = self.device.info.get('currentPackageName') 49 | return pkg_name 50 | 51 | def get_current_activity(self): 52 | return self.device.app_current()['activity'] 53 | 54 | def do_forever(self, action, activity='', max_num=200, *args, **kwargs): 55 | num = 0 56 | """TODO 更改为装饰器 57 | """ 58 | while True: 59 | num += 1 60 | if num > max_num: 61 | break 62 | action_name = action.__name__ 63 | self.logger.info(f'{self.device_serial}: {action_name} action to do...; num: {num}') 64 | # 被封禁，稍后再重试 65 | retry_text = '加载失败，点击重试' 66 | self.click_if_exist(text=retry_text) 67 | if self.exists(text=retry_text): 68 | self.logger.info('被封禁，稍后再重试') 69 | # time.sleep(60 * 2) 70 | sys.exit() 71 | if self.is_app_alive(): 72 | # print(self.get_source()) 73 | self.close_popup() 74 | if self.is_need_break(): 75 | break 76 | action(*args, **kwargs) 77 | if activity and self.get_current_activity() != activity: 78 | self.logger.info( 79 | f'{self.device_serial}: {action_name} 跳出了页面; current_activity: {self.get_current_activity()}' 80 | ) 81 | break 82 | # time.sleep(random.randint(1, 5)/10) 83 | self.logger.info(f'{action_name} action done!') 84 | else: 85 | self.close_app() 86 | self.click_if_exist(text="确认") 87 | time.sleep(2) 88 | self.app_start() 89 | # TODO 状态记忆和恢复 90 | self.logger.info(f'{self.device_serial}: {action_name} 状态异常导致应用重启') 91 | 92 | def swipe(self, *args): 93 | self.device.swipe(*args) 94 | 95 | def swipe_right(self, t=0.1, delay=1): 96 | self.device.swipe(0.1, 0.5, 0.9, 0.5, t) 97 | time.sleep(delay) 98 | 99 | def swipe_left(self, t=0.1, delay=1): 100 | self.device.swipe(0.9, 0.5, 0.1, 0.5, t) 101 | time.sleep(delay) 102 | 103 | def swipe_up(self, t=0.02, delay=0.5): 104 | self.device.swipe(self.width * 0.5, self.height * 0.8, self.width * 0.5, self.height * 0.3, t) 105 | time.sleep(delay) 106 | 107 | def swipe_down(self, t=0.02, delay=0.5): 108 | self.device.swipe(self.width * 0.5, self.height * 0.1, self.width * 0.5, self.height * 0.8, t) 109 | time.sleep(delay) 110 | 111 | def scroll(self): 112 | self.device(scrollable=True).scroll(steps=10) 113 | 114 | def fling(self): 115 | self.device(scrollable=True).fling() 116 | # self.device.wait_activity() 117 | 118 | def exists(self, **kwargs): 119 | return self.device(**kwargs).exists 120 | 121 | def click_if_exist(self, **kwargs): 122 | d = self.device(**kwargs) 123 | if d.exists: 124 | try: 125 | d.click(timeout=2) 126 | except Exception as e: 127 | self.logger.info(e) 128 | 129 | def close_popup(self): 130 | for text in self.popup_list: 131 | try: 132 | self.click_if_exist(text=text) 133 | except Exception: 134 | self.logger.info(f'close_popup: {text}') 135 | 136 | def is_need_break(self): 137 | for text in self.break_list: 138 | try: 139 | if self.exists(text=text): 140 | print(text) 141 | return True 142 | except Exception: 143 | self.logger.info(f'need_break: {text}') 144 | return False 145 | 146 | def is_app_alive(self): 147 | if self.get_current_package() != self.pkg_name: 148 | self.logger.info('应用已经退出') 149 | return False 150 | else: 151 | return True 152 | 153 | def close_app(self): 154 | if self.session: 155 | self.session.close() 156 | else: 157 | self.logger.info('Please open_app first') 158 | -------------------------------------------------------------------------------- /app/driver/douyin.py: -------------------------------------------------------------------------------- 1 | import time 2 | from app.driver.base import BaseDriver 3 | from app.utils.decorator import retry 4 | from app.models import session 5 | from app.models.douyin import User 6 | from app.service.redis_service import DouyinTopVideo, DouyinUserBigV 7 | 8 | 9 | class DouyinDriver(BaseDriver): 10 | 11 | # https://s3.pstatp.com/ies/resource/falcon/douyin_falcon/pkg/common_4276a6c.js 12 | URL_SCHEMA_MAP = { 13 | 'home': "snssdk1128://feed?refer=web", 14 | 'user': 'snssdk1128://user/profile/{uid}?refer=web', 15 | 'detail': 'snssdk1128://aweme/detail/{aweme_id}?refer=web', 16 | 'challenge': 'snssdk1128://challenge/detail/{challenge_id}?refer=web', 17 | 'music': 'snssdk1128://music/detail/{music_id}?refer=web', 18 | 'live': 'snssdk1128://live?room_id={room_id}&user_id={user_id}&from=webview&refer=web', 19 | 'poi":': 'snssdk1128://poi/?id={poi_id}', 20 | 'webview': 'snssdk1128://webview?url={url}&from=webview&refer=web', 21 | 'webview_fullscreen': 'snssdk1128://webview?url={url}&from=webview&hide_nav_bar=1&refer=web', 22 | 'poidetail': 'snssdk1128://poi/detail?id={id}&from=webview&refer=web', 23 | 'forward': 'snssdk1128://forward/detail/{id}', 24 | 'billboard_word': 'snssdk1128://search/trending', 25 | 'billboard_video': "snssdk1128://search/trending?type=1", 26 | 'billboard_music': "snssdk1128://search/trending?type=2", 27 | 'billboard_positive': "snssdk1128://search/trending?type=3", 28 | 'billboard_star': "snssdk1128://search/trending?type=4", 29 | } 30 | 31 | def __init__(self, device_serial=None): 32 | super().__init__(device_serial) 33 | self.pkg_name = 'com.ss.android.ugc.aweme' 34 | self.activity = '.main.MainActivity' 35 | self.popup_list = ['以后再说', '取消', '我知道了', '暂不', '同意', '不允许', '长按屏幕使用更多功能', '确定'] 36 | self.break_list = ['没有更多了', '没有更多了~', 'TA还没有关注任何人', '暂时没有更多了', '该用户还没有发布过作品', '上拉加载更多'] 37 | self.app_start() 38 | 39 | def open_user_home(self, uid): 40 | """ 41 | :param uid: 58958068057 42 | :return: 43 | """ 44 | self.open_schema(self.URL_SCHEMA_MAP['user'].format(uid=uid)) 45 | 46 | def open_tag(self, cid): 47 | """ 48 | 49 | :param cid: 1643291726734347 50 | :return: 51 | """ 52 | self.open_schema(self.URL_SCHEMA_MAP['challenge'].format(challenge_id=cid)) 53 | 54 | def open_video(self, aweme_id): 55 | """ 56 | :param aweme_id: 6747930437261298951 57 | :return: 58 | """ 59 | self.open_schema(self.URL_SCHEMA_MAP['detail'].format(aweme_id=aweme_id)) 60 | 61 | @retry(10) 62 | def crawler_users(self, max_num=200): 63 | """抓取大v用户信息 64 | """ 65 | while DouyinUserBigV.nums(): 66 | self.app_close() 67 | self.app_start() 68 | time.sleep(0.2) 69 | uid = DouyinUserBigV.pop_min() 70 | self.crawler_user(uid, max_num=max_num) 71 | self.app_close() 72 | else: 73 | # user import 74 | pass 75 | 76 | @retry(3) 77 | def crawler_user(self, uid, max_num=200): 78 | self.logger.info(f'爬取用户信息: {uid}') 79 | self.open_user_home(uid) 80 | time.sleep(0.2) 81 | self.close_popup() 82 | # 添加关注 83 | # self.session(resourceId='com.ss.android.ugc.aweme:id/c6v').click_exists(timeout=1) 84 | time.sleep(0.1) 85 | self.session(textContains='作品').click() 86 | self.do_forever(self.fling, activity='com.ss.android.ugc.aweme.profile.ui.UserProfileActivity', max_num=max_num) 87 | # self.device.press('back') 88 | 89 | @retry(3) 90 | def crawler_comments(self): 91 | while DouyinTopVideo.nums(): 92 | aweme_id, _ = DouyinTopVideo.pop_max() 93 | self.crawler_comment(aweme_id) 94 | 95 | @retry(3) 96 | def crawler_comment(self, aweme_id): 97 | self.app_close() 98 | self.app_start() 99 | self.logger.info(f'爬取视频评论: {aweme_id}') 100 | self.open_video(aweme_id) 101 | time.sleep(0.2) 102 | self.close_popup() 103 | self.session(resourceId="com.ss.android.ugc.aweme:id/v3").click() 104 | time.sleep(0.2) 105 | for i in range(150): 106 | # self.swipe_up() 107 | self.fling() 108 | self.logger.info(f'fling: {i}') 109 | time.sleep(0.1) 110 | if self.exists(text='暂时没有更多了'): 111 | break 112 | # if self.exists(text='暂无评论，来抢沙发'): 113 | # break 114 | self.app_close() 115 | 116 | @retry(10) 117 | def crawler_following(self): 118 | """抓取用户关注列表 119 | """ 120 | time.sleep(2) 121 | users = session.query(User.uid) 122 | for user in users: 123 | uid = user['uid'] 124 | self.logger.info(f'爬取用户关注列表: {uid}') 125 | self.open_schema(self.URL_SCHEMA_MAP['user'].format(uid=uid)) 126 | time.sleep(0.2) 127 | if self.session(resourceId="com.ss.android.ugc.aweme:id/bq3").exists: # 用户昵称 128 | # if self.session(text='这是私密帐号').exists: # 跳过私密账号 129 | # continue 130 | self.session(resourceId='com.ss.android.ugc.aweme:id/ah1').click() # 关注列表按钮 131 | self.do_forever(self.fling) 132 | else: 133 | self.logger.info(f'用户异常: {uid}') 134 | self.device.press('back') 135 | time.sleep(0.2) 136 | 137 | @retry(10) 138 | def crawler_follower(self, max_num=200): 139 | """抓取用户粉丝列表 140 | """ 141 | while DouyinUserBigV.nums(): 142 | self.app_close() 143 | self.app_start() 144 | time.sleep(0.2) 145 | uid = DouyinUserBigV.pop_max() 146 | self.logger.info(f'爬取用户粉丝列表: {uid}') 147 | self.open_schema(self.URL_SCHEMA_MAP['user'].format(uid=uid)) 148 | time.sleep(0.5) 149 | if self.session(resourceId='com.ss.android.ugc.aweme:id/ahb').exists: 150 | self.session(text='粉丝').click() 151 | time.sleep(0.5) 152 | self.do_forever(self.fling, activity='com.ss.android.ugc.aweme.following.ui.FollowRelationTabActivity', 153 | max_num=max_num) 154 | else: 155 | self.logger.info(f'用户异常: {uid}') 156 | self.app_close() 157 | -------------------------------------------------------------------------------- /app/event.py: -------------------------------------------------------------------------------- 1 | import json 2 | import mitmproxy.http 3 | import mitmproxy.proxy.protocol 4 | from app.models.douyin import User, Video, Comment 5 | from app.service.redis_service import DouyinUser 6 | 7 | 8 | class Events: 9 | def response(self, flow: mitmproxy.http.HTTPFlow): 10 | response = flow.response 11 | content = response.text 12 | url = flow.request.url 13 | # print(url) 14 | 15 | # 粉丝关注列表（可获取少于5000条) 16 | if '/aweme/v1/user/follower/list/' in url: 17 | data = json.loads(content) 18 | if data['status_code'] == 0 and data['has_more'] == 1: 19 | uid = flow.request.query.get('user_id') 20 | for follower in data['followers']: 21 | follow_uid = follower['uid'] 22 | follow_data = { 23 | 'uid': uid, 24 | 'follow_uid': follow_uid, 25 | 'sex': follower['gender'], 26 | 'birthday': follower['birthday'], 27 | 'city': '', 28 | } 29 | # db['douyin_follow'].replace_one({'uid': uid, 'follow_uid': follow_uid}, follow_data, upsert=True) 30 | # todo 存储 31 | pass 32 | 33 | # 用户关注列表 34 | elif '/aweme/v1/user/following/list/' in url: 35 | data = json.loads(content) 36 | if data['status_code'] == 0 and data['has_more']: 37 | for following in data['followings']: 38 | uid = following['uid'] 39 | # DouyinUser.add(uid) 40 | 41 | # 用户信息 42 | elif '/aweme/v1/user/' in url: 43 | data = json.loads(content) 44 | if data['status_code'] == 0: 45 | user = data['user'] 46 | user_data = { 47 | 'uid': user['uid'], 48 | 'short_id': user['short_id'], 49 | 'unique_id': user['unique_id'], 50 | 'nickname': user['nickname'], 51 | 'signature': user['signature'], 52 | 'custom_verify': user['custom_verify'], 53 | 'gender': user['gender'], 54 | 'school_name': user.get('school_name', ''), 55 | 'avatar_uri': user['avatar_uri'], 56 | 'share_qrcode_uri': user['share_qrcode_uri'], 57 | 'birthday': user['birthday'], 58 | 'region': user['region'], 59 | 'country': user['country'], 60 | 'province': user['province'], 61 | 'city': user['city'], 62 | 'is_verified': user['is_verified'], 63 | 'verify_info': user['verify_info'], 64 | 'is_star': user['is_star'], 65 | 'room_id': user['room_id'], 66 | 'aweme_count': user['aweme_count'], 67 | 'following_count': user['following_count'], 68 | 'favoriting_count': user['favoriting_count'], 69 | 'total_favorited': user['total_favorited'], 70 | 'dongtai_count': user['dongtai_count'], 71 | 'follower_count': user['follower_count'], 72 | 'is_gov_media_vip': user['is_gov_media_vip'], 73 | 'followers_detail': user['followers_detail'], 74 | } 75 | User(**user_data).save() 76 | 77 | # 用户视频列表 78 | elif '/aweme/v1/aweme/post/' in url: 79 | if content: 80 | data = json.loads(content) 81 | if data['status_code'] == 0: 82 | aweme_list = data['aweme_list'] 83 | for detail in aweme_list: 84 | Video(data=detail).save() 85 | else: 86 | print('返回空') 87 | 88 | # 评论列表 (似乎没有限制条数) 89 | elif '/aweme/v2/comment/list' in url: 90 | data = json.loads(content) 91 | if data['status_code'] == 0 and data['has_more'] == 1: 92 | for comment in data['comments']: 93 | cid = comment['cid'] 94 | user = comment['user'] 95 | 96 | comment = { 97 | 'cid': cid, 98 | 'aweme_id': comment['aweme_id'], 99 | 'uid': user['uid'], 100 | 'avatar_url': user['avatar_medium']['url_list'][0], 101 | 'nickname': user['nickname'], 102 | 'create_time': comment['create_time'], 103 | 'text': comment['text'], 104 | 'digg_count': comment['digg_count'], 105 | } 106 | 107 | Comment(data=comment).save() 108 | 109 | -------------------------------------------------------------------------------- /app/models/__init__.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import create_engine 2 | from sqlalchemy.orm import sessionmaker 3 | from app.settings import MYSQL 4 | from sqlalchemy.ext.declarative import as_declarative 5 | 6 | 7 | engine = create_engine('mysql+pymysql://{user}:{password}@{host}:{port}/{db}'.format(**MYSQL), echo=True) 8 | 9 | Session = sessionmaker(bind=engine) 10 | session = Session() 11 | 12 | 13 | @as_declarative() 14 | class Base: 15 | def save(self): 16 | session.add(self) 17 | session.commit() 18 | 19 | def delete(self): 20 | session.delete(self) 21 | session.commit() 22 | -------------------------------------------------------------------------------- /app/models/douyin.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from sqlalchemy import Column, Integer, JSON, String, Boolean, DateTime, SMALLINT 3 | from app.models import engine 4 | from app.models import Base 5 | 6 | 7 | class User(Base): 8 | __tablename__ = 'user' 9 | 10 | id = Column(Integer, primary_key=True) 11 | uid = Column(String(50), index=True) 12 | short_id = Column(String(50), nullable=False, default='') 13 | unique_id = Column(String(50), nullable=False, default='') 14 | nickname = Column(String(50), nullable=False, default='') 15 | signature = Column(String(200), nullable=False, default='') 16 | custom_verify = Column(String(200), nullable=False, default='') 17 | gender = Column(SMALLINT, nullable=False, default=0) # 0 未知, 1 男, 2 女 18 | school_name = Column(String(50), nullable=False, default='') 19 | avatar_uri = Column(String(100), nullable=False, default='') 20 | share_qrcode_uri = Column(String(100), nullable=False, default='') 21 | birthday = Column(String(100), nullable=False, default='') 22 | region = Column(String(50), nullable=False, default='') 23 | country = Column(String(20), nullable=False, default='') 24 | province = Column(String(20), nullable=False, default='') 25 | city = Column(String(50), nullable=False, default='') 26 | is_verified = Column(Boolean, default=False) 27 | verify_info = Column(String(100), nullable=False, default='') 28 | is_star = Column(Boolean, default=False) 29 | room_id = Column(String(50), nullable=False, default='') 30 | aweme_count = Column(Integer, index=True) 31 | following_count = Column(Integer, index=True) 32 | favoriting_count = Column(Integer, index=True) 33 | total_favorited = Column(Integer, index=True) 34 | dongtai_count = Column(Integer, index=True) 35 | follower_count = Column(Integer, index=True) 36 | is_gov_media_vip = Column(Boolean, default=False) 37 | followers_detail = Column(JSON) 38 | create_time = Column(DateTime, default=datetime.datetime.now) 39 | update_time = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now) 40 | 41 | 42 | class Video(Base): 43 | __tablename__ = 'video' 44 | id = Column(Integer, primary_key=True) 45 | data = Column(JSON) 46 | create_time = Column(DateTime, default=datetime.datetime.now) 47 | update_time = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now) 48 | 49 | 50 | class Comment(Base): 51 | __tablename__ = 'comment' 52 | id = Column(Integer, primary_key=True) 53 | data = Column(JSON) 54 | create_time = Column(DateTime, default=datetime.datetime.now) 55 | update_time = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now) 56 | 57 | 58 | # Base.metadata.create_all(engine) 59 | -------------------------------------------------------------------------------- /app/service/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maguowei/app-crawler/5a8211671efc86d01f72c9bfdb4543aff135c5d7/app/service/__init__.py -------------------------------------------------------------------------------- /app/service/redis_service.py: -------------------------------------------------------------------------------- 1 | import redis 2 | from app import settings 3 | 4 | 5 | redis_client = redis.Redis(**settings.REDIS) 6 | 7 | 8 | class DouyinZsetBase: 9 | @classmethod 10 | def get_key(cls): 11 | raise NotImplementedError() 12 | 13 | @classmethod 14 | def add(cls, uid, score): 15 | """ 16 | :param uid: 17 | :param score: follow_count 18 | :return: 19 | """ 20 | key = cls.get_key() 21 | redis_client.zadd(key, {uid: score}) 22 | 23 | @classmethod 24 | def exist(cls, uid): 25 | return redis_client.zscore(cls.get_key(), uid) 26 | 27 | @classmethod 28 | def nums(cls): 29 | key = cls.get_key() 30 | return redis_client.zcard(key) 31 | 32 | @classmethod 33 | def pop_max(cls): 34 | values = redis_client.zrevrange(cls.get_key(), 0, 0) 35 | value = values[0] if values else None 36 | if value: 37 | redis_client.zrem(cls.get_key(), value) 38 | return value 39 | 40 | @classmethod 41 | def pop_min(cls): 42 | values = redis_client.zrange(cls.get_key(), 0, 0) 43 | value = values[0] if values else None 44 | if value: 45 | redis_client.zrem(cls.get_key(), value) 46 | return value 47 | 48 | @classmethod 49 | def export(cls): 50 | for uid, score in redis_client.zscan_iter(cls.get_key()): 51 | print(uid, score) 52 | 53 | def __contains__(self, item): 54 | value = redis_client.zscore(self.get_key(), item) 55 | return True if value else False 56 | 57 | def __str__(self): 58 | return f'{self.__class__.__name__}: {self.nums()}' 59 | 60 | 61 | class DouyinBase: 62 | @classmethod 63 | def get_key(cls): 64 | raise NotImplementedError() 65 | 66 | @classmethod 67 | def add(cls, uid): 68 | key = cls.get_key() 69 | redis_client.sadd(key, uid) 70 | 71 | @classmethod 72 | def pop(cls): 73 | return redis_client.spop(cls.get_key()) 74 | 75 | @classmethod 76 | def remove(cls, uid): 77 | key = cls.get_key() 78 | redis_client.srem(key, uid) 79 | 80 | @classmethod 81 | def nums(cls): 82 | key = cls.get_key() 83 | return redis_client.scard(key) 84 | 85 | @classmethod 86 | def export(cls): 87 | for uid in redis_client.sscan_iter(cls.get_key()): 88 | print(uid) 89 | 90 | def __contains__(self, item): 91 | return redis_client.sismember(self.get_key(), item) 92 | 93 | def __str__(self): 94 | return f'{self.__class__.__name__}: {self.nums()}' 95 | 96 | 97 | class DouyinUser(DouyinBase): 98 | """待处理抖音用户""" 99 | @classmethod 100 | def get_key(cls): 101 | return 'dy:user' 102 | 103 | 104 | class DouyinUserBigV(DouyinZsetBase): 105 | """抖音大V用户池""" 106 | @classmethod 107 | def get_key(cls): 108 | return 'dy:user:score' 109 | 110 | 111 | class DouyinUserFollowing(DouyinBase): 112 | """已经爬取粉丝的用户""" 113 | @classmethod 114 | def get_key(cls): 115 | return 'dy:user_following' 116 | 117 | 118 | class DouyinUserFollower(DouyinBase): 119 | """已经爬取关注的用户""" 120 | @classmethod 121 | def get_key(cls): 122 | return 'dy:user_follower' 123 | 124 | 125 | class DouyinTopVideo(DouyinZsetBase): 126 | """高赞视频""" 127 | @classmethod 128 | def get_key(cls): 129 | return 'dy:video_top' 130 | 131 | 132 | if __name__ == '__main__': 133 | print(DouyinUser()) 134 | print(DouyinUserBigV()) 135 | print(DouyinUserFollower()) 136 | print(DouyinUserFollowing()) 137 | print(DouyinTopVideo()) 138 | # DouyinUserBigV.export() 139 | -------------------------------------------------------------------------------- /app/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | # Redis 4 | REDIS = { 5 | 'host': os.environ.get('REDIS_HOST', '127.0.0.1'), 6 | 'port': os.environ.get('REDIS_PORT', 6379), 7 | 'db': os.environ.get('REDIS_DB', 0), 8 | 'decode_responses': True, 9 | } 10 | 11 | # MySQL 12 | MYSQL = { 13 | 'host': os.environ.get('DB_HOST', '127.0.0.1'), 14 | 'port': os.environ.get('DB_PORT', 3306), 15 | 'user': os.environ.get('DB_USER', 'root'), 16 | 'password': os.environ.get('DB_PASSWORD', 'root'), 17 | 'db': 'crawler', 18 | } 19 | -------------------------------------------------------------------------------- /app/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maguowei/app-crawler/5a8211671efc86d01f72c9bfdb4543aff135c5d7/app/tools/__init__.py -------------------------------------------------------------------------------- /app/tools/simple_data_import.py: -------------------------------------------------------------------------------- 1 | """导入测试数据 2 | """ 3 | 4 | from app.service.redis_service import DouyinUserBigV, DouyinTopVideo 5 | 6 | # 测试数据 7 | uids = ['84990209480', '88445518961', '104255897823', '76055758243', '6556303280', '80812090202'] 8 | video_ids = ['6768618740750748936', '6768648652828249355', '6768653399891217668', '6768647119449394435'] 9 | 10 | 11 | if __name__ == '__main__': 12 | for uid in uids: 13 | DouyinUserBigV.add(uid, 1) 14 | 15 | for video_id in video_ids: 16 | DouyinTopVideo.add(video_id, 1) 17 | -------------------------------------------------------------------------------- /app/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maguowei/app-crawler/5a8211671efc86d01f72c9bfdb4543aff135c5d7/app/utils/__init__.py -------------------------------------------------------------------------------- /app/utils/decorator.py: -------------------------------------------------------------------------------- 1 | def retry(times=3): 2 | def decorator(func): 3 | def wrapper(*args, **kwargs): 4 | t = 0 5 | while t <= times: 6 | try: 7 | return func(*args, **kwargs) 8 | except Exception as e: 9 | print(f'Exception: {e}, times: {t}') 10 | t += 1 11 | return wrapper 12 | return decorator 13 | 14 | 15 | @retry(times=3) 16 | def test_retry(): 17 | raise Exception 18 | 19 | 20 | if __name__ == '__main__': 21 | test_retry() 22 | -------------------------------------------------------------------------------- /conf/systemd/app-crawler.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=App Crawler Service 3 | 4 | [Service] 5 | Type=simple 6 | User=nobody 7 | Restart=always 8 | RuntimeMaxSec=3600 9 | ExecStart=python3 crawler.py run 10 | 11 | [Install] 12 | WantedBy=multi-user.target 13 | -------------------------------------------------------------------------------- /crawler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | 4 | import fire 5 | import random 6 | from concurrent.futures import ThreadPoolExecutor, as_completed 7 | import adbutils 8 | from app.driver.douyin import DouyinDriver 9 | 10 | 11 | class DeviceManager: 12 | def __init__(self): 13 | adb = adbutils.AdbClient(host="127.0.0.1", port=5037) 14 | self.devices = adb.device_list() 15 | self.drivers = [] 16 | for device in self.devices: 17 | self.drivers.append(DouyinDriver(device.serial)) 18 | 19 | def run(self): 20 | with ThreadPoolExecutor(max_workers=len(self.drivers)) as executor: 21 | driver_for_comment = random.choice(self.drivers) 22 | self.drivers.remove(driver_for_comment) 23 | future_to_driver = {executor.submit(driver.crawler_users, 1): driver for driver in self.drivers} 24 | executor.submit(driver_for_comment.crawler_comments) 25 | for future in as_completed(future_to_driver): 26 | driver = future_to_driver[future] 27 | executor.submit(driver.crawler_comments) 28 | 29 | def crawler_follower(self, max_num=200): 30 | with ThreadPoolExecutor(max_workers=len(self.drivers)) as executor: 31 | for driver in self.drivers: 32 | executor.submit(driver.crawler_follower, max_num) 33 | 34 | 35 | if __name__ == '__main__': 36 | fire.Fire(DeviceManager) 37 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | mysql: 5 | image: mysql 6 | restart: always 7 | ports: 8 | - 3306:3306 9 | environment: 10 | - MYSQL_ROOT_PASSWORD=root 11 | - MYSQL_DATABASE=crawler 12 | volumes: 13 | - mysql:/var/lib/mysql 14 | command: 15 | - --character-set-server=utf8mb4 16 | - --collation-server=utf8mb4_unicode_ci 17 | 18 | redis: 19 | image: redis 20 | restart: always 21 | ports: 22 | - 6379:6379 23 | volumes: 24 | - redis:/data 25 | 26 | volumes: 27 | redis: 28 | mysql: -------------------------------------------------------------------------------- /dy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import fire 4 | from app.driver.douyin import DouyinDriver 5 | 6 | if __name__ == '__main__': 7 | fire.Fire(DouyinDriver) 8 | 9 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | from app.event import Events 2 | 3 | addons = [ 4 | Events() 5 | ] 6 | --------------------------------------------------------------------------------