├── .github ├── ISSUE_TEMPLATE │ └── 问题反馈.md └── workflows │ ├── arm-builder.yml │ ├── builder.yml │ ├── push-docker-manifest.yml │ └── release.yml ├── .gitignore ├── Aabyss-Bot.jpg ├── Disclaimer.md ├── LICENSE ├── LICENSE.md ├── README.md ├── Team.jpg ├── app ├── __init__.py ├── __main__.py ├── celerytask.py ├── config.py ├── config.yaml.example ├── dicts │ ├── altdnsdict.txt │ ├── black_asset_site.txt │ ├── blackdomain.txt │ ├── blackhexie.txt │ ├── cdn_info.json │ ├── dnsserver.txt │ ├── domain_2w.txt │ ├── domain_dict_test.txt │ ├── file_test.txt │ ├── file_top_200.txt │ ├── file_top_2000.txt │ ├── noscreenshot.jpg │ ├── webapp.json │ └── wih_rules.yml ├── helpers │ ├── __init__.py │ ├── asset_domain.py │ ├── asset_site.py │ ├── asset_site_monitor.py │ ├── asset_wih.py │ ├── asset_wih_monitor.py │ ├── domain.py │ ├── message_notify.py │ ├── policy.py │ ├── scheduler.py │ ├── scope.py │ ├── task.py │ ├── task_schedule.py │ └── url.py ├── main.py ├── modules │ ├── __init__.py │ ├── baseInfo.py │ ├── domainInfo.py │ ├── ipInfo.py │ ├── pageInfo.py │ └── wihRecord.py ├── routes │ ├── __init__.py │ ├── assetDomain.py │ ├── assetIP.py │ ├── assetScope.py │ ├── assetSite.py │ ├── assetWih.py │ ├── batchExport.py │ ├── cert.py │ ├── cip.py │ ├── console.py │ ├── domain.py │ ├── export.py │ ├── fileleak.py │ ├── fingerprint.py │ ├── github_monitor_result.py │ ├── github_result.py │ ├── github_scheduler.py │ ├── github_task.py │ ├── image.py │ ├── ip.py │ ├── npoc_service.py │ ├── nuclei_result.py │ ├── poc.py │ ├── policy.py │ ├── scheduler.py │ ├── service.py │ ├── site.py │ ├── stat_finger.py │ ├── task.py │ ├── taskFofa.py │ ├── task_schedule.py │ ├── url.py │ ├── user.py │ ├── vuln.py │ └── wih.py ├── scheduler.py ├── services │ ├── __init__.py │ ├── altDNS.py │ ├── asset_site_monitor.py │ ├── asset_wih_monitor.py │ ├── autoTag.py │ ├── baseThread.py │ ├── baseUpdateTask.py │ ├── buildDomainInfo.py │ ├── checkHTTP.py │ ├── commonTask.py │ ├── dns_query.py │ ├── dns_query_plugin │ │ ├── __init__.py │ │ ├── alienvault.py │ │ ├── certspotter.py │ │ ├── chaos.py │ │ ├── crtsh.py │ │ ├── fofa.py │ │ ├── hunter_qax.py │ │ ├── passivetotal.py │ │ ├── quake_360.py │ │ ├── rapiddns.py │ │ ├── securitytrails.py │ │ ├── virustotal.py │ │ └── zoomeye.py │ ├── domainSiteUpdate.py │ ├── expr.py │ ├── fetchCert.py │ ├── fetchSite.py │ ├── fileLeak.py │ ├── findVhost.py │ ├── fingerprint.py │ ├── fingerprint_cache.py │ ├── fofaClient.py │ ├── githubSearch.py │ ├── infoHunter.py │ ├── massdns.py │ ├── npoc.py │ ├── nuclei_scan.py │ ├── pageFetch.py │ ├── portScan.py │ ├── probeHTTP.py │ ├── resolverDomain.py │ ├── searchEngines.py │ ├── siteScreenshot.py │ ├── siteUrlSpider.py │ ├── syncAsset.py │ ├── webAnalyze.py │ └── webhook.py ├── tasks │ ├── __init__.py │ ├── asset_site.py │ ├── asset_wih.py │ ├── domain.py │ ├── github.py │ ├── ip.py │ ├── poc.py │ └── scheduler.py ├── tools │ ├── apps.json │ ├── driver.js │ ├── massdns │ ├── phantomjs │ ├── screenshot.js │ ├── targetGen.py │ └── wappalyzer.js └── utils │ ├── IPy.py │ ├── __init__.py │ ├── arl.py │ ├── arlupdate.py │ ├── cdn.py │ ├── cert.py │ ├── conn.py │ ├── cron.py │ ├── device.py │ ├── domain.py │ ├── fingerprint.py │ ├── github_task.py │ ├── http.py │ ├── ip.py │ ├── nmap.py │ ├── push.py │ ├── query_loader.py │ ├── time.py │ ├── url.py │ └── user.py ├── arl_tool ├── GithubLeak.py └── fileleakV2.py ├── docker ├── ARMWorker │ └── Dockerfile ├── config-docker.yaml ├── docker-compose.yml ├── frontend │ ├── css │ │ ├── app~d0ae3f07.4bf36d09.css │ │ ├── assetsMonitor~f71cff67.90e05005.css │ │ ├── chunk-367f8a7a.01e58279.css │ │ ├── chunk-7fa6e08b.42412e6b.css │ │ ├── chunk-a2febf90.c9ac60be.css │ │ ├── chunk-b4506206.e8a39534.css │ │ ├── groupAssetsDetail~f71cff67.69a66a2f.css │ │ ├── groupAssetsManagement~f71cff67.10c25a72.css │ │ ├── npm.ant-design-vue~301ae65c.698f9410.css │ │ ├── npm.ant-design-vue~4acf2f4a.ab870312.css │ │ ├── npm.ant-design-vue~5f8813c1.680b71d4.css │ │ ├── npm.ant-design-vue~731d2fff.1b7a0c24.css │ │ ├── npm.ant-design-vue~9a6beb85.3a1e913e.css │ │ ├── npm.ant-design-vue~ac50015d.5698995a.css │ │ ├── pocList~31ecd969.83581b2f.css │ │ ├── search~f71cff67.d188b78c.css │ │ ├── taskDetail~f71cff67.0ff6bde8.css │ │ └── taskList~f71cff67.1de399be.css │ ├── favicon.ico │ ├── index.html │ └── js │ │ ├── app~d0ae3f07.210036a0.js │ │ ├── assetsMonitor~f71cff67.169bf576.js │ │ ├── chunk-367f8a7a.d0a009a4.js │ │ ├── chunk-7fa6e08b.b27d6ea5.js │ │ ├── chunk-a2febf90.d19ba28d.js │ │ ├── chunk-b4506206.17ea8349.js │ │ ├── groupAssetsDetail~f71cff67.551ee1f9.js │ │ ├── groupAssetsManagement~f71cff67.adf08fed.js │ │ ├── npm.ant-design-vue~1a7f21e9.8e943d65.js │ │ ├── npm.ant-design-vue~301ae65c.74d1b758.js │ │ ├── npm.ant-design-vue~33d5c1c7.34a1fc3f.js │ │ ├── npm.ant-design-vue~4acf2f4a.d775f7f3.js │ │ ├── npm.ant-design-vue~5f8813c1.d2140e0d.js │ │ ├── npm.ant-design-vue~731d2fff.3c247f2e.js │ │ ├── npm.ant-design-vue~85ffde94.a113f60f.js │ │ ├── npm.ant-design-vue~9a6beb85.01c84af1.js │ │ ├── npm.ant-design-vue~ac50015d.7c330d84.js │ │ ├── npm.ant-design-vue~f99c446b.e6b72f64.js │ │ ├── npm.core-js~987e6011.b6479e3d.js │ │ ├── npm.lodash~2930ad93.97832723.js │ │ ├── npm.moment~0a56fd24.02de3ecf.js │ │ ├── npm.moment~0a56fd24.02de3ecf.js.LICENSE.txt │ │ ├── npm.vue~daa565d3.f8374a41.js │ │ ├── npm.vue~daa565d3.f8374a41.js.LICENSE.txt │ │ ├── pocList~31ecd969.5ff0f285.js │ │ ├── runtime.be27a04b.js │ │ ├── search~f71cff67.e94d4c0f.js │ │ ├── taskDetail~f71cff67.aa553549.js │ │ ├── taskList~f71cff67.163aa2af.js │ │ ├── vendors~app~253ae210.6ccf677e.js │ │ ├── vendors~app~678f84af.2f5e7f43.js │ │ ├── vendors~app~678f84af.2f5e7f43.js.LICENSE.txt │ │ ├── vendors~app~d939e436.211f1ac1.js │ │ └── vendors~app~d939e436.211f1ac1.js.LICENSE.txt ├── mongo-init.js ├── nginx.conf └── worker │ ├── Dockerfile │ ├── gen_crt.sh │ └── wait-for-it.sh ├── image ├── domain.png ├── github_monitor.png ├── login.png ├── monitor.png ├── policy.png ├── scan.png ├── site.png ├── task.png └── task_scheduler.png ├── misc ├── ADD-ARL-Finger.py ├── arl-scheduler.service ├── arl-web.service ├── arl-worker-github.service ├── arl-worker.service ├── arl.conf ├── finger.json ├── manage.sh ├── nginx.conf ├── setup-arl.sh └── setup-docker-arl.sh ├── requirements.txt ├── test ├── __init__.py ├── ip.py ├── main.py ├── scheduler.py ├── sync_asset.py ├── task.py ├── test_asset_site_monitor.py ├── test_asset_site_update.py ├── test_asset_wih_monitor.py ├── test_asset_wih_task.py ├── test_autotag.py ├── test_cdn.py ├── test_check_domain_black.py ├── test_doamin.py ├── test_domain_site_update.py ├── test_domain_task_search_engines_spider.py ├── test_expr.py ├── test_fetch_favicon.py ├── test_fetch_site.py ├── test_findvhost.py ├── test_finger.py ├── test_fofa.py ├── test_geoip2.py ├── test_github_cron.py ├── test_github_search.py ├── test_github_task.py ├── test_helper_domain.py ├── test_ip_excutor.py ├── test_normal_url.py ├── test_npoc.py ├── test_nuclei_scan.py ├── test_policy_edit.py ├── test_probe_http.py ├── test_proxy_url.py ├── test_query_plugin.py ├── test_risk_cruising.py ├── test_scan_port.py ├── test_scheduler.py ├── test_search_engines.py ├── test_site_spider_thread.py ├── test_task_helpers.py ├── test_util_gen_cip.py ├── test_utils.py ├── test_utils_ports.py ├── test_utils_push.py ├── test_webhook.py └── test_wih.py └── version.txt /.github/ISSUE_TEMPLATE/问题反馈.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 问题反馈 3 | about: 提交问题以帮助我们改进 4 | title: 问题反馈 5 | labels: '' 6 | assignees: 1c3z 7 | 8 | --- 9 | 10 | ## 描述问题 11 | 12 | 请在此处提供对问题的清晰而简洁的描述。 13 | 14 | 15 | ## 环境信息 16 | 17 | - 操作系统： 18 | - Docker版本： 19 | - 系统登录页面显示的 ARL 版本: 20 | 21 | 22 | ### 错误输出 23 | 24 | 请执行以下命令，并提供输出结果： 25 | 26 | `docker compose ps` 27 | 截图： 28 | 29 | 30 | `docker compose logs -f --tail=10` 31 | 截图： 32 | 33 | 34 | `tail -f arl_worker.log` 35 | 截图： 36 | 37 | 38 | 39 | ## 复现步骤 40 | 41 | 请提供重现问题所需的详细步骤。 42 | 43 | 1. 步骤一 44 | 2. 步骤二 45 | 3. 步骤三 46 | 47 | ## 预期行为 48 | 49 | 请描述您预期发生的事情。 50 | 51 | ## 实际行为 52 | 53 | 请描述实际发生的事情。 54 | 55 | 56 | ## 附加上下文 57 | 58 | 在此处提供其他有关问题的任何其他上下文。 59 | 60 | 61 | 62 | ## 卸载或者重装文档。 63 | https://tophanttechnology.github.io/ARL-doc/faq/#12-docker -------------------------------------------------------------------------------- /.github/workflows/arm-builder.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: ARM-Builder 4 | 5 | # Controls when the action will run. 6 | on: 7 | # Allows you to run this workflow manually from the Actions tab 8 | workflow_dispatch: 9 | 10 | 11 | env: 12 | IMAGE_URL: tophant/arl 13 | 14 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 15 | jobs: 16 | # This workflow contains a single job called "build" 17 | build: 18 | # The type of runner that the job will run on 19 | runs-on: ubuntu-latest 20 | 21 | # Steps represent a sequence of tasks that will be executed as part of the job 22 | steps: 23 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 24 | - uses: actions/checkout@master 25 | 26 | - name: Set up QEMU 27 | uses: docker/setup-qemu-action@v2 28 | 29 | - name: Set up Docker Buildx 30 | uses: docker/setup-buildx-action@v2 31 | 32 | - name: Download GeoLite2 33 | run: | 34 | pwd 35 | mkdir -p docker/GeoLite2/ 36 | wget -O docker/GeoLite2/GeoLite2-ASN.mmdb https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-ASN.mmdb 37 | wget -O docker/GeoLite2/GeoLite2-City.mmdb https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb 38 | ls -al docker/GeoLite2 39 | 40 | - name: Download NPoC 41 | run: | 42 | git clone https://github.com/Aabyss-Team/ARL-NPoC docker/ARL-NPoC 43 | 44 | - name: Login to Docker Hub 45 | uses: docker/login-action@v2 46 | with: 47 | username: ${{ secrets.DOCKER_USERNAME }} 48 | password: ${{ secrets.DOCKER_PASSWORD }} 49 | 50 | 51 | - name: Build ARM64 image and push 52 | run: | 53 | VERSION=$(cat version.txt) 54 | 55 | docker buildx build --file ./docker/ARMWorker/Dockerfile \ 56 | --platform linux/arm64/v8 \ 57 | --output type=docker \ 58 | --tag ${IMAGE_URL}:arm-${VERSION} . 59 | 60 | docker push ${IMAGE_URL}:arm-${VERSION} 61 | 62 | docker buildx imagetools inspect ${IMAGE_URL}:arm-${VERSION} 63 | 64 | 65 | -------------------------------------------------------------------------------- /.github/workflows/builder.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: build amd64 image 4 | 5 | # Controls when the action will run. 6 | on: 7 | # Triggers the workflow on push or pull request events but only for the master branch 8 | # push: 9 | # branches: [ master ] 10 | 11 | # Allows you to run this workflow manually from the Actions tab 12 | workflow_dispatch: 13 | 14 | env: 15 | IMAGE_URL: tophant/arl 16 | 17 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 18 | jobs: 19 | # This workflow contains a single job called "build" 20 | build: 21 | # The type of runner that the job will run on 22 | runs-on: ubuntu-latest 23 | 24 | # Steps represent a sequence of tasks that will be executed as part of the job 25 | steps: 26 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 27 | - uses: actions/checkout@master 28 | 29 | - name: Download GeoLite2 30 | run: | 31 | pwd 32 | mkdir -p docker/GeoLite2/ 33 | wget -O docker/GeoLite2/GeoLite2-ASN.mmdb https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-ASN.mmdb 34 | wget -O docker/GeoLite2/GeoLite2-City.mmdb https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb 35 | ls -al docker/GeoLite2 36 | 37 | - name: Download ncrack 38 | run: | 39 | wget -O docker/ncrack https://github.com/Aabyss-Team/arl_files/raw/master/ncrack 40 | wget -O docker/ncrack-services https://github.com/Aabyss-Team/arl_files/raw/master/ncrack-services 41 | chmod +x docker/ncrack 42 | 43 | - name: Download NPoC 44 | run: | 45 | git clone https://github.com/Aabyss-Team/ARL-NPoC docker/ARL-NPoC 46 | 47 | # Runs a single command using the runners shell 48 | - name: Login Registry 49 | run: | 50 | docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} 51 | 52 | # Runs a set of commands using the runners shell 53 | - name: Build Docker image 54 | run: | 55 | export VERSION=$(cat version.txt) 56 | docker build -t ${IMAGE_URL}:${VERSION} -f docker/worker/Dockerfile . 57 | 58 | - name: Publish 59 | run: | 60 | export VERSION=$(cat version.txt) 61 | docker push ${IMAGE_URL}:${VERSION} 62 | 63 | 64 | -------------------------------------------------------------------------------- /.github/workflows/push-docker-manifest.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: Push Docker Manifest 4 | 5 | # Controls when the action will run. 6 | on: 7 | # Triggers the workflow on push or pull request events but only for the master branch 8 | # push: 9 | # branches: [ master ] 10 | 11 | # Allows you to run this workflow manually from the Actions tab 12 | workflow_dispatch: 13 | 14 | env: 15 | IMAGE_URL: tophant/arl 16 | 17 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 18 | jobs: 19 | # This workflow contains a single job called "build" 20 | build: 21 | # The type of runner that the job will run on 22 | runs-on: ubuntu-latest 23 | 24 | # Steps represent a sequence of tasks that will be executed as part of the job 25 | steps: 26 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 27 | - uses: actions/checkout@master 28 | 29 | - name: Setup Docker 30 | uses: docker/setup-buildx-action@v1 31 | 32 | # Runs a single command using the runners shell 33 | - name: Login Registry 34 | run: | 35 | docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} 36 | 37 | - name: Pull images 38 | run: | 39 | export DOCKER_CLI_EXPERIMENTAL=enabled 40 | docker pull ${IMAGE_URL}:$(cat version.txt) 41 | docker pull ${IMAGE_URL}:arm-$(cat version.txt) 42 | 43 | - name: Create and publish Docker manifests 44 | run: | 45 | export VERSION=$(cat version.txt) 46 | export DOCKER_CLI_EXPERIMENTAL=enabled 47 | 48 | docker manifest create ${IMAGE_URL}:latest \ 49 | --amend ${IMAGE_URL}:arm-${VERSION} \ 50 | --amend ${IMAGE_URL}:${VERSION} 51 | 52 | docker manifest create ${IMAGE_URL}:${VERSION} \ 53 | --amend ${IMAGE_URL}:arm-${VERSION} \ 54 | --amend ${IMAGE_URL}:${VERSION} 55 | 56 | docker manifest push ${IMAGE_URL}:${VERSION} 57 | docker manifest push ${IMAGE_URL}:latest 58 | 59 | 60 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: pack Docker.zip file 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@master 14 | 15 | - name: Pack files 16 | run: | 17 | cd docker 18 | zip -r docker.zip .env arl_web.log arl_worker.log config-docker.yaml docker-compose.yml mongo-init.js 19 | 20 | - name: Upload artifact 21 | uses: xresloader/upload-to-github-release@v1 22 | env: 23 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 24 | with: 25 | file: docker/docker.zip 26 | tags: true 27 | draft: true 28 | 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | develop-eggs/ 12 | dist/ 13 | wheels/ 14 | *.egg-info/ 15 | .installed.cfg 16 | *.egg 17 | MANIFEST 18 | 19 | # PyInstaller 20 | # Usually these files are written by a python script from a template 21 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 22 | *.manifest 23 | *.spec 24 | 25 | # Installer logs 26 | pip-log.txt 27 | pip-delete-this-directory.txt 28 | 29 | # Unit test / coverage reports 30 | htmlcov/ 31 | .tox/ 32 | .coverage 33 | .coverage.* 34 | .cache 35 | nosetests.xml 36 | coverage.xml 37 | *.cover 38 | .hypothesis/ 39 | .pytest_cache/ 40 | 41 | # Translations 42 | *.mo 43 | *.pot 44 | 45 | # Django stuff: 46 | *.log 47 | local_settings.py 48 | db.sqlite3 49 | 50 | # Flask stuff: 51 | instance/ 52 | .webassets-cache 53 | 54 | # Scrapy stuff: 55 | .scrapy 56 | 57 | # Sphinx documentation 58 | docs/_build/ 59 | 60 | # PyBuilder 61 | target/ 62 | 63 | # Jupyter Notebook 64 | .ipynb_checkpoints 65 | 66 | # pyenv 67 | .python-version 68 | 69 | # celery beat schedule file 70 | celerybeat-schedule 71 | 72 | # SageMath parsed files 73 | *.sage.py 74 | 75 | # Environments 76 | .env 77 | .venv 78 | env/ 79 | venv/ 80 | ENV/ 81 | env.bak/ 82 | venv.bak/ 83 | 84 | # Spyder project settings 85 | .spyderproject 86 | .spyproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | # mkdocs documentation 92 | /site 93 | 94 | # mypy 95 | .mypy_cache/ 96 | celeryd.pid 97 | tmp/ 98 | tmp_capture/ 99 | .idea/ 100 | 101 | config.yaml 102 | supervisord.conf 103 | 104 | node_modules/ 105 | package-lock.json 106 | 107 | # FileChangeMonitor 108 | lib-cov 109 | *.seed 110 | *.log 111 | *.csv 112 | *.dat 113 | *.out 114 | *.pid 115 | *.gz 116 | 117 | pids 118 | logs 119 | results 120 | 121 | npm-debug.log 122 | node_modules 123 | .env 124 | 125 | *.zip 126 | 127 | docker/image/ 128 | docker/arl_web.log 129 | docker/arl_worker.log -------------------------------------------------------------------------------- /Aabyss-Bot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aabyss-Team/ARL/b2a12ce20b5f4ee1af8f687720735d154ef5c113/Aabyss-Bot.jpg -------------------------------------------------------------------------------- /Disclaimer.md: -------------------------------------------------------------------------------- 1 | ## 免责声明 2 | 3 | 如果您下载、安装、使用、修改本系统及相关代码，即表明您信任本系统。在使用本系统时造成对您自己或他人任何形式的损失和伤害，我们不承担任何责任。 4 | 如您在使用本系统的过程中存在任何非法行为，您需自行承担相应后果，我们将不承担任何法律及连带责任。 5 | 请您务必审慎阅读、充分理解各条款内容，特别是免除或者限制责任的条款，并选择接受或不接受。 6 | 除非您已阅读并接受本协议所有条款，否则您无权下载、安装或使用本系统。您的下载、安装、使用等行为即视为您已阅读并同意上述协议的约束。 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Aabyss-Team 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT许可证 2 | 3 | 版权所有 (c) 2023 tophant 4 | 5 | 特此授予任何获得本软件及相关文档文件（以下简称“软件”）的人免费使用本软件，包括但不限于使用、复制、修改、合并、出版、发行、再授权及贩售软件的副本，以及允许其他人这样做，但需要遵守以下条件： 6 | 7 | 在软件和软件的所有副本中都必须包含上述版权声明和本许可声明。 8 | 9 | 本软件按“原样”提供，没有任何形式的明示或暗示担保，包括但不限于适销性、特定用途适用性和非侵权性的保证。在任何情况下，作者或版权持有人均不对任何索赔、损害赔偿或其他责任承担法律责任，无论是在合同、侵权或其他方面，由使用本软件引起或与之相关的。 10 | 11 | 如果您在自己的项目中使用本软件，请在您的项目介绍页中注明本项目 Github 仓库的地址。 12 | 13 | 同意 https://github.com/TophantTechnology/ARL/blob/master/Disclaimer.md 免责声明 14 | -------------------------------------------------------------------------------- /Team.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aabyss-Team/ARL/b2a12ce20b5f4ee1af8f687720735d154ef5c113/Team.jpg -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | # 关闭警告 4 | warnings.filterwarnings("ignore", category=UserWarning, 5 | message="Python 3.6 is no longer supported by the Python core team") 6 | 7 | # 关闭高权限使用celery警告 8 | warnings.filterwarnings("ignore", category=UserWarning, 9 | message="You're running the worker with superuser privileges") 10 | 11 | -------------------------------------------------------------------------------- /app/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aabyss-Team/ARL/b2a12ce20b5f4ee1af8f687720735d154ef5c113/app/__main__.py -------------------------------------------------------------------------------- /app/dicts/altdnsdict.txt: -------------------------------------------------------------------------------- 1 | adm 2 | admin 3 | ai 4 | alpha 5 | api 6 | apidoc 7 | app 8 | auth 9 | aws 10 | backend 11 | beta 12 | brand 13 | bucket 14 | cdn 15 | cert 16 | chef 17 | ci 18 | client 19 | cms 20 | cn 21 | confluence 22 | container 23 | control 24 | cvs 25 | data 26 | demo 27 | dev 28 | dev1 29 | developer 30 | devops 31 | devs 32 | doc 33 | docker 34 | docs 35 | edge 36 | elastic 37 | email 38 | en 39 | engine 40 | eu 41 | eureka 42 | europe 43 | events 44 | ex 45 | ext 46 | front 47 | frontpage 48 | fw 49 | gateway 50 | gh 51 | git 52 | gitlab 53 | grafana 54 | gray 55 | gw 56 | h5 57 | help 58 | inner 59 | int 60 | internal 61 | intra 62 | it 63 | jenkins 64 | lab 65 | latin 66 | lax 67 | lb 68 | legacy 69 | login 70 | maven 71 | mirror 72 | monitor 73 | nacos 74 | net 75 | new 76 | node 77 | oid 78 | openapi 79 | ops 80 | org 81 | origin 82 | page 83 | partner 84 | pass 85 | pay 86 | payment 87 | php 88 | pre 89 | pre1 90 | pre2 91 | preview 92 | priv 93 | private 94 | pro 95 | prod 96 | production 97 | profile 98 | proxy 99 | qa 100 | raw 101 | region 102 | reset 103 | s3 104 | sandbox 105 | scan 106 | scm 107 | search 108 | secure 109 | security 110 | sentinel 111 | server 112 | service 113 | signed 114 | sit 115 | skins 116 | sre 117 | ssl 118 | st 119 | staff 120 | stage 121 | stage1 122 | staging 123 | static 124 | stg 125 | support 126 | svc 127 | swagger 128 | system 129 | team 130 | test 131 | test1 132 | testing 133 | testing1 134 | tpe 135 | train 136 | trial 137 | uat 138 | us 139 | ut 140 | v 141 | v1 142 | v2 143 | vpn 144 | we 145 | wiki 146 | wx 147 | xxljob -------------------------------------------------------------------------------- /app/dicts/black_asset_site.txt: -------------------------------------------------------------------------------- 1 | https://qiangzhan.qq.com 2 | https://nz.qq.com 3 | https://kxyx.qq.com 4 | https://history.qq.com 5 | https://ty.qq.com 6 | https://v.news.qq.com 7 | https://rf.gamebbs.qq.com 8 | https://v.sports.qq.com 9 | https://history.news.qq.com 10 | https://re.qq.com 11 | https://boao.qq.com 12 | https://dalian.qq.com 13 | https://golf.qq.com 14 | https://tnw.qq.com 15 | https://xw.time.qq.com 16 | https://green.news.qq.com 17 | https://biznext.qq.com 18 | https://thr.qq.com 19 | https://res.wx.qq.com 20 | https://time.qq.com 21 | https://space.qq.com -------------------------------------------------------------------------------- /app/dicts/blackdomain.txt: -------------------------------------------------------------------------------- 1 | .ime.galileo.baidu.com 2 | .upsiloncdn.net 3 | .aliyunwaf.com 4 | .qzone.qq.com 5 | .114.qq.com 6 | .docs.qq.com 7 | .cloud.tc.qq.com 8 | .tcdn.qq.com 9 | .photo.store.qq.com 10 | .shop.dianping.com 11 | .baijia.baidu.com 12 | .cloudapp.azure.com 13 | .bastion.azure.com 14 | .waf.huaweicloud.com 15 | .fe.baidu.com 16 | .hiphotos.baidu.com 17 | .openwebgame.qq.com 18 | .ke.qq.com 19 | .book.2.qq.com 20 | .tcdn.1.qq.com 21 | .tcdn.2.qq.com 22 | .book.3.qq.com 23 | .book.4.qq.com 24 | .o.tencent.com 25 | .o.qcloud.com 26 | .mur.qq.com 27 | .waf.taobao.com 28 | .vs.tencent.com 29 | .dev.tencent.com 30 | .code.tencent.com 31 | .tc.qq.com 32 | media.gamer.qq.com 33 | .ccs.tencent-cloud.com 34 | .peck.gamebonfire.com 35 | .mail.qq.com 36 | .meeting.tencent.com 37 | .txmeeting.tencent.com 38 | .powerfl.qq.com 39 | .livelink.qq.com 40 | .odp.qq.com 41 | .identity.tencent.com 42 | .account.tencent.com 43 | .sdc.tencent.com 44 | .workspace.tencent.com -------------------------------------------------------------------------------- /app/dicts/blackhexie.txt: -------------------------------------------------------------------------------- 1 | google 2 | youtube 3 | facebook 4 | openvpn 5 | webproxy 6 | wire.comm 7 | 69.muc 8 | .oxy. 9 | _ldap 10 | _udp 11 | qltx.live -------------------------------------------------------------------------------- /app/dicts/dnsserver.txt: -------------------------------------------------------------------------------- 1 | 119.29.29.29 2 | 182.254.118.118 3 | 223.5.5.5 4 | 223.6.6.6 -------------------------------------------------------------------------------- /app/dicts/domain_dict_test.txt: -------------------------------------------------------------------------------- 1 | www 2 | test 3 | blog 4 | service 5 | mi 6 | ebank 7 | active 8 | un -------------------------------------------------------------------------------- /app/dicts/file_test.txt: -------------------------------------------------------------------------------- 1 | access_token -------------------------------------------------------------------------------- /app/dicts/noscreenshot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Aabyss-Team/ARL/b2a12ce20b5f4ee1af8f687720735d154ef5c113/app/dicts/noscreenshot.jpg -------------------------------------------------------------------------------- /app/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | from .policy import get_options_by_policy_id 2 | from .task import ( 3 | submit_task, 4 | build_task_data, 5 | get_ip_domain_list, 6 | submit_task_task, 7 | submit_risk_cruising, 8 | target2list, 9 | submit_add_asset_site_task 10 | ) 11 | from .scope import get_scope_by_scope_id, check_target_in_scope 12 | from .url import get_url_by_task_id 13 | from .scheduler import have_same_site_update_monitor, have_same_wih_update_monitor 14 | from .asset_site import find_asset_site_not_in_scope 15 | from .domain import ( 16 | find_domain_by_task_id, 17 | find_private_domain_by_task_id, 18 | find_public_ip_by_task_id 19 | ) 20 | 21 | -------------------------------------------------------------------------------- /app/helpers/asset_domain.py: -------------------------------------------------------------------------------- 1 | from app import utils 2 | 3 | 4 | def find_domain_by_scope_id(scope_id): 5 | query = { 6 | "scope_id": scope_id 7 | } 8 | items = utils.conn_db('asset_domain').distinct("domain", query) 9 | return list(items) 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /app/helpers/asset_site.py: -------------------------------------------------------------------------------- 1 | from app import utils 2 | from .scope import get_scope_by_scope_id 3 | 4 | 5 | def build_show_filed_map(fields): 6 | q = {} 7 | for field in fields: 8 | q[field] = 1 9 | 10 | return q 11 | 12 | 13 | def find_site_info_by_scope_id(scope_id): 14 | query = { 15 | "scope_id": scope_id 16 | } 17 | fields = ["site", "title", "status"] 18 | show_map = build_show_filed_map(fields) 19 | items = utils.conn_db('asset_site').find(query, show_map) 20 | return list(items) 21 | 22 | 23 | def find_site_by_scope_id(scope_id): 24 | query = { 25 | "scope_id": scope_id 26 | } 27 | items = utils.conn_db('asset_site').distinct("site", query) 28 | return list(items) 29 | 30 | 31 | # 检查用户提交的站点，判断是否符合范围, 32 | def check_asset_site_in_scope(site: str, scope_array: list) -> bool: 33 | for scope in scope_array: 34 | # 简单判断下 35 | if scope in site: 36 | return True 37 | return False 38 | 39 | 40 | # 用户提交的站点，判断出哪些是不符合范围的 41 | def find_asset_site_not_in_scope(sites: list, scope_id: str) -> list: 42 | ret = [] 43 | scopes = get_scope_by_scope_id(scope_id) 44 | scope_array = scopes.get("scope_array", []) 45 | for site in sites: 46 | if not check_asset_site_in_scope(site, scope_array): 47 | ret.append(site) 48 | 49 | return ret 50 | 51 | -------------------------------------------------------------------------------- /app/helpers/asset_site_monitor.py: -------------------------------------------------------------------------------- 1 | from app.modules import CeleryAction, SchedulerStatus, AssetScopeType, TaskStatus, TaskType 2 | from app import celerytask, utils 3 | from app.config import Config 4 | logger = utils.get_logger() 5 | 6 | 7 | def submit_asset_site_monitor_job(scope_id, name, scheduler_id): 8 | from app.helpers.task import submit_task 9 | 10 | task_data = { 11 | 'name': name, 12 | 'target': "资产站点更新", 13 | 'start_time': '-', 14 | 'status': TaskStatus.WAITING, 15 | 'type': TaskType.ASSET_SITE_UPDATE, 16 | "task_tag": TaskType.ASSET_SITE_UPDATE, 17 | 'options': { 18 | "scope_id": scope_id, 19 | "scheduler_id": scheduler_id 20 | }, 21 | "end_time": "-", 22 | "service": [], 23 | "celery_id": "" 24 | } 25 | 26 | submit_task(task_data) 27 | 28 | 29 | black_asset_site_list = None 30 | 31 | 32 | def is_black_asset_site(site): 33 | global black_asset_site_list 34 | if black_asset_site_list is None: 35 | with open(Config.black_asset_site) as f: 36 | black_asset_site_list = f.readlines() 37 | 38 | for item in black_asset_site_list: 39 | item = item.strip() 40 | if not item: 41 | continue 42 | if site.startswith(item): 43 | return True 44 | 45 | return False 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /app/helpers/asset_wih.py: -------------------------------------------------------------------------------- 1 | from app import utils 2 | 3 | 4 | def get_wih_record_fnv_hash(scope_id): 5 | query = { 6 | "scope_id": scope_id 7 | } 8 | items = utils.conn_db('asset_wih').distinct("fnv_hash", query) 9 | return list(items) -------------------------------------------------------------------------------- /app/helpers/asset_wih_monitor.py: -------------------------------------------------------------------------------- 1 | from app.modules import TaskStatus, TaskType 2 | 3 | 4 | def submit_asset_wih_monitor_job(scope_id, name, scheduler_id): 5 | from app.helpers.task import submit_task 6 | 7 | task_data = { 8 | 'name': name, 9 | 'target': "资产分组 WIH 更新", 10 | 'start_time': '-', 11 | 'status': TaskStatus.WAITING, 12 | 'type': TaskType.ASSET_WIH_UPDATE, 13 | "task_tag": TaskType.ASSET_WIH_UPDATE, 14 | 'options': { 15 | "scope_id": scope_id, 16 | "scheduler_id": scheduler_id 17 | }, 18 | "end_time": "-", 19 | "service": [], 20 | "celery_id": "" 21 | } 22 | 23 | submit_task(task_data) -------------------------------------------------------------------------------- /app/helpers/domain.py: -------------------------------------------------------------------------------- 1 | from app import utils 2 | 3 | 4 | def find_private_domain_by_task_id(task_id): 5 | query = { 6 | "task_id": task_id, 7 | "ip_type": "PRIVATE" 8 | } 9 | domains = [] 10 | items = utils.conn_db('ip').find(query) 11 | for item in list(items): 12 | if not item.get("domain"): 13 | continue 14 | domains.extend(item["domain"]) 15 | 16 | return list(set(domains)) 17 | 18 | 19 | def find_public_ip_by_task_id(task_id): 20 | query = { 21 | "task_id": task_id, 22 | "ip_type": "PUBLIC" 23 | } 24 | items = utils.conn_db('ip').distinct("ip", query) 25 | return list(items) 26 | 27 | 28 | def find_domain_by_task_id(task_id): 29 | query = { 30 | "task_id": task_id 31 | } 32 | items = utils.conn_db('domain').distinct("domain", query) 33 | return list(items) 34 | 35 | 36 | -------------------------------------------------------------------------------- /app/helpers/message_notify.py: -------------------------------------------------------------------------------- 1 | from app.config import Config 2 | from app.utils import get_logger, push 3 | 4 | logger = get_logger() 5 | 6 | 7 | def push_email(title, html_report): 8 | try: 9 | if Config.EMAIL_HOST and Config.EMAIL_USERNAME and Config.EMAIL_PASSWORD: 10 | push.send_email(host=Config.EMAIL_HOST, port=Config.EMAIL_PORT, mail=Config.EMAIL_USERNAME, 11 | password=Config.EMAIL_PASSWORD, to=Config.EMAIL_TO, 12 | title=title, html=html_report) 13 | logger.info("send email succ") 14 | return True 15 | except Exception as e: 16 | logger.info("error on send email {}".format(title)) 17 | logger.warning(e) 18 | 19 | 20 | def push_dingding(markdown_report): 21 | try: 22 | if Config.DINGDING_ACCESS_TOKEN and Config.DINGDING_SECRET: 23 | data = push.dingding_send(access_token=Config.DINGDING_ACCESS_TOKEN, 24 | secret=Config.DINGDING_SECRET, msgtype="markdown", 25 | msg=markdown_report) 26 | if data.get("errcode", -1) == 0: 27 | logger.info("push dingding succ") 28 | return True 29 | else: 30 | logger.info("{}".format(data)) 31 | 32 | except Exception as e: 33 | logger.info("error on send dingding {}".format(markdown_report[:15])) 34 | logger.warning(e) 35 | 36 | 37 | -------------------------------------------------------------------------------- /app/helpers/policy.py: -------------------------------------------------------------------------------- 1 | import bson 2 | from app import utils 3 | from app.modules import TaskTag 4 | 5 | 6 | def get_options_by_policy_id(policy_id, task_tag): 7 | query = { 8 | "_id": bson.ObjectId(policy_id) 9 | } 10 | data = utils.conn_db("policy").find_one(query) 11 | if not data: 12 | return 13 | 14 | policy = data["policy"] 15 | options = { 16 | "policy_name": data["name"] 17 | } 18 | domain_config = policy.pop("domain_config") 19 | ip_config = policy.pop("ip_config") 20 | site_config = policy.pop("site_config") 21 | 22 | if "scope_config" in policy: 23 | scope_config = policy.pop("scope_config") 24 | options["related_scope_id"] = scope_config["scope_id"] 25 | 26 | """仅仅资产发现任务需要这些""" 27 | if task_tag == TaskTag.TASK: 28 | options.update(domain_config) 29 | options.update(ip_config) 30 | 31 | options.update(site_config) 32 | 33 | options.update(policy) 34 | return options 35 | -------------------------------------------------------------------------------- /app/helpers/scheduler.py: -------------------------------------------------------------------------------- 1 | from app import utils 2 | 3 | 4 | def have_same_site_update_monitor(scope_id): 5 | query = { 6 | "scope_id": scope_id, 7 | "scope_type": "site_update_monitor" 8 | } 9 | 10 | result = utils.conn_db('scheduler').find_one(query) 11 | if result: 12 | return True 13 | 14 | return False 15 | 16 | 17 | def have_same_wih_update_monitor(scope_id): 18 | query = { 19 | "scope_id": scope_id, 20 | "scope_type": "wih_update_monitor" 21 | } 22 | 23 | result = utils.conn_db('scheduler').find_one(query) 24 | if result: 25 | return True 26 | 27 | return False -------------------------------------------------------------------------------- /app/helpers/scope.py: -------------------------------------------------------------------------------- 1 | import bson 2 | from app import utils 3 | from app.utils.ip import ip_in_scope 4 | from app.utils.domain import is_in_scopes 5 | 6 | 7 | def check_target_in_scope(target, scope_list): 8 | from .task import get_ip_domain_list 9 | ip_list, domain_list = get_ip_domain_list(target) 10 | for ip in ip_list: 11 | if not ip_in_scope(ip, scope_list): 12 | raise Exception("{}不在范围{}中".format(ip, ",".join(scope_list))) 13 | 14 | for domain in domain_list: 15 | if not is_in_scopes(domain, scope_list): 16 | raise Exception("{}不在范围{}中".format(domain, ",".join(scope_list))) 17 | 18 | return ip_list, domain_list 19 | 20 | 21 | def get_scope_by_scope_id(scope_id): 22 | query = { 23 | "_id": bson.ObjectId(scope_id) 24 | } 25 | data = utils.conn_db("asset_scope").find_one(query) 26 | return data 27 | 28 | 29 | -------------------------------------------------------------------------------- /app/helpers/url.py: -------------------------------------------------------------------------------- 1 | from app import utils 2 | 3 | 4 | def get_url_by_task_id(task_id): 5 | query = { 6 | "task_id": task_id 7 | } 8 | items = utils.conn_db('url').distinct("url", query) 9 | return list(items) 10 | -------------------------------------------------------------------------------- /app/main.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from flask_restx import Api 3 | 4 | from app import routes 5 | from app.utils import arl_update 6 | 7 | arl_app = Flask(__name__) 8 | arl_app.config['BUNDLE_ERRORS'] = True 9 | 10 | authorizations = { 11 | "ApiKeyAuth": { 12 | "type": "apiKey", 13 | "in": "header", 14 | "name": "Token" 15 | } 16 | } 17 | 18 | api = Api(arl_app, prefix="/api", doc="/api/doc", title='ARL backend API', authorizations=authorizations, 19 | description='ARL（Asset Reconnaissance Lighthouse）资产侦察灯塔系统', security="ApiKeyAuth", version="2.6.2") 20 | 21 | api.add_namespace(routes.task_ns) 22 | api.add_namespace(routes.site_ns) 23 | api.add_namespace(routes.domain_ns) 24 | api.add_namespace(routes.ip_ns) 25 | api.add_namespace(routes.url_ns) 26 | api.add_namespace(routes.user_ns) 27 | api.add_namespace(routes.image_ns) 28 | api.add_namespace(routes.cert_ns) 29 | api.add_namespace(routes.service_ns) 30 | api.add_namespace(routes.fileleak_ns) 31 | api.add_namespace(routes.export_ns) 32 | api.add_namespace(routes.asset_scope_ns) 33 | api.add_namespace(routes.asset_domain_ns) 34 | api.add_namespace(routes.asset_ip_ns) 35 | api.add_namespace(routes.asset_site_ns) 36 | api.add_namespace(routes.scheduler_ns) 37 | api.add_namespace(routes.poc_ns) 38 | api.add_namespace(routes.vuln_ns) 39 | api.add_namespace(routes.batch_export_ns) 40 | api.add_namespace(routes.policy_ns) 41 | api.add_namespace(routes.npoc_service_ns) 42 | api.add_namespace(routes.task_fofa_ns) 43 | api.add_namespace(routes.console_ns) 44 | api.add_namespace(routes.cip_ns) 45 | api.add_namespace(routes.fingerprint_ns) 46 | api.add_namespace(routes.stat_finger_ns) 47 | api.add_namespace(routes.github_task_ns) 48 | api.add_namespace(routes.github_result_ns) 49 | api.add_namespace(routes.github_scheduler_ns) 50 | api.add_namespace(routes.github_monitor_result_ns) 51 | api.add_namespace(routes.task_schedule_ns) 52 | api.add_namespace(routes.nuclei_result_ns) 53 | api.add_namespace(routes.wih_ns) 54 | api.add_namespace(routes.asset_wih_ns) 55 | 56 | 57 | arl_update() 58 | 59 | if __name__ == '__main__': 60 | arl_app.run(debug=True, port=5018, host="0.0.0.0") 61 | -------------------------------------------------------------------------------- /app/modules/baseInfo.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | 4 | class BaseInfo: 5 | def __str__(self): 6 | return self.dump_json() 7 | 8 | def __repr__(self): 9 | return self.dump_json() 10 | 11 | def dump_json(self, flag = True): 12 | item = self._dump_json() 13 | if flag: 14 | return json.dumps(item) 15 | else: 16 | return item 17 | 18 | def _dump_json(self): 19 | raise NotImplementedError() -------------------------------------------------------------------------------- /app/modules/domainInfo.py: -------------------------------------------------------------------------------- 1 | from .baseInfo import BaseInfo 2 | 3 | 4 | class DomainInfo(BaseInfo): 5 | def __init__(self, domain, record, type, ips): 6 | self.record_list = record 7 | self.domain = domain 8 | self.type = type 9 | self.ip_list = ips 10 | 11 | def __eq__(self, other): 12 | if isinstance(other, DomainInfo): 13 | if self.domain == other.domain: 14 | return True 15 | 16 | def __hash__(self): 17 | return hash(self.domain) 18 | 19 | def _dump_json(self): 20 | item = { 21 | "domain": self.domain, 22 | "record": self.record_list, 23 | "type": self.type, 24 | "ips": self.ip_list 25 | } 26 | return item -------------------------------------------------------------------------------- /app/modules/ipInfo.py: -------------------------------------------------------------------------------- 1 | from .baseInfo import BaseInfo 2 | from app import utils 3 | 4 | 5 | class IPInfo(BaseInfo): 6 | def __init__(self, ip, port_info, os_info, domain, cdn_name): 7 | self.ip = ip 8 | self.port_info_list = port_info 9 | self.os_info = os_info 10 | self.domain = domain 11 | self._geo_asn = None 12 | self._geo_city = None 13 | self._ip_type = None 14 | self.cdn_name = cdn_name 15 | 16 | @property 17 | def geo_asn(self): 18 | if self._geo_asn: 19 | return self._geo_asn 20 | 21 | else: 22 | if self.ip_type == "PUBLIC": 23 | self._geo_asn = utils.get_ip_asn(self.ip) 24 | else: 25 | self._geo_asn = {} 26 | 27 | return self._geo_asn 28 | 29 | @property 30 | def geo_city(self): 31 | if self._geo_city: 32 | return self._geo_city 33 | 34 | else: 35 | if self.ip_type == "PUBLIC": 36 | self._geo_city = utils.get_ip_city(self.ip) 37 | else: 38 | self._geo_city = {} 39 | 40 | return self._geo_city 41 | 42 | @property 43 | def ip_type(self): 44 | if self._ip_type: 45 | return self._ip_type 46 | 47 | else: 48 | self._ip_type = utils.get_ip_type(self.ip) 49 | 50 | return self._ip_type 51 | 52 | def __eq__(self, other): 53 | if isinstance(other, IPInfo): 54 | if self.ip == other.ip: 55 | return True 56 | 57 | def __hash__(self): 58 | return hash(self.ip) 59 | 60 | def _dump_json(self): 61 | port_info = [] 62 | for x in self.port_info_list: 63 | port_info.append(x.dump_json(flag=False)) 64 | 65 | item = { 66 | "ip": self.ip, 67 | "domain": self.domain, 68 | "port_info": port_info, 69 | "os_info": self.os_info, 70 | "ip_type": self.ip_type, 71 | "geo_asn": self.geo_asn, 72 | "geo_city": self.geo_city, 73 | "cdn_name": self.cdn_name 74 | } 75 | return item 76 | 77 | 78 | class PortInfo(BaseInfo): 79 | def __init__(self, port_id, service_name = "", version = "", protocol = "tcp", product=""): 80 | self.port_id = port_id 81 | self.service_name = service_name 82 | self.version = version 83 | self.protocol = protocol 84 | self.product = product 85 | 86 | def __eq__(self, other): 87 | if isinstance(other, PortInfo): 88 | if self.port_id == other.port_id: 89 | return True 90 | 91 | def __hash__(self): 92 | return hash(self.port_id) 93 | 94 | 95 | def _dump_json(self): 96 | item = { 97 | "port_id": self.port_id, 98 | "service_name": self.service_name, 99 | "version": self.version, 100 | "protocol": self.protocol, 101 | "product": self.product 102 | } 103 | return item 104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /app/modules/pageInfo.py: -------------------------------------------------------------------------------- 1 | from .baseInfo import BaseInfo 2 | 3 | class PageInfo(BaseInfo): 4 | def __init__(self, title, url, content_length, status_code): 5 | self.title = title 6 | self.url = url 7 | self.content_length = content_length 8 | self.status_code = status_code 9 | 10 | def __eq__(self, other): 11 | if isinstance(other, PageInfo): 12 | if self.url == other.url: 13 | return True 14 | 15 | def __hash__(self): 16 | return hash(self.url) 17 | 18 | def _dump_json(self): 19 | item = { 20 | "title": self.title, 21 | "url": self.url, 22 | "content_length": self.content_length, 23 | "status_code": self.status_code 24 | } 25 | return item -------------------------------------------------------------------------------- /app/modules/wihRecord.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class WihRecord: 4 | def __init__(self, record_type, content, source, site, fnv_hash): 5 | self.recordType = record_type 6 | self.content = content 7 | self.source = source 8 | self.site = site 9 | self.fnv_hash = fnv_hash 10 | 11 | def __str__(self): 12 | return "{} {} {} {}".format(self.recordType, self.content, self.source, self.site) 13 | 14 | def __repr__(self): 15 | return "" + self.__str__() 16 | 17 | def __eq__(self, other): 18 | return self.fnv_hash == other.fnv_hash 19 | 20 | def __hash__(self): 21 | return self.fnv_hash 22 | 23 | def dump_json(self): 24 | return { 25 | "record_type": self.recordType, 26 | "content": self.content, 27 | "site": self.site, 28 | "source": self.source, 29 | "fnv_hash": str(self.fnv_hash), 30 | } -------------------------------------------------------------------------------- /app/routes/assetWih.py: -------------------------------------------------------------------------------- 1 | from flask_restx import fields, Namespace 2 | from app.utils import get_logger, auth 3 | from . import base_query_fields, ARLResource, get_arl_parser 4 | from bson import ObjectId 5 | from app.modules import ErrorMsg 6 | from app import utils 7 | 8 | ns = Namespace('asset_wih', description="资产组 WEB Info Hunter 信息") 9 | 10 | logger = get_logger() 11 | 12 | base_search_fields = { 13 | 'record_type': fields.String(required=False, description="记录类型"), 14 | 'record_type__neq': fields.String(required=False, description="记录类型不等于（全匹配）"), 15 | 'record_type__not': fields.String(required=False, description="记录类型不包含"), 16 | 'content': fields.String(description="内容"), 17 | 'source': fields.String(description="来源 JS URL"), 18 | 'site': fields.String(description="站点URL"), 19 | "update_date__dgt": fields.String(description="更新时间大于"), 20 | "update_date__dlt": fields.String(description="更新时间小于"), 21 | 'scope_id': fields.String(description="范围 ID") 22 | } 23 | 24 | 25 | base_search_fields.update(base_query_fields) 26 | 27 | 28 | @ns.route('/') 29 | class ARLAssetWebInfoHunter(ARLResource): 30 | parser = get_arl_parser(base_search_fields, location='args') 31 | 32 | @auth 33 | @ns.expect(parser) 34 | def get(self): 35 | """ 36 | 资产组 WEB Info Hunter 信息查询 37 | """ 38 | args = self.parser.parse_args() 39 | data = self.build_data(args=args, collection='asset_wih') 40 | 41 | return data 42 | 43 | 44 | @ns.route('/export/') 45 | class ARLAssetWIHExport(ARLResource): 46 | parser = get_arl_parser(base_search_fields, location='args') 47 | 48 | @auth 49 | @ns.expect(parser) 50 | def get(self): 51 | """ 52 | 资产分组 WIH 导出 53 | """ 54 | args = self.parser.parse_args() 55 | response = self.send_export_file(args=args, _type="asset_wih") 56 | 57 | return response 58 | 59 | 60 | delete_asset_wih_fields = ns.model('deleteAssetWih', { 61 | '_id': fields.List(fields.String(required=True, description="数据_id")) 62 | }) 63 | 64 | 65 | @ns.route('/delete/') 66 | class DeleteARLAssetWIH(ARLResource): 67 | @auth 68 | @ns.expect(delete_asset_wih_fields) 69 | def post(self): 70 | """ 71 | 删除资产组中的 wih 数据 72 | """ 73 | args = self.parse_args(delete_asset_wih_fields) 74 | id_list = args.pop('_id', "") 75 | for _id in id_list: 76 | query = {'_id': ObjectId(_id)} 77 | utils.conn_db('asset_wih').delete_one(query) 78 | 79 | return utils.build_ret(ErrorMsg.Success, {'_id': id_list}) 80 | 81 | -------------------------------------------------------------------------------- /app/routes/cert.py: -------------------------------------------------------------------------------- 1 | from bson import ObjectId 2 | from flask_restx import Resource, Api, reqparse, fields, Namespace 3 | from app.utils import get_logger, auth 4 | from . import base_query_fields, ARLResource, get_arl_parser 5 | from app import utils 6 | from app.modules import ErrorMsg 7 | 8 | ns = Namespace('cert', description="证书信息") 9 | 10 | logger = get_logger() 11 | 12 | base_search_fields = { 13 | 'ip': fields.String(description="ip"), 14 | 'port': fields.Integer(description="端口"), 15 | 'cert.subject_dn': fields.String(description="主题名称"), 16 | 'cert.issuer_dn': fields.String(description="签发者名称"), 17 | 'cert.serial_number ': fields.String(description="序列号"), 18 | 'cert.validity.start': fields.String(description="开始时间"), 19 | 'cert.validity.end': fields.String(description="结束时间"), 20 | 'cert.fingerprint.sha256': fields.String(description="SHA-256"), 21 | 'cert.fingerprint.sha1': fields.String(description="SHA-1"), 22 | 'cert.fingerprint.md5': fields.String(description="MD5"), 23 | 'cert.extensions.subjectAltName': fields.String(description="备用名称"), 24 | 'task_id': fields.String(description="任务 ID"), 25 | } 26 | 27 | base_search_fields.update(base_query_fields) 28 | 29 | 30 | @ns.route('/') 31 | class ARLCert(ARLResource): 32 | parser = get_arl_parser(base_search_fields, location='args') 33 | 34 | @auth 35 | @ns.expect(parser) 36 | def get(self): 37 | """ 38 | SSL证书查询 39 | """ 40 | args = self.parser.parse_args() 41 | data = self.build_data(args=args, collection='cert') 42 | 43 | return data 44 | 45 | 46 | delete_cert_fields = ns.model('deleteCertFields', { 47 | '_id': fields.List(fields.String(required=True, description="证书 _id")) 48 | }) 49 | 50 | 51 | @ns.route('/delete/') 52 | class DeleteARLCert(ARLResource): 53 | @auth 54 | @ns.expect(delete_cert_fields) 55 | def post(self): 56 | """ 57 | 删除 SSL 证书信息 58 | """ 59 | args = self.parse_args(delete_cert_fields) 60 | id_list = args.pop('_id', []) 61 | for _id in id_list: 62 | query = {'_id': ObjectId(_id)} 63 | utils.conn_db('cert').delete_one(query) 64 | 65 | return utils.build_ret(ErrorMsg.Success, {'_id': id_list}) -------------------------------------------------------------------------------- /app/routes/cip.py: -------------------------------------------------------------------------------- 1 | from flask_restx import Resource, Api, reqparse, fields, Namespace 2 | from app.utils import get_logger, auth 3 | from . import base_query_fields, ARLResource, get_arl_parser 4 | 5 | ns = Namespace('cip', description="C段 ip 统计信息") 6 | 7 | logger = get_logger() 8 | 9 | base_search_fields = { 10 | 'cidr_ip': fields.String(required=False, description="C段"), 11 | "task_id": fields.String(description="任务 ID"), 12 | "ip_count": fields.Integer(description="IP 个数"), 13 | "domain_count": fields.Integer(description="解析到该 C 段域名个数") 14 | } 15 | 16 | base_search_fields.update(base_query_fields) 17 | 18 | 19 | @ns.route('/') 20 | class ARLCIPList(ARLResource): 21 | parser = get_arl_parser(base_search_fields, location='args') 22 | 23 | @auth 24 | @ns.expect(parser) 25 | def get(self): 26 | """ 27 | C 段统计信息查询 28 | """ 29 | args = self.parser.parse_args() 30 | data = self.build_data(args=args, collection='cip') 31 | 32 | return data 33 | 34 | 35 | @ns.route('/export/') 36 | class ARLCIPExport(ARLResource): 37 | parser = get_arl_parser(base_search_fields, location='args') 38 | 39 | @auth 40 | @ns.expect(parser) 41 | def get(self): 42 | """ 43 | C 段 IP 导出 44 | """ 45 | args = self.parser.parse_args() 46 | response = self.send_export_file(args=args, _type="cip") 47 | 48 | return response 49 | -------------------------------------------------------------------------------- /app/routes/console.py: -------------------------------------------------------------------------------- 1 | from flask_restx import fields, Namespace 2 | from app.utils import get_logger, auth 3 | from app import utils 4 | from app.modules import ErrorMsg 5 | from . import base_query_fields, ARLResource, get_arl_parser 6 | 7 | ns = Namespace('console', description="控制台信息") 8 | 9 | logger = get_logger() 10 | 11 | 12 | @ns.route('/info') 13 | class ARLConsole(ARLResource): 14 | 15 | @auth 16 | def get(self): 17 | """ 18 | 控制台信息查看 19 | """ 20 | 21 | data = { 22 | "device_info": utils.device_info() # 包含 CPU 内存和磁盘信息 23 | } 24 | 25 | return utils.build_ret(ErrorMsg.Success, data) 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /app/routes/domain.py: -------------------------------------------------------------------------------- 1 | from bson import ObjectId 2 | from flask_restx import Resource, Api, reqparse, fields, Namespace 3 | from app.utils import get_logger, auth 4 | from app import utils 5 | from app.modules import ErrorMsg 6 | from . import base_query_fields, ARLResource, get_arl_parser 7 | 8 | ns = Namespace('domain', description="域名信息") 9 | 10 | logger = get_logger() 11 | 12 | base_search_fields = { 13 | 'domain': fields.String(required=False, description="域名"), 14 | 'record': fields.String(description="解析值"), 15 | 'type': fields.String(description="解析类型"), 16 | 'ips': fields.String(description="IP"), 17 | 'source': fields.String(description="来源"), 18 | "task_id": fields.String(description="任务ID") 19 | } 20 | 21 | base_search_fields.update(base_query_fields) 22 | 23 | 24 | @ns.route('/') 25 | class ARLDomain(ARLResource): 26 | parser = get_arl_parser(base_search_fields, location='args') 27 | 28 | @auth 29 | @ns.expect(parser) 30 | def get(self): 31 | """ 32 | 域名信息查询 33 | """ 34 | args = self.parser.parse_args() 35 | data = self.build_data(args=args, collection='domain') 36 | 37 | return data 38 | 39 | 40 | @ns.route('/export/') 41 | class ARLDomainExport(ARLResource): 42 | parser = get_arl_parser(base_search_fields, location='args') 43 | 44 | @auth 45 | @ns.expect(parser) 46 | def get(self): 47 | """ 48 | 域名导出 49 | """ 50 | args = self.parser.parse_args() 51 | response = self.send_export_file(args=args, _type="domain") 52 | 53 | return response 54 | 55 | 56 | delete_domain_fields = ns.model('deleteDomainFields', { 57 | '_id': fields.List(fields.String(required=True, description="域名 _id")) 58 | }) 59 | 60 | 61 | @ns.route('/delete/') 62 | class DeleteARLDomain(ARLResource): 63 | @auth 64 | @ns.expect(delete_domain_fields) 65 | def post(self): 66 | """ 67 | 删除 域名 68 | """ 69 | args = self.parse_args(delete_domain_fields) 70 | id_list = args.pop('_id', []) 71 | for _id in id_list: 72 | query = {'_id': ObjectId(_id)} 73 | utils.conn_db('domain').delete_one(query) 74 | 75 | return utils.build_ret(ErrorMsg.Success, {'_id': id_list}) 76 | -------------------------------------------------------------------------------- /app/routes/fileleak.py: -------------------------------------------------------------------------------- 1 | from bson import ObjectId 2 | from flask_restx import Resource, Api, reqparse, fields, Namespace 3 | from app.utils import get_logger, auth 4 | from . import base_query_fields, ARLResource, get_arl_parser 5 | from app import utils 6 | from app.modules import ErrorMsg 7 | 8 | ns = Namespace('fileleak', description="文件泄漏信息") 9 | 10 | logger = get_logger() 11 | 12 | base_search_fields = { 13 | 'url': fields.String(required=False, description="URL"), 14 | 'site': fields.String(description="站点"), 15 | 'content_length': fields.Integer(description="body 长度"), 16 | 'status_code': fields.Integer(description="状态码"), 17 | 'title': fields.String(description="标题"), 18 | "task_id": fields.String(description="任务ID") 19 | } 20 | 21 | base_search_fields.update(base_query_fields) 22 | 23 | 24 | @ns.route('/') 25 | class ARLFileLeak(ARLResource): 26 | parser = get_arl_parser(base_search_fields, location='args') 27 | 28 | @auth 29 | @ns.expect(parser) 30 | def get(self): 31 | """ 32 | 文件泄露信息查询 33 | """ 34 | args = self.parser.parse_args() 35 | data = self.build_data(args=args, collection='fileleak') 36 | 37 | return data 38 | 39 | 40 | delete_fileleak_fields = ns.model('deleteFileleakFields', { 41 | '_id': fields.List(fields.String(required=True, description="文件泄漏 _id")) 42 | }) 43 | 44 | 45 | @ns.route('/delete/') 46 | class DeleteARLFileleak(ARLResource): 47 | @auth 48 | @ns.expect(delete_fileleak_fields) 49 | def post(self): 50 | """ 51 | 删除 文件泄漏 52 | """ 53 | args = self.parse_args(delete_fileleak_fields) 54 | id_list = args.pop('_id', []) 55 | for _id in id_list: 56 | query = {'_id': ObjectId(_id)} 57 | utils.conn_db('fileleak').delete_one(query) 58 | 59 | return utils.build_ret(ErrorMsg.Success, {'_id': id_list}) 60 | 61 | -------------------------------------------------------------------------------- /app/routes/github_monitor_result.py: -------------------------------------------------------------------------------- 1 | from flask_restx import fields, Namespace 2 | from app.utils import get_logger, auth 3 | from . import base_query_fields, ARLResource, get_arl_parser 4 | 5 | ns = Namespace('github_monitor_result', description="Github 监控结果详情") 6 | 7 | logger = get_logger() 8 | 9 | base_search_fields = { 10 | 'path': fields.String(required=False, description="路径名称"), 11 | 'repo_full_name': fields.String(description="仓库名称"), 12 | 'human_content': fields.String(description="内容"), 13 | 'keyword': fields.String(description="关键字"), 14 | "github_scheduler_id": fields.String(description="Github 监控ID"), 15 | 'github_task_id': fields.String(description="任务ID") 16 | } 17 | 18 | base_search_fields.update(base_query_fields) 19 | 20 | 21 | @ns.route('/') 22 | class ARLGithubMonitorResult(ARLResource): 23 | parser = get_arl_parser(base_search_fields, location='args') 24 | 25 | @auth 26 | @ns.expect(parser) 27 | def get(self): 28 | """ 29 | Github 监控结果详情 30 | """ 31 | args = self.parser.parse_args() 32 | data = self.build_data(args=args, collection='github_monitor_result') 33 | 34 | return data 35 | 36 | 37 | -------------------------------------------------------------------------------- /app/routes/github_result.py: -------------------------------------------------------------------------------- 1 | from flask_restx import fields, Namespace 2 | from app.utils import get_logger, auth 3 | from . import base_query_fields, ARLResource, get_arl_parser 4 | 5 | ns = Namespace('github_result', description="Github 结果详情") 6 | 7 | logger = get_logger() 8 | 9 | base_search_fields = { 10 | 'path': fields.String(required=False, description="路径名称"), 11 | 'repo_full_name': fields.String(description="仓库名称"), 12 | 'human_content': fields.String(description="内容"), 13 | 'github_task_id': fields.String(description="任务ID") 14 | } 15 | 16 | base_search_fields.update(base_query_fields) 17 | 18 | 19 | @ns.route('/') 20 | class ARLGithubResult(ARLResource): 21 | parser = get_arl_parser(base_search_fields, location='args') 22 | 23 | @auth 24 | @ns.expect(parser) 25 | def get(self): 26 | """ 27 | Github 结果详情查询 28 | """ 29 | args = self.parser.parse_args() 30 | data = self.build_data(args=args, collection='github_result') 31 | 32 | return data 33 | 34 | 35 | -------------------------------------------------------------------------------- /app/routes/image.py: -------------------------------------------------------------------------------- 1 | from flask import make_response 2 | from flask_restx import Resource, Namespace 3 | import os 4 | from app.config import Config 5 | from app.utils import get_logger 6 | from werkzeug.utils import secure_filename 7 | 8 | ns = Namespace('image', description="截图信息") 9 | 10 | logger = get_logger() 11 | 12 | def allowed_file(filename): 13 | ALLOWED_EXTENSIONS = ['jpg','png'] 14 | return '.' in filename and \ 15 | filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS 16 | 17 | @ns.route('//') 18 | class ARLImage(Resource): 19 | 20 | def get(self, task_id, file_name): 21 | task_id = secure_filename(task_id) 22 | file_name = secure_filename(file_name) 23 | if not allowed_file(file_name): 24 | return 25 | imgpath = os.path.join(Config.SCREENSHOT_DIR, 26 | '{task_id}/{file_name}'.format(task_id=task_id, 27 | file_name=file_name)) 28 | if os.path.exists(imgpath): 29 | image_data = open(imgpath, "rb").read() 30 | response = make_response(image_data) 31 | response.headers['Content-Type'] = 'image/jpg' 32 | return response 33 | else: 34 | image_data = open(Config.SCREENSHOT_FAIL_IMG, "rb").read() 35 | response = make_response(image_data) 36 | response.headers['Content-Type'] = 'image/jpg' 37 | return response 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /app/routes/npoc_service.py: -------------------------------------------------------------------------------- 1 | from flask_restx import Resource, Api, reqparse, fields, Namespace 2 | from app.utils import get_logger, auth 3 | from . import base_query_fields, ARLResource, get_arl_parser 4 | 5 | ns = Namespace('npoc_service', description="系统服务(python)信息") 6 | 7 | logger = get_logger() 8 | 9 | base_search_fields = { 10 | 'scheme': fields.String(description="系统服务名称"), 11 | 'host': fields.String(required=False, description="host"), 12 | 'port': fields.String(description="端口号"), 13 | 'target': fields.String(description="目标"), 14 | "task_id": fields.String(description="任务ID") 15 | } 16 | 17 | base_search_fields.update(base_query_fields) 18 | 19 | 20 | @ns.route('/') 21 | class NpocService(ARLResource): 22 | parser = get_arl_parser(base_search_fields, location='args') 23 | 24 | @auth 25 | @ns.expect(parser) 26 | def get(self): 27 | """ 28 | 服务信息(python)查询 29 | """ 30 | args = self.parser.parse_args() 31 | data = self.build_data(args=args, collection='npoc_service') 32 | 33 | return data 34 | -------------------------------------------------------------------------------- /app/routes/nuclei_result.py: -------------------------------------------------------------------------------- 1 | from bson import ObjectId 2 | from flask_restx import Resource, Api, reqparse, fields, Namespace 3 | from app.utils import get_logger, auth 4 | from . import base_query_fields, ARLResource, get_arl_parser 5 | from app import utils 6 | from app.modules import ErrorMsg 7 | 8 | ns = Namespace('nuclei_result', description="nuclei 扫描结果") 9 | 10 | logger = get_logger() 11 | 12 | base_search_fields = { 13 | 'template_url': fields.String(required=False, description="模版文件URL"), 14 | 'template_id': fields.String(description="模版id"), 15 | 'vuln_name': fields.String(description="漏洞名称"), 16 | 'vuln_severity': fields.String(description="漏洞等级"), 17 | 'vuln_url': fields.String(description="漏洞URL"), 18 | 'curl_command': fields.String(description="curl 命令"), 19 | 'target': fields.String(description="目标"), 20 | "task_id": fields.String(description="任务ID") 21 | } 22 | 23 | base_search_fields.update(base_query_fields) 24 | 25 | 26 | @ns.route('/') 27 | class ARLUrl(ARLResource): 28 | parser = get_arl_parser(base_search_fields, location='args') 29 | 30 | @auth 31 | @ns.expect(parser) 32 | def get(self): 33 | """ 34 | nuclei 扫描结果查询 35 | """ 36 | args = self.parser.parse_args() 37 | data = self.build_data(args=args, collection='nuclei_result') 38 | 39 | return data 40 | 41 | 42 | delete_nuclei_result_fields = ns.model('deleteNucleiResultFields', { 43 | '_id': fields.List(fields.String(required=True, description="nuclei 扫描结果 _id")) 44 | }) 45 | 46 | 47 | @ns.route('/delete/') 48 | class DeleteNucleiResult(ARLResource): 49 | @auth 50 | @ns.expect(delete_nuclei_result_fields) 51 | def post(self): 52 | """ 53 | 删除 nuclei 扫描结果 54 | """ 55 | args = self.parse_args(delete_nuclei_result_fields) 56 | id_list = args.pop('_id', []) 57 | for _id in id_list: 58 | query = {'_id': ObjectId(_id)} 59 | utils.conn_db('nuclei_result').delete_one(query) 60 | 61 | return utils.build_ret(ErrorMsg.Success, {'_id': id_list}) 62 | 63 | -------------------------------------------------------------------------------- /app/routes/poc.py: -------------------------------------------------------------------------------- 1 | from bson import ObjectId 2 | from flask_restx import Resource, Api, reqparse, fields, Namespace 3 | from app.utils import get_logger, auth 4 | from . import base_query_fields, ARLResource, get_arl_parser 5 | from app.services.npoc import NPoC 6 | from app import utils, celerytask 7 | from app.modules import ErrorMsg, TaskStatus, CeleryAction 8 | import copy 9 | 10 | ns = Namespace('poc', description="PoC信息") 11 | 12 | logger = get_logger() 13 | 14 | base_search_fields = { 15 | 'plugin_name': fields.String(description="PoC 名称 ID"), 16 | 'app_name': fields.String(description="应用名称"), 17 | 'scheme': fields.String(description="支持的协议"), 18 | 'vul_name': fields.String(description="漏洞名称"), 19 | 'plugin_type': fields.String(description="插件类别", enum=['poc', 'brute']), 20 | 'update_date': fields.String(description="更新时间"), 21 | 'category': fields.String(description="PoC 分类") 22 | } 23 | 24 | base_search_fields.update(base_query_fields) 25 | 26 | 27 | @ns.route('/') 28 | class ARLPoC(ARLResource): 29 | parser = get_arl_parser(base_search_fields, location='args') 30 | 31 | @auth 32 | @ns.expect(parser) 33 | def get(self): 34 | """ 35 | PoC 信息查询 36 | """ 37 | args = self.parser.parse_args() 38 | data = self.build_data(args=args, collection='poc') 39 | 40 | return data 41 | 42 | 43 | @ns.route('/sync/') 44 | class ARLPoCSync(ARLResource): 45 | 46 | @auth 47 | def get(self): 48 | """ 49 | 更新 PoC 信息 50 | """ 51 | n = NPoC() 52 | plugin_cnt = len(n.plugin_name_list) 53 | n.sync_to_db() 54 | n.delete_db() 55 | 56 | return utils.build_ret(ErrorMsg.Success, {"plugin_cnt": plugin_cnt}) 57 | 58 | 59 | @ns.route('/delete/') 60 | class ARLPoCDelete(ARLResource): 61 | 62 | @auth 63 | def get(self): 64 | """ 65 | 清空 PoC 信息 66 | """ 67 | result = utils.conn_db('poc').delete_many({}) 68 | 69 | delete_cnt = result.deleted_count 70 | 71 | return utils.build_ret(ErrorMsg.Success, {"delete_cnt": delete_cnt}) 72 | 73 | 74 | -------------------------------------------------------------------------------- /app/routes/service.py: -------------------------------------------------------------------------------- 1 | from flask_restx import Resource, Api, reqparse, fields, Namespace 2 | from app.utils import get_logger, auth 3 | from . import base_query_fields, ARLResource, get_arl_parser 4 | 5 | ns = Namespace('service', description="系统服务信息") 6 | 7 | logger = get_logger() 8 | 9 | base_search_fields = { 10 | 'service_name': fields.String(description="系统服务名称"), 11 | 'service_info.ip': fields.String(required=False, description="IP"), 12 | 'service_info.port_id': fields.Integer(description="端口号"), 13 | 'service_info.version': fields.String(description="系统服务版本"), 14 | 'service_info.product': fields.String(description="产品"), 15 | "task_id": fields.String(description="任务ID") 16 | } 17 | 18 | base_search_fields.update(base_query_fields) 19 | 20 | 21 | @ns.route('/') 22 | class ARLService(ARLResource): 23 | parser = get_arl_parser(base_search_fields, location='args') 24 | 25 | @auth 26 | @ns.expect(parser) 27 | def get(self): 28 | """ 29 | 服务信息查询 30 | """ 31 | args = self.parser.parse_args() 32 | data = self.build_data(args=args, collection='service') 33 | 34 | return data 35 | -------------------------------------------------------------------------------- /app/routes/stat_finger.py: -------------------------------------------------------------------------------- 1 | from flask_restx import fields, Namespace 2 | from app.utils import get_logger, auth 3 | from . import base_query_fields, ARLResource, get_arl_parser 4 | 5 | ns = Namespace('stat_finger', description="指纹统计信息") 6 | 7 | logger = get_logger() 8 | 9 | base_search_fields = { 10 | 'name': fields.String(required=False, description="指纹名称"), # 字段名没搞好 11 | "task_id": fields.String(description="任务 ID"), 12 | "cnt": fields.Integer(description="数目"), 13 | } 14 | 15 | base_search_fields.update(base_query_fields) 16 | 17 | 18 | @ns.route('/') 19 | class ARLStatFingerprint(ARLResource): 20 | parser = get_arl_parser(base_search_fields, location='args') 21 | 22 | @auth 23 | @ns.expect(parser) 24 | def get(self): 25 | """ 26 | 指纹统计信息查询 27 | """ 28 | args = self.parser.parse_args() 29 | data = self.build_data(args=args, collection='stat_finger') 30 | 31 | return data 32 | -------------------------------------------------------------------------------- /app/routes/url.py: -------------------------------------------------------------------------------- 1 | from flask_restx import Resource, Api, reqparse, fields, Namespace 2 | from app.utils import get_logger, auth 3 | from . import base_query_fields, ARLResource, get_arl_parser 4 | 5 | ns = Namespace('url', description="URL信息") 6 | 7 | logger = get_logger() 8 | 9 | base_search_fields = { 10 | 'fld': fields.String(required=False, description="IP"), 11 | 'site': fields.String(description="域名"), 12 | 'url': fields.String(required=False, description="URL"), 13 | 'content_length': fields.Integer(description="body 长度"), 14 | 'status_code': fields.Integer(description="状态码"), 15 | 'title': fields.String(description="标题"), 16 | 'source': fields.String(description="来源"), 17 | "task_id": fields.String(description="任务ID") 18 | } 19 | 20 | base_search_fields.update(base_query_fields) 21 | 22 | 23 | @ns.route('/') 24 | class ARLUrl(ARLResource): 25 | parser = get_arl_parser(base_search_fields, location='args') 26 | 27 | @auth 28 | @ns.expect(parser) 29 | def get(self): 30 | """ 31 | URL信息查询 32 | """ 33 | args = self.parser.parse_args() 34 | data = self.build_data(args=args, collection='url') 35 | 36 | return data 37 | 38 | 39 | @ns.route('/export/') 40 | class ARLUrlExport(ARLResource): 41 | parser = get_arl_parser(base_search_fields, location='args') 42 | 43 | @auth 44 | @ns.expect(parser) 45 | def get(self): 46 | """ 47 | URL 导出 48 | """ 49 | args = self.parser.parse_args() 50 | response = self.send_export_file(args=args, _type="url") 51 | 52 | return response 53 | -------------------------------------------------------------------------------- /app/routes/user.py: -------------------------------------------------------------------------------- 1 | from flask import request 2 | from flask_restx import fields, Namespace 3 | from app.utils import get_logger 4 | from app import utils 5 | from . import ARLResource 6 | from app import modules 7 | 8 | ns = Namespace('user', description="管理员登录认证") 9 | 10 | logger = get_logger() 11 | 12 | 13 | 14 | login_fields = ns.model('LoginARL', { 15 | 'username': fields.String(required=True, description="用户名"), 16 | 'password': fields.String(required=True, description="密码"), 17 | }) 18 | 19 | 20 | @ns.route('/login') 21 | class LoginARL(ARLResource): 22 | 23 | @ns.expect(login_fields) 24 | def post(self): 25 | """ 26 | 用户登录 27 | """ 28 | args = self.parse_args(login_fields) 29 | 30 | 31 | return build_data(utils.user_login(**args)) 32 | 33 | 34 | 35 | 36 | @ns.route('/logout') 37 | class LogoutARL(ARLResource): 38 | 39 | def get(self): 40 | """ 41 | 用户退出 42 | """ 43 | token = request.headers.get("Token") 44 | utils.user_logout(token) 45 | 46 | return build_data({}) 47 | 48 | 49 | change_pass_fields = ns.model('ChangePassARL', { 50 | 'old_password': fields.String(required=True, description="旧密码"), 51 | 'new_password': fields.String(required=True, description="新密码"), 52 | 'check_password': fields.String(required=True, description="确认密码"), 53 | }) 54 | 55 | 56 | @ns.route('/change_pass') 57 | class ChangePassARL(ARLResource): 58 | @ns.expect(change_pass_fields) 59 | def post(self): 60 | """ 61 | 密码修改 62 | """ 63 | args = self.parse_args(change_pass_fields) 64 | ret = { 65 | "message": "success", 66 | "code": 200, 67 | "data": {} 68 | } 69 | token = request.headers.get("Token") 70 | 71 | if args["new_password"] != args["check_password"]: 72 | ret["code"] = 301 73 | ret["message"] = "新密码和确定密码不一致" 74 | return ret 75 | 76 | if not args["new_password"]: 77 | ret["code"] = 302 78 | ret["message"] = "新密码不能为空" 79 | return ret 80 | 81 | if utils.change_pass(token, args["old_password"], args["new_password"]): 82 | utils.user_logout(token) 83 | else: 84 | ret["message"] = "旧密码错误" 85 | ret["code"] = 303 86 | 87 | return ret 88 | 89 | 90 | def build_data(data): 91 | ret = { 92 | "message": "success", 93 | "code": 200, 94 | "data": {} 95 | } 96 | 97 | if data: 98 | ret["data"] = data 99 | else: 100 | ret["code"] = 401 101 | 102 | return ret 103 | 104 | 105 | -------------------------------------------------------------------------------- /app/routes/vuln.py: -------------------------------------------------------------------------------- 1 | from bson import ObjectId 2 | from flask_restx import Resource, Api, reqparse, fields, Namespace 3 | from app.utils import get_logger, auth 4 | from . import base_query_fields, ARLResource, get_arl_parser 5 | from app import utils 6 | from app.modules import ErrorMsg 7 | 8 | ns = Namespace('vuln', description="漏洞信息") 9 | 10 | logger = get_logger() 11 | 12 | base_search_fields = { 13 | 'plg_name': fields.String(required=False, description="plugin ID"), 14 | 'plg_type': fields.String(description="类别"), 15 | 'vul_name': fields.String(description="漏洞名称"), 16 | 'app_name': fields.String(description="应用名"), 17 | 'target': fields.String(description="目标"), 18 | "task_id": fields.String(description="任务ID") 19 | } 20 | 21 | base_search_fields.update(base_query_fields) 22 | 23 | 24 | @ns.route('/') 25 | class ARLUrl(ARLResource): 26 | parser = get_arl_parser(base_search_fields, location='args') 27 | 28 | @auth 29 | @ns.expect(parser) 30 | def get(self): 31 | """ 32 | 漏洞 信息查询 33 | """ 34 | args = self.parser.parse_args() 35 | data = self.build_data(args=args, collection='vuln') 36 | 37 | return data 38 | 39 | 40 | delete_vuln_fields = ns.model('deleteVulnFields', { 41 | '_id': fields.List(fields.String(required=True, description="风险信息 _id")) 42 | }) 43 | 44 | 45 | @ns.route('/delete/') 46 | class DeleteARLVuln(ARLResource): 47 | @auth 48 | @ns.expect(delete_vuln_fields) 49 | def post(self): 50 | """ 51 | 删除 风险信息 52 | """ 53 | args = self.parse_args(delete_vuln_fields) 54 | id_list = args.pop('_id', []) 55 | for _id in id_list: 56 | query = {'_id': ObjectId(_id)} 57 | utils.conn_db('vuln').delete_one(query) 58 | 59 | return utils.build_ret(ErrorMsg.Success, {'_id': id_list}) 60 | 61 | -------------------------------------------------------------------------------- /app/routes/wih.py: -------------------------------------------------------------------------------- 1 | from flask_restx import fields, Namespace 2 | from app.utils import get_logger, auth 3 | from . import base_query_fields, ARLResource, get_arl_parser 4 | from bson import ObjectId 5 | from app.modules import ErrorMsg 6 | from app import utils 7 | 8 | ns = Namespace('wih', description="WEB Info Hunter 信息") 9 | 10 | logger = get_logger() 11 | 12 | base_search_fields = { 13 | 'record_type': fields.String(required=False, description="记录类型"), 14 | 'record_type__neq': fields.String(required=False, description="记录类型不等于（全匹配）"), 15 | 'record_type__not': fields.String(required=False, description="记录类型不包含"), 16 | 'content': fields.String(description="内容"), 17 | 'source': fields.String(description="来源 JS URL"), 18 | 'site': fields.String(description="站点URL"), 19 | 'task_id': fields.String(description="任务ID"), 20 | } 21 | 22 | 23 | base_search_fields.update(base_query_fields) 24 | 25 | 26 | @ns.route('/') 27 | class ARLWebInfoHunter(ARLResource): 28 | parser = get_arl_parser(base_search_fields, location='args') 29 | 30 | @auth 31 | @ns.expect(parser) 32 | def get(self): 33 | """ 34 | WEB Info Hunter 信息查询 35 | """ 36 | args = self.parser.parse_args() 37 | data = self.build_data(args=args, collection='wih') 38 | 39 | return data 40 | 41 | 42 | @ns.route('/export/') 43 | class ARLWihExport(ARLResource): 44 | parser = get_arl_parser(base_search_fields, location='args') 45 | 46 | @auth 47 | @ns.expect(parser) 48 | def get(self): 49 | """ 50 | WIH 导出 51 | """ 52 | args = self.parser.parse_args() 53 | response = self.send_export_file(args=args, _type="wih") 54 | 55 | return response 56 | 57 | 58 | delete_wih_fields = ns.model('deleteWih', { 59 | '_id': fields.List(fields.String(required=True, description="数据_id")) 60 | }) 61 | 62 | 63 | @ns.route('/delete/') 64 | class DeleteARLWIH(ARLResource): 65 | @auth 66 | @ns.expect(delete_wih_fields) 67 | def post(self): 68 | """ 69 | 删除任务中的 wih 数据 70 | """ 71 | args = self.parse_args(delete_wih_fields) 72 | id_list = args.pop('_id', "") 73 | for _id in id_list: 74 | query = {'_id': ObjectId(_id)} 75 | utils.conn_db('wih').delete_one(query) 76 | 77 | return utils.build_ret(ErrorMsg.Success, {'_id': id_list}) -------------------------------------------------------------------------------- /app/services/__init__.py: -------------------------------------------------------------------------------- 1 | from .altDNS import alt_dns 2 | from .massdns import mass_dns 3 | from .portScan import port_scan 4 | from .resolverDomain import resolver_domain 5 | from .checkHTTP import check_http 6 | from .siteScreenshot import site_screenshot 7 | from .fetchSite import fetch_favicon, fetch_site 8 | from .probeHTTP import probe_http 9 | from .buildDomainInfo import build_domain_info 10 | from .searchEngines import baidu_search, bing_search 11 | from .siteUrlSpider import site_spider, site_spider_thread 12 | from .webAnalyze import web_analyze 13 | from .fofaClient import fofa_query 14 | from .fetchCert import fetch_cert 15 | from .fileLeak import file_leak 16 | from .pageFetch import page_fetch 17 | from .syncAsset import sync_asset 18 | from .npoc import run_risk_cruising, run_sniffer 19 | from .autoTag import auto_tag 20 | from .githubSearch import github_search 21 | from .infoHunter import run_wih 22 | from .baseUpdateTask import BaseUpdateTask 23 | from .domainSiteUpdate import domain_site_update 24 | from .dns_query import run_query_plugin 25 | from .fingerprint_cache import finger_db_cache, finger_db_identify, have_human_rule_from_db 26 | from .fingerprint import FingerPrint 27 | from .expr import evaluate_expression, check_expression, check_expression_with_error 28 | -------------------------------------------------------------------------------- /app/services/baseThread.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import collections 3 | import time 4 | import requests.exceptions 5 | from lxml import etree 6 | from app import utils 7 | from app.modules import DomainInfo 8 | logger = utils.get_logger() 9 | 10 | 11 | class BaseThread(object): 12 | def __init__(self, targets, concurrency=6): 13 | self.concurrency = concurrency 14 | self.semaphore = threading.Semaphore(concurrency) 15 | self.targets = targets 16 | 17 | def work(self, site): 18 | raise NotImplementedError() 19 | 20 | def _work(self, url): 21 | try: 22 | self.work(url) 23 | except requests.exceptions.RequestException as e: 24 | pass 25 | 26 | except etree.Error as e: 27 | pass 28 | 29 | except Exception as e: 30 | logger.warning("error on {}".format(url)) 31 | logger.exception(e) 32 | 33 | except BaseException as e: 34 | logger.warning("BaseException on {}".format(url)) 35 | raise e 36 | finally: 37 | self.semaphore.release() 38 | 39 | def _run(self): 40 | deque = collections.deque(maxlen=5000) 41 | cnt = 0 42 | 43 | for target in self.targets: 44 | if isinstance(target, str): 45 | target = target.strip() 46 | 47 | cnt += 1 48 | logger.debug("[{}/{}] work on {}".format(cnt, len(self.targets), target)) 49 | 50 | if not target: 51 | continue 52 | 53 | self.semaphore.acquire() 54 | t1 = threading.Thread(target=self._work, args=(target,)) 55 | # 可以快速结束程序 56 | t1.setDaemon(True) 57 | t1.start() 58 | 59 | deque.append(t1) 60 | 61 | for t in list(deque): 62 | while t.is_alive(): 63 | time.sleep(0.2) 64 | 65 | 66 | class ThreadMap(BaseThread): 67 | def __init__(self, fun, items, arg=None, concurrency=6): 68 | super(ThreadMap, self).__init__(targets=items, concurrency=concurrency) 69 | if not callable(fun): 70 | raise TypeError("fun must be callable.") 71 | 72 | self._arg = arg 73 | self._fun = fun 74 | self._result_map = {} 75 | 76 | def work(self, item): 77 | if self._arg: 78 | result = self._fun(item, self._arg) 79 | else: 80 | result = self._fun(item) 81 | 82 | if result: 83 | self._result_map[str(item)] = result 84 | 85 | def run(self): 86 | self._run() 87 | return self._result_map 88 | 89 | 90 | def thread_map(fun, items, arg=None, concurrency=6): 91 | t = ThreadMap(fun=fun, items=items, arg=arg, concurrency=concurrency) 92 | return t.run() 93 | 94 | 95 | -------------------------------------------------------------------------------- /app/services/baseUpdateTask.py: -------------------------------------------------------------------------------- 1 | from bson import ObjectId 2 | from app import utils 3 | 4 | 5 | # 用于更新任务状态 6 | class BaseUpdateTask(object): 7 | def __init__(self, task_id: str): 8 | self.task_id = task_id 9 | 10 | def update_services(self, service_name: str, elapsed: float): 11 | elapsed = "{:.2f}".format(elapsed) 12 | self.update_task_field("status", service_name) 13 | query = {"_id": ObjectId(self.task_id)} 14 | update = {"$push": {"service": {"name": service_name, "elapsed": float(elapsed)}}} 15 | utils.conn_db('task').update_one(query, update) 16 | 17 | def update_task_field(self, field=None, value=None): 18 | query = {"_id": ObjectId(self.task_id)} 19 | update = {"$set": {field: value}} 20 | utils.conn_db('task').update_one(query, update) -------------------------------------------------------------------------------- /app/services/buildDomainInfo.py: -------------------------------------------------------------------------------- 1 | import time 2 | from app import modules 3 | from app import utils 4 | from .baseThread import BaseThread 5 | logger = utils.get_logger() 6 | 7 | 8 | class BuildDomainInfo(BaseThread): 9 | def __init__(self, domains, concurrency=6): 10 | super().__init__(domains, concurrency=concurrency) 11 | self.domain_info_list = [] 12 | 13 | def work(self, target): 14 | domain = target 15 | if hasattr(target, "domain"): 16 | domain = target.domain 17 | 18 | # 不记录日志 19 | ips = utils.get_ip(domain, log_flag=False) 20 | if not ips: 21 | return 22 | 23 | cnames = utils.get_cname(domain, False) 24 | 25 | info = { 26 | "domain": domain, 27 | "type": "A", 28 | "record": ips, 29 | "ips": ips 30 | } 31 | 32 | if cnames: 33 | info["type"] = 'CNAME' 34 | info["record"] = cnames 35 | 36 | self.domain_info_list.append(modules.DomainInfo(**info)) 37 | 38 | def run(self): 39 | t1 = time.time() 40 | logger.info("start build Domain info {}".format(len(self.targets))) 41 | self._run() 42 | elapse = time.time() - t1 43 | logger.info("end build Domain info {} elapse {}".format(len(self.domain_info_list), elapse)) 44 | 45 | return self.domain_info_list 46 | 47 | 48 | def build_domain_info(domains, concurrency=15): 49 | p = BuildDomainInfo(domains, concurrency=concurrency) 50 | return p.run() 51 | -------------------------------------------------------------------------------- /app/services/checkHTTP.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from app import utils 4 | from .baseThread import BaseThread 5 | 6 | import requests.exceptions 7 | logger = utils.get_logger() 8 | 9 | 10 | class CheckHTTP(BaseThread): 11 | def __init__(self, urls, concurrency=10): 12 | super().__init__(urls, concurrency=concurrency) 13 | self.timeout = (5, 3) 14 | self.checkout_map = {} 15 | 16 | def check(self, url): 17 | conn = utils.http_req(url, method="get", timeout=self.timeout, stream=True) 18 | conn.close() 19 | 20 | if conn.status_code == 400: 21 | # 特殊情况排除 22 | etag = conn.headers.get("ETag") 23 | date = conn.headers.get("Date") 24 | if not etag or not date: 25 | return None 26 | 27 | # *** 特殊情况过滤 28 | if conn.status_code == 422 or conn.status_code == 410: 29 | return None 30 | 31 | if (conn.status_code >= 501) and (conn.status_code < 600): 32 | return None 33 | 34 | if conn.status_code == 403: 35 | conn2 = utils.http_req(url) 36 | check = b'