├── .github └── workflows │ └── docker.yml ├── .gitignore ├── .idea ├── .gitignore ├── inspectionProfiles │ └── Project_Default.xml ├── misc.xml ├── modules.xml ├── monitorizerv3.iml └── workspace.xml ├── .pre-commit-config.yaml ├── Dockerfile ├── README.md ├── assets ├── dash_light.png ├── discord_report.png ├── login.png ├── scans_dark.png └── telegram_report.png ├── conf └── nginx.conf ├── docker-compose.yml ├── monitorizer ├── __init__.py ├── configure.py ├── default.yml ├── inventory │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── callback.py │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_alter_domainscan_status.py │ │ └── __init__.py │ ├── models.py │ └── tasks.py ├── manage.py ├── report │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── migrations │ │ ├── 0001_initial.py │ │ └── __init__.py │ ├── models.py │ └── tasks.py ├── server │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── asgi.py │ ├── celery.py │ ├── management │ │ └── commands │ │ │ ├── openresty.conf.jinja │ │ │ └── openresty_template.py │ ├── models.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── settings.py ├── templates │ └── admin │ │ ├── components │ │ └── chart │ │ │ └── pie.html │ │ └── index.html └── utils │ └── engine.py ├── monitorizerv3.iml ├── pyproject.toml └── webserver-entrypoint.sh /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Docker Image 2 | 3 | on: 4 | push: 5 | branches: [ "*" ] 6 | tags: [ 'v*.*.*' ] 7 | paths: 8 | - '.github/workflows/docker.yml' 9 | - 'conf/**' 10 | - 'monitorizer/**' 11 | - 'pyproject.toml' 12 | - 'Dockerfile' 13 | - '*-entrypoint.sh' 14 | pull_request: 15 | branches: [ "*" ] 16 | paths: 17 | - '.github/workflows/docker.yml' 18 | - 'conf/**' 19 | - 'monitorizer/**' 20 | - 'pyproject.toml' 21 | - 'Dockerfile' 22 | - '*-entrypoint.sh' 23 | env: 24 | REGISTRY: ghcr.io 25 | PEP_IMAGE: ${{ github.repository }} 26 | BRANCH_NAME: ${{ github.head_ref || github.ref_name }} 27 | 28 | jobs: 29 | build: 30 | runs-on: ubuntu-latest 31 | permissions: 32 | contents: read 33 | packages: write 34 | id-token: write 35 | 36 | steps: 37 | - name: Checkout repository 38 | uses: actions/checkout@v2 39 | 40 | - name: Setup Docker buildx 41 | uses: docker/setup-buildx-action@v2 42 | 43 | - name: Log into registry ${{ env.REGISTRY }} 44 | if: github.event_name != 'pull_request' 45 | uses: docker/login-action@v2 46 | with: 47 | registry: ${{ env.REGISTRY }} 48 | username: ${{ github.actor }} 49 | password: ${{ secrets.GITHUB_TOKEN }} 50 | 51 | - name: Extract Docker metadata 52 | id: meta_base 53 | uses: docker/metadata-action@v4 54 | with: 55 | images: ${{ env.REGISTRY }}/${{ env.PEP_IMAGE }} 56 | 57 | - name: Build and push pep image 58 | uses: docker/build-push-action@v4 59 | with: 60 | context: . 61 | build-args: | 62 | version=${{ github.sha }}@${{ github.ref_name }} 63 | push: ${{ github.event_name != 'pull_request' }} 64 | tags: ${{ steps.meta_base.outputs.tags }} 65 | labels: ${{ steps.meta_base.outputs.labels }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### App Files 2 | **/*.sqlite3 3 | **/.*.rdb 4 | poetry.lock 5 | 6 | ### Python template 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | cover/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | .pybuilder/ 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | # For a library or package, you might want to ignore these files since the code is 93 | # intended to run in multiple environments; otherwise, check them in: 94 | # .python-version 95 | 96 | # pipenv 97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 100 | # install all needed dependencies. 101 | #Pipfile.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | 110 | # pdm 111 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 112 | #pdm.lock 113 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 114 | # in version control. 115 | # https://pdm.fming.dev/#use-with-ide 116 | .pdm.toml 117 | 118 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 119 | __pypackages__/ 120 | 121 | # Celery stuff 122 | celerybeat-schedule 123 | celerybeat.pid 124 | 125 | # SageMath parsed files 126 | *.sage.py 127 | 128 | # Environments 129 | .env 130 | .venv 131 | env/ 132 | venv/ 133 | ENV/ 134 | env.bak/ 135 | venv.bak/ 136 | 137 | # Spyder project settings 138 | .spyderproject 139 | .spyproject 140 | 141 | # Rope project settings 142 | .ropeproject 143 | 144 | # mkdocs documentation 145 | /site 146 | 147 | # mypy 148 | .mypy_cache/ 149 | .dmypy.json 150 | dmypy.json 151 | 152 | # Pyre type checker 153 | .pyre/ 154 | 155 | # pytype static type analyzer 156 | .pytype/ 157 | 158 | # Cython debug symbols 159 | cython_debug/ 160 | 161 | # PyCharm 162 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 163 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 164 | # and can be added to the global gitignore or merged into this file. For a more nuclear 165 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 166 | #.idea/ 167 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitTheByte/Monitorizer/8cf2a8c60076bdb7bcd4d9a7cefe2615c676f8c5/.idea/.gitignore -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 30 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/monitorizerv3.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 16 | 17 | -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | 13 | 14 | 16 | 19 | 20 | 21 | 24 | 41 | 42 | 43 | 44 | 45 | 1712759248287 46 | 51 | 52 | 53 | 54 | 56 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pycqa/isort 3 | rev: 5.6.4 4 | hooks: 5 | - id: isort 6 | args: ["--profile", "black", "--filter-files"] 7 | 8 | - repo: https://github.com/psf/black 9 | rev: 22.3.0 10 | hooks: 11 | - id: black 12 | language_version: python3.9 -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-bullseye 2 | 3 | ENV PYTHONUNBUFFERED=1 4 | 5 | # Install openresty 6 | RUN apt-get update && apt-get install -y --no-install-recommends gnupg curl ca-certificates && \ 7 | curl https://openresty.org/package/pubkey.gpg -o pubkey.gpg && apt-key add pubkey.gpg && rm -rf pubkey.gpg && \ 8 | echo "deb http://openresty.org/package/debian bullseye openresty" | tee /etc/apt/sources.list.d/openresty.list && \ 9 | apt-get update && \ 10 | apt-get -y --no-install-recommends install openresty 11 | 12 | # Add docker binary 13 | COPY --from=docker:latest /usr/local/bin/docker /usr/local/bin/ 14 | 15 | # Add main entrypoint with tini 16 | ADD https://github.com/krallin/tini/releases/download/v0.19.0/tini /usr/bin/tini 17 | RUN chmod +x /usr/bin/tini 18 | 19 | # Fake package to enable dependency caching 20 | RUN mkdir -p /opt/src/monitorizer && touch /opt/src/monitorizer/__init__.py 21 | COPY README.md /opt/src/README.md 22 | COPY pyproject.toml /opt/src/pyproject.toml 23 | RUN pip3 --default-timeout=1000 install /opt/src 24 | 25 | # Real codebase 26 | COPY monitorizer /opt/src/monitorizer 27 | RUN pip3 install /opt/src/ --break-system-package && rm -rf /opt/src 28 | 29 | COPY conf/nginx.conf /etc/openresty/nginx.conf 30 | 31 | COPY *-entrypoint.sh / 32 | RUN chmod +x /*-entrypoint.sh 33 | 34 | ENTRYPOINT ["/usr/bin/tini", "-g", "--"] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Monitorizer 2 |

3 | 4 | 5 | 6 |

The Ultimate Subdomain Monitoring Framework

7 |

8 | 9 |

10 | 11 | 12 | 13 | 14 | 15 | 16 |
17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 |

27 | 28 | 29 | # Installation 30 | 31 | | :exclamation: **Disclaimer** | 32 | |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 33 | | **Admin users are able to execute code on the local server make sure to change the default password at otherwise use it at your own risk** | 34 | 35 | ### Using docker 36 | ```bash 37 | $ curl https://raw.githubusercontent.com/BitTheByte/Monitorizer/main/docker-compose.yml | docker compose -f - up 38 | ``` 39 | 40 | ### Default Credentials: 41 | Once everything is ready you will be able to login to the dashboard at http://127.0.0.1:8000/dashboard/ 42 | 43 | | Username | Email | Password | 44 | |----------|----------------------------|----------| 45 | | admin | monitorizer@bitthebyte.com | P@ssW0rd | 46 | 47 | # Features 48 | ### Dark-Light Mode 49 | ![](assets/dash_light.png) 50 | 51 | ![](assets/scans_dark.png) 52 | 53 | ### Scalable 54 | Monitorizer fully designed to run on large scale and handle thousands of distributed operations effortlessly 55 | 56 | ### Events 57 | Monitorizer supports various reporting channels to ensure you're always informed. 58 | 59 | **Telegram**: Updates and alerts are directly sent to your specified Telegram channel, allowing for instant notifications and immediate team collaboration. 60 | 61 | ![](assets/telegram_report.png) 62 | 63 | **Webhook**: Receive detailed reports and alerts through your webhook server to keep you aligned. 64 | 65 | ![](assets/discord_report.png) 66 | 67 | 68 | ### Extendable 69 | Employ a zero-code strategy to integrate your own tools for domain enumeration and discovery effortlessly. Customize and extend functionalities without writing any new code. 70 | 71 | ### Advanced Search 72 | Utilize the dashboard to conduct comprehensive searches across all Monitorizer assets. This feature allows for quick location of necessary data points, streamlined through an intuitive search interface. 73 | 74 | ### Import-Export 75 | Utilize the dashboard to import and export assets easily -------------------------------------------------------------------------------- /assets/dash_light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitTheByte/Monitorizer/8cf2a8c60076bdb7bcd4d9a7cefe2615c676f8c5/assets/dash_light.png -------------------------------------------------------------------------------- /assets/discord_report.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitTheByte/Monitorizer/8cf2a8c60076bdb7bcd4d9a7cefe2615c676f8c5/assets/discord_report.png -------------------------------------------------------------------------------- /assets/login.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitTheByte/Monitorizer/8cf2a8c60076bdb7bcd4d9a7cefe2615c676f8c5/assets/login.png -------------------------------------------------------------------------------- /assets/scans_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitTheByte/Monitorizer/8cf2a8c60076bdb7bcd4d9a7cefe2615c676f8c5/assets/scans_dark.png -------------------------------------------------------------------------------- /assets/telegram_report.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitTheByte/Monitorizer/8cf2a8c60076bdb7bcd4d9a7cefe2615c676f8c5/assets/telegram_report.png -------------------------------------------------------------------------------- /conf/nginx.conf: -------------------------------------------------------------------------------- 1 | worker_processes auto; 2 | worker_rlimit_nofile 1047552; 3 | 4 | events { 5 | multi_accept on; 6 | worker_connections 16384; 7 | use epoll; 8 | } 9 | 10 | http { 11 | ## 12 | # Basic Settings 13 | ## 14 | aio threads; 15 | aio_write on; 16 | sendfile on; 17 | tcp_nopush on; 18 | tcp_nodelay on; 19 | server_tokens off; 20 | reset_timedout_connection on; 21 | 22 | keepalive_timeout 120s; 23 | keepalive_requests 25000; 24 | 25 | client_body_temp_path /tmp/client-body; 26 | proxy_temp_path /tmp/proxy-temp; 27 | 28 | client_header_buffer_size 1k; 29 | client_header_timeout 60s; 30 | large_client_header_buffers 4 8k; 31 | client_body_buffer_size 8k; 32 | client_body_timeout 60s; 33 | client_max_body_size 4096M; 34 | 35 | http2_max_concurrent_streams 128; 36 | 37 | types_hash_max_size 2048; 38 | server_names_hash_max_size 1024; 39 | server_names_hash_bucket_size 32; 40 | map_hash_bucket_size 64; 41 | 42 | proxy_headers_hash_max_size 512; 43 | proxy_headers_hash_bucket_size 64; 44 | 45 | variables_hash_bucket_size 256; 46 | variables_hash_max_size 2048; 47 | 48 | underscores_in_headers off; 49 | ignore_invalid_headers on; 50 | 51 | resolver 1.1.1.1 8.8.8.8 valid=300s ipv6=off; 52 | include mime.types; 53 | default_type application/octet-stream; 54 | 55 | 56 | ## 57 | # Gzip Settings 58 | ## 59 | gzip on; 60 | gzip_comp_level 2; 61 | gzip_http_version 1.0; 62 | gzip_min_length 1024; 63 | gzip_proxied expired no-cache no-store private auth; 64 | gzip_types 65 | text/css 66 | text/plain 67 | text/javascript 68 | application/javascript 69 | application/json 70 | application/x-javascript 71 | application/xml 72 | application/xml+rss 73 | application/xhtml+xml 74 | application/x-font-ttf 75 | application/x-font-opentype 76 | application/vnd.ms-fontobject 77 | image/svg+xml 78 | image/x-icon 79 | application/rss+xml 80 | application/atom_xml; 81 | 82 | ## 83 | # Logging Settings 84 | ## 85 | access_log off; 86 | error_log /var/log/openresty/error.log; 87 | 88 | map $http_upgrade $connection_upgrade { 89 | default upgrade; 90 | '' ''; 91 | } 92 | 93 | include /etc/openresty/conf.d/*.conf; 94 | } -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | db: 5 | image: postgres:latest 6 | restart: always 7 | command: "-c max_connections=512" 8 | environment: 9 | POSTGRES_USER: postgres 10 | POSTGRES_PASSWORD: postgres 11 | POSTGRES_DB: postgres 12 | volumes: 13 | - db_data:/var/lib/postgresql/data 14 | pgbouncer: 15 | image: edoburu/pgbouncer:latest 16 | environment: 17 | - DB_HOST=db 18 | - DB_PORT=5432 19 | - DB_USER=postgres 20 | - DB_PASSWORD=postgres 21 | - AUTH_TYPE=scram-sha-256 22 | - DEFAULT_POOL_SIZE=512 23 | - MAX_CLIENT_CONN=10000 24 | - POOL_MODE=transaction 25 | rabbitmq: 26 | image: rabbitmq:3-management-alpine 27 | restart: always 28 | volumes: 29 | - mq_data:/var/lib/rabbitmq/ 30 | healthcheck: 31 | test: rabbitmq-diagnostics -q ping 32 | interval: 30s 33 | timeout: 60s 34 | retries: 5 35 | web: 36 | build: . 37 | restart: always 38 | image: ghcr.io/bitthebyte/monitorizer:main 39 | command: /webserver-entrypoint.sh 40 | environment: 41 | POSTGRES_HOST: pgbouncer 42 | CELERY_BROKER_URL: 'amqp://guest:guest@rabbitmq:5672' 43 | ports: 44 | - "8000:8000" 45 | depends_on: 46 | - db 47 | - rabbitmq 48 | 49 | beat_worker: 50 | restart: always 51 | image: ghcr.io/bitthebyte/monitorizer:main 52 | command: celery -A monitorizer.server beat -l info 53 | depends_on: 54 | - db 55 | - rabbitmq 56 | environment: 57 | CELERY_BROKER_URL: 'amqp://guest:guest@rabbitmq:5672' 58 | POSTGRES_HOST: pgbouncer 59 | 60 | report_worker: 61 | restart: always 62 | image: ghcr.io/bitthebyte/monitorizer:main 63 | command: celery -A monitorizer.server worker -Q reports -l info 64 | depends_on: 65 | - db 66 | - rabbitmq 67 | environment: 68 | CELERY_BROKER_URL: 'amqp://guest:guest@rabbitmq:5672' 69 | POSTGRES_HOST: pgbouncer 70 | 71 | job_worker: 72 | restart: always 73 | privileged: true 74 | image: ghcr.io/bitthebyte/monitorizer:main 75 | command: celery -A monitorizer.server worker -Q default -l info 76 | depends_on: 77 | - db 78 | - rabbitmq 79 | environment: 80 | CELERY_BROKER_URL: 'amqp://guest:guest@rabbitmq:5672' 81 | POSTGRES_HOST: pgbouncer 82 | volumes: 83 | - /var/run/docker.sock:/var/run/docker.sock 84 | - /home/.monitorizer:/home/.monitorizer:shared 85 | 86 | volumes: 87 | db_data: 88 | mq_data: 89 | -------------------------------------------------------------------------------- /monitorizer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitTheByte/Monitorizer/8cf2a8c60076bdb7bcd4d9a7cefe2615c676f8c5/monitorizer/__init__.py -------------------------------------------------------------------------------- /monitorizer/configure.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import yaml 4 | 5 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "monitorizer.server.settings") 6 | 7 | from django.core.wsgi import get_wsgi_application 8 | 9 | application = get_wsgi_application() 10 | 11 | from django.conf import settings 12 | from django.contrib.auth.models import User 13 | 14 | from monitorizer.inventory import models as inventory_models 15 | 16 | email = os.environ.get("ADMIN_EMAIL", "monitorizer@bitthebyte.com") 17 | username = os.getenv("ADMIN_USERNAME", "admin") 18 | password = os.getenv("ADMIN_PASSWORD", "P@ssW0rd") 19 | 20 | system_user = User.objects.filter(username=username).first() 21 | if not system_user: 22 | User.objects.create_superuser( 23 | username=username, 24 | email=email, 25 | password=password, 26 | is_active=True, 27 | is_staff=True, 28 | is_superuser=True, 29 | ) 30 | 31 | defaults_config = yaml.safe_load(open(settings.BASE_DIR / "default.yml")) 32 | for command in defaults_config.get("commands", []): 33 | inventory_models.CommandTemplate.objects.get_or_create( 34 | id=command["id"], 35 | defaults={ 36 | "name": command["name"], 37 | "cmd": command["cmd"], 38 | "parser": command["parser"], 39 | }, 40 | ) 41 | -------------------------------------------------------------------------------- /monitorizer/default.yml: -------------------------------------------------------------------------------- 1 | commands: 2 | - id: 756db6c3-f2d2-4dd7-8b9e-1ce71772567e 3 | name: subfinder 4 | cmd: docker run -v /home/.monitorizer:/home/.monitorizer --rm projectdiscovery/subfinder:latest -d {domain} -o {output:file} -all 5 | parser: | 6 | import os 7 | 8 | output_file = __scan__['vars']['output:file'] 9 | if os.path.exists(output_file): 10 | __result__ = [v.strip() for v in open(output_file, 'r').readlines()] 11 | -------------------------------------------------------------------------------- /monitorizer/inventory/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitTheByte/Monitorizer/8cf2a8c60076bdb7bcd4d9a7cefe2615c676f8c5/monitorizer/inventory/__init__.py -------------------------------------------------------------------------------- /monitorizer/inventory/admin.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.contrib import admin 3 | from django_ace import AceWidget 4 | from import_export import resources 5 | from import_export.admin import ImportExportModelAdmin 6 | from unfold.admin import ModelAdmin, StackedInline 7 | from unfold.contrib.import_export.forms import ExportForm, ImportForm 8 | from unfold.decorators import display 9 | 10 | from monitorizer.inventory import models 11 | 12 | 13 | class SeedDomainResource(resources.ModelResource): 14 | class Meta: 15 | model = models.SeedDomain 16 | fields = ("value",) 17 | 18 | 19 | class CommandTemplateResource(resources.ModelResource): 20 | class Meta: 21 | model = models.CommandTemplate 22 | fields = ("name", "cmd", "parser") 23 | 24 | 25 | class ScanDescriptorAdmin(StackedInline): 26 | extra = 0 27 | model = models.ScanAutoSubmitter 28 | 29 | 30 | @admin.register(models.DiscoveredDomain) 31 | class DiscoveredDomainAdmin( 32 | ImportExportModelAdmin, 33 | ModelAdmin, 34 | ): 35 | export_form_class = ExportForm 36 | import_form_class = ImportForm 37 | list_display = ["id", "value", "created_at"] 38 | search_fields = ["id", "value"] 39 | 40 | def has_import_permission(self, request): 41 | return False 42 | 43 | def has_add_permission(self, request): 44 | return False 45 | 46 | def has_change_permission(self, request, obj=None): 47 | return False 48 | 49 | def get_queryset(self, request): 50 | return models.DiscoveredDomain.objects.order_by("-created_at") 51 | 52 | 53 | @admin.register(models.SeedDomain) 54 | class SeedDomainAdmin( 55 | ImportExportModelAdmin, 56 | ModelAdmin, 57 | ): 58 | resource_class = SeedDomainResource 59 | export_form_class = ExportForm 60 | import_form_class = ImportForm 61 | inlines = [ScanDescriptorAdmin] 62 | search_fields = ["id", "value"] 63 | list_filter = ["enabled"] 64 | list_display = ["id", "value", "enabled", "created_at"] 65 | add_fieldsets = ((None, {"fields": ["value"]}),) 66 | 67 | def get_readonly_fields(self, request, obj=None): 68 | if obj: 69 | return ["value"] 70 | return [] 71 | 72 | def get_queryset(self, request): 73 | return models.SeedDomain.objects.order_by("-created_at") 74 | 75 | 76 | @admin.register(models.DomainScan) 77 | class DomainScanAdmin(ModelAdmin, ImportExportModelAdmin): 78 | export_form_class = ExportForm 79 | import_form_class = ImportForm 80 | add_fieldsets = ((None, {"fields": ["domain", "command_tpl"]}),) 81 | list_display = [ 82 | "id", 83 | "command_name", 84 | "scan_domain", 85 | "status_label", 86 | "created_at", 87 | "finished_at", 88 | ] 89 | 90 | @display( 91 | description="Status", 92 | ordering="status", 93 | label={ 94 | models.DomainScan.ScanStatus.PENDING: "warning", 95 | models.DomainScan.ScanStatus.RUNNING: "info", 96 | models.DomainScan.ScanStatus.SUCCESS: "success", 97 | models.DomainScan.ScanStatus.ERROR: "danger", 98 | }, 99 | ) 100 | def status_label(self, obj): 101 | return obj.status 102 | 103 | @staticmethod 104 | def scan_domain(obj): 105 | return obj.domain.value 106 | 107 | @staticmethod 108 | def command_name(obj): 109 | return obj.command_tpl.name 110 | 111 | def has_change_permission(self, request, obj=None): 112 | return False 113 | 114 | def has_import_permission(self, request): 115 | return False 116 | 117 | def get_queryset(self, request): 118 | return models.DomainScan.objects.order_by("-created_at") 119 | 120 | 121 | class CommandTemplateForm(forms.ModelForm): 122 | class Meta: 123 | model = models.CommandTemplate 124 | fields = "__all__" 125 | widgets = { 126 | "parser": AceWidget( 127 | mode="python", 128 | theme="dracula", 129 | wordwrap=False, 130 | width="100%", 131 | showprintmargin=True, 132 | showinvisibles=False, 133 | usesofttabs=True, 134 | fontsize="14px", 135 | toolbar=False, 136 | showgutter=True, 137 | behaviours=True, 138 | ), 139 | } 140 | 141 | 142 | @admin.register(models.CommandTemplate) 143 | class CommandTemplateAdmin(ModelAdmin, ImportExportModelAdmin): 144 | export_form_class = ExportForm 145 | import_form_class = ImportForm 146 | resource_class = CommandTemplateResource 147 | list_display = ["id", "name"] 148 | form = CommandTemplateForm 149 | 150 | def get_queryset(self, request): 151 | return models.CommandTemplate.objects.order_by("-created_at") 152 | -------------------------------------------------------------------------------- /monitorizer/inventory/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class InventoryConfig(AppConfig): 5 | default_auto_field = "django.db.models.BigAutoField" 6 | name = "monitorizer.inventory" 7 | -------------------------------------------------------------------------------- /monitorizer/inventory/callback.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.conf import settings 4 | from django.db.models import Count, Q 5 | 6 | from monitorizer.inventory import models 7 | 8 | DEFAULT_CHART_OPTIONS = { 9 | "barPercentage": 1, 10 | "base": 0, 11 | "grouped": True, 12 | "maxBarThickness": 6, 13 | "responsive": True, 14 | "maintainAspectRatio": False, 15 | "datasets": { 16 | "bar": {"borderRadius": 12, "border": {"width": 0}, "borderSkipped": "middle"}, 17 | }, 18 | "plugins": { 19 | "legend": { 20 | "align": "end", 21 | "display": True, 22 | "position": "top", 23 | "labels": { 24 | "boxHeight": 5, 25 | "boxWidth": 5, 26 | "color": "#9ca3af", 27 | "pointStyle": "circle", 28 | "usePointStyle": True, 29 | }, 30 | }, 31 | "tooltip": {"enabled": True}, 32 | }, 33 | } 34 | 35 | 36 | def environment_callback(request): 37 | if not settings.DEBUG: 38 | return 39 | return ["Development", "warning"] 40 | 41 | 42 | def dashboard_callback(request, context): 43 | scans_per_day = { 44 | group["created_day"]: group 45 | for group in models.DomainScan.objects.extra( 46 | {"created_day": "date(created_at)"} 47 | ) 48 | .values("created_day") 49 | .annotate( 50 | success=Count( 51 | "status", filter=Q(status=models.DomainScan.ScanStatus.SUCCESS) 52 | ), 53 | error=Count("status", filter=Q(status=models.DomainScan.ScanStatus.ERROR)), 54 | running=Count( 55 | "status", filter=Q(status=models.DomainScan.ScanStatus.RUNNING) 56 | ), 57 | pending=Count( 58 | "status", filter=Q(status=models.DomainScan.ScanStatus.PENDING) 59 | ), 60 | ) 61 | .order_by("created_day")[:30] 62 | } 63 | top_seeds = { 64 | group["seeds__value"]: group["count"] 65 | for group in models.DiscoveredDomain.objects.values("seeds__value") 66 | .annotate(count=Count("seeds__value")) 67 | .order_by("-count")[:5] 68 | } 69 | context.update( 70 | { 71 | "DEFAULT_CHART_OPTIONS": json.dumps(DEFAULT_CHART_OPTIONS), 72 | "discovered_breakdown": json.dumps( 73 | { 74 | "labels": list(top_seeds.keys()), 75 | "datasets": [{"data": list(top_seeds.values())}], 76 | } 77 | ), 78 | "activity_per_day_chart": json.dumps( 79 | { 80 | "labels": [str(v) for v in scans_per_day.keys()], 81 | "datasets": [ 82 | { 83 | "label": "Pending", 84 | "data": [v["pending"] for v in scans_per_day.values()], 85 | "backgroundColor": "#f6ad55", 86 | }, 87 | { 88 | "label": "Running", 89 | "data": [v["running"] for v in scans_per_day.values()], 90 | "backgroundColor": "#63b3ed", 91 | }, 92 | { 93 | "label": "Success", 94 | "data": [v["success"] for v in scans_per_day.values()], 95 | "backgroundColor": "#48bb78", 96 | }, 97 | { 98 | "label": "Error", 99 | "data": [v["error"] for v in scans_per_day.values()], 100 | "backgroundColor": "#f56565", 101 | }, 102 | ], 103 | } 104 | ), 105 | "cards": [ 106 | { 107 | "title": "Seeds", 108 | "value": models.SeedDomain.objects.count(), 109 | }, 110 | { 111 | "title": "Overall Scans", 112 | "value": models.DomainScan.objects.count(), 113 | }, 114 | { 115 | "title": "Discovered", 116 | "value": models.DiscoveredDomain.objects.count(), 117 | }, 118 | ], 119 | } 120 | ) 121 | return context 122 | -------------------------------------------------------------------------------- /monitorizer/inventory/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.0.4 on 2024-04-11 05:19 2 | 3 | import uuid 4 | 5 | import django.db.models.deletion 6 | from django.db import migrations, models 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | initial = True 12 | 13 | dependencies = [ 14 | ("django_celery_beat", "0018_improve_crontab_helptext"), 15 | ] 16 | 17 | operations = [ 18 | migrations.CreateModel( 19 | name="CommandTemplate", 20 | fields=[ 21 | ( 22 | "id", 23 | models.UUIDField( 24 | default=uuid.uuid4, 25 | editable=False, 26 | primary_key=True, 27 | serialize=False, 28 | ), 29 | ), 30 | ("created_at", models.DateTimeField(auto_now_add=True)), 31 | ("updated_at", models.DateTimeField(auto_now=True)), 32 | ("name", models.CharField()), 33 | ("cmd", models.CharField()), 34 | ("parser", models.TextField()), 35 | ], 36 | options={ 37 | "abstract": False, 38 | }, 39 | ), 40 | migrations.CreateModel( 41 | name="SeedDomain", 42 | fields=[ 43 | ( 44 | "id", 45 | models.UUIDField( 46 | default=uuid.uuid4, 47 | editable=False, 48 | primary_key=True, 49 | serialize=False, 50 | ), 51 | ), 52 | ("created_at", models.DateTimeField(auto_now_add=True)), 53 | ("updated_at", models.DateTimeField(auto_now=True)), 54 | ("value", models.CharField(db_index=True, max_length=255, unique=True)), 55 | ( 56 | "enabled", 57 | models.BooleanField( 58 | default=True, 59 | help_text="If set to false. disables all enumeration operations related to this seed including scans and submitters.", 60 | ), 61 | ), 62 | ], 63 | options={ 64 | "abstract": False, 65 | }, 66 | ), 67 | migrations.CreateModel( 68 | name="ScanAutoSubmitter", 69 | fields=[ 70 | ( 71 | "id", 72 | models.UUIDField( 73 | default=uuid.uuid4, 74 | editable=False, 75 | primary_key=True, 76 | serialize=False, 77 | ), 78 | ), 79 | ("created_at", models.DateTimeField(auto_now_add=True)), 80 | ("updated_at", models.DateTimeField(auto_now=True)), 81 | ("enabled", models.BooleanField(default=True)), 82 | ("commands", models.ManyToManyField(to="inventory.commandtemplate")), 83 | ( 84 | "interval", 85 | models.ForeignKey( 86 | on_delete=django.db.models.deletion.CASCADE, 87 | to="django_celery_beat.intervalschedule", 88 | ), 89 | ), 90 | ( 91 | "domain", 92 | models.ForeignKey( 93 | on_delete=django.db.models.deletion.CASCADE, 94 | to="inventory.seeddomain", 95 | ), 96 | ), 97 | ], 98 | options={ 99 | "abstract": False, 100 | }, 101 | ), 102 | migrations.CreateModel( 103 | name="DomainScan", 104 | fields=[ 105 | ( 106 | "id", 107 | models.UUIDField( 108 | default=uuid.uuid4, 109 | editable=False, 110 | primary_key=True, 111 | serialize=False, 112 | ), 113 | ), 114 | ("created_at", models.DateTimeField(auto_now_add=True)), 115 | ("updated_at", models.DateTimeField(auto_now=True)), 116 | ( 117 | "status", 118 | models.CharField( 119 | choices=[ 120 | ("pending", "Pending"), 121 | ("running", "Running"), 122 | ("success", "Success"), 123 | ("error", "Error"), 124 | ], 125 | default="pending", 126 | max_length=32, 127 | ), 128 | ), 129 | ("command_tpl_vars", models.JSONField(blank=True, null=True)), 130 | ("exit_code", models.PositiveIntegerField(blank=True, null=True)), 131 | ("error", models.TextField(blank=True, null=True)), 132 | ("finished_at", models.DateTimeField(blank=True, null=True)), 133 | ( 134 | "command_tpl", 135 | models.ForeignKey( 136 | on_delete=django.db.models.deletion.CASCADE, 137 | to="inventory.commandtemplate", 138 | ), 139 | ), 140 | ( 141 | "descriptor", 142 | models.ForeignKey( 143 | blank=True, 144 | null=True, 145 | on_delete=django.db.models.deletion.CASCADE, 146 | to="inventory.scanautosubmitter", 147 | ), 148 | ), 149 | ( 150 | "domain", 151 | models.ForeignKey( 152 | blank=True, 153 | on_delete=django.db.models.deletion.CASCADE, 154 | to="inventory.seeddomain", 155 | ), 156 | ), 157 | ], 158 | options={ 159 | "abstract": False, 160 | }, 161 | ), 162 | migrations.CreateModel( 163 | name="DiscoveredDomain", 164 | fields=[ 165 | ( 166 | "id", 167 | models.UUIDField( 168 | default=uuid.uuid4, 169 | editable=False, 170 | primary_key=True, 171 | serialize=False, 172 | ), 173 | ), 174 | ("created_at", models.DateTimeField(auto_now_add=True)), 175 | ("updated_at", models.DateTimeField(auto_now=True)), 176 | ("value", models.CharField(db_index=True, max_length=255, unique=True)), 177 | ( 178 | "seeds", 179 | models.ManyToManyField( 180 | related_name="discovered", to="inventory.seeddomain" 181 | ), 182 | ), 183 | ], 184 | options={ 185 | "abstract": False, 186 | }, 187 | ), 188 | ] 189 | -------------------------------------------------------------------------------- /monitorizer/inventory/migrations/0002_alter_domainscan_status.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.0.4 on 2024-04-12 20:28 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("inventory", "0001_initial"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="domainscan", 15 | name="status", 16 | field=models.CharField( 17 | choices=[ 18 | ("pending", "Pending"), 19 | ("running", "Running"), 20 | ("success", "Success"), 21 | ("error", "Error"), 22 | ], 23 | db_index=True, 24 | default="pending", 25 | max_length=32, 26 | ), 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /monitorizer/inventory/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitTheByte/Monitorizer/8cf2a8c60076bdb7bcd4d9a7cefe2615c676f8c5/monitorizer/inventory/migrations/__init__.py -------------------------------------------------------------------------------- /monitorizer/inventory/models.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | 4 | from django.contrib.contenttypes.models import ContentType 5 | from django.db import models 6 | from django.db.models.signals import post_delete 7 | from django.dispatch import receiver 8 | from django_celery_beat.models import IntervalSchedule, PeriodicTask 9 | 10 | from monitorizer.server.models import BaseModel, TimedModel 11 | from monitorizer.utils.engine import tmp_file 12 | 13 | 14 | class SeedDomain(BaseModel): 15 | value = models.CharField(max_length=255, unique=True, db_index=True) 16 | enabled = models.BooleanField( 17 | default=True, 18 | help_text="If set to false." 19 | " disables all enumeration operations related to this seed" 20 | " including scans and submitters.", 21 | ) 22 | 23 | def __str__(self): 24 | return self.value 25 | 26 | 27 | class DiscoveredDomain(BaseModel): 28 | value = models.CharField(max_length=255, unique=True, db_index=True) 29 | seeds = models.ManyToManyField(SeedDomain, blank=False, related_name="discovered") 30 | 31 | def __str__(self): 32 | return self.value 33 | 34 | 35 | class CommandTemplate(BaseModel): 36 | name = models.CharField(null=False, blank=False) 37 | cmd = models.CharField(null=False, blank=False) 38 | parser = models.TextField() 39 | 40 | def __str__(self): 41 | return self.name 42 | 43 | 44 | class ScanAutoSubmitter(BaseModel): 45 | enabled = models.BooleanField(default=True) 46 | interval = models.ForeignKey(IntervalSchedule, on_delete=models.CASCADE) 47 | commands = models.ManyToManyField(CommandTemplate, blank=False) 48 | domain = models.ForeignKey(SeedDomain, on_delete=models.CASCADE) 49 | 50 | @staticmethod 51 | @receiver(post_delete, sender="inventory.ScanAutoSubmitter") 52 | def on_delete(sender, instance, using, **kwargs): 53 | PeriodicTask.objects.filter(name=f"descriptor_{instance.pk}_schedule").delete() 54 | 55 | def save(self, *args, **kwargs): 56 | result = super().save(*args, **kwargs) 57 | PeriodicTask.objects.update_or_create( 58 | name=f"descriptor_{self.pk}_schedule", 59 | task="monitorizer.inventory.tasks.execute_scan_descriptor", 60 | defaults={ 61 | "kwargs": json.dumps({"descriptor_pk": str(self.pk)}), 62 | "interval": self.interval, 63 | "enabled": self.enabled, 64 | }, 65 | ) 66 | return result 67 | 68 | 69 | class DomainScan(BaseModel): 70 | class ScanStatus(models.TextChoices): 71 | PENDING = "pending", "Pending" 72 | RUNNING = "running", "Running" 73 | SUCCESS = "success", "Success" 74 | ERROR = "error", "Error" 75 | 76 | descriptor = models.ForeignKey( 77 | ScanAutoSubmitter, on_delete=models.CASCADE, null=True, blank=True 78 | ) 79 | 80 | domain = models.ForeignKey(SeedDomain, on_delete=models.CASCADE, blank=True) 81 | status = models.CharField( 82 | choices=ScanStatus.choices, 83 | max_length=32, 84 | default=ScanStatus.PENDING, 85 | db_index=True, 86 | ) 87 | command_tpl = models.ForeignKey(CommandTemplate, on_delete=models.CASCADE) 88 | command_tpl_vars = models.JSONField(null=True, blank=True) 89 | exit_code = models.PositiveIntegerField(null=True, blank=True) 90 | error = models.TextField(null=True, blank=True) 91 | finished_at = models.DateTimeField(null=True, blank=True) 92 | 93 | built_in_template_vars = {"domain", "domain:file", "output", "output:file"} 94 | 95 | @property 96 | def command(self): 97 | command = self.command_tpl.cmd 98 | for var, value in (self.command_tpl_vars or {}).items(): 99 | command = command.replace("{%s}" % var, value) 100 | return command 101 | 102 | def save(self, *args, **kwargs): 103 | maybe_vars = re.findall(r"{(.*?)}", self.command_tpl.cmd) 104 | self.command_tpl_vars = self.command_tpl_vars or {} 105 | if self.descriptor: 106 | self.domain = self.descriptor.domain 107 | 108 | for var in maybe_vars: 109 | if ( 110 | var not in self.built_in_template_vars 111 | and var not in self.command_tpl_vars 112 | ): 113 | continue 114 | if var == "domain": 115 | self.command_tpl_vars[var] = self.domain.value 116 | if var == "domain:file": 117 | path = tmp_file() 118 | open(path, "w").write(self.domain.value) 119 | self.command_tpl_vars[var] = path 120 | if var == "output:file": 121 | self.command_tpl_vars[var] = tmp_file() 122 | 123 | return super().save(*args, **kwargs) 124 | -------------------------------------------------------------------------------- /monitorizer/inventory/tasks.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | from celery.signals import worker_ready 4 | from django.db import transaction 5 | from django.utils import timezone 6 | from django_celery_beat.models import IntervalSchedule, PeriodicTask 7 | 8 | from monitorizer.inventory import models 9 | from monitorizer.server import celery_app 10 | 11 | 12 | @worker_ready.connect 13 | def at_start(sender, **k): 14 | schedule, _ = IntervalSchedule.objects.get_or_create( 15 | every=10, period=IntervalSchedule.SECONDS 16 | ) 17 | for task in [pick_and_start_scan]: 18 | PeriodicTask.objects.get_or_create( 19 | name=task.name, 20 | task=task.name, 21 | defaults={"interval": schedule}, 22 | ) 23 | 24 | 25 | @celery_app.task 26 | def execute_scan_descriptor(descriptor_pk): 27 | descriptor = models.ScanAutoSubmitter.objects.get(pk=descriptor_pk) 28 | if not descriptor.domain.enabled: 29 | return 30 | 31 | if ( 32 | models.DomainScan.objects.filter( 33 | status=models.DomainScan.ScanStatus.PENDING 34 | ).count() 35 | > 1500 36 | ): 37 | return 38 | 39 | for command in descriptor.commands.all(): 40 | models.DomainScan.objects.create( 41 | descriptor=descriptor, 42 | command_tpl=command, 43 | ) 44 | 45 | 46 | @celery_app.task 47 | def pick_and_start_scan(): 48 | with transaction.atomic(): 49 | scan: models.DomainScan = ( 50 | models.DomainScan.objects.select_for_update(skip_locked=True) 51 | .filter(status=models.DomainScan.ScanStatus.PENDING, domain__enabled=True) 52 | .order_by("?") 53 | .first() 54 | ) 55 | if not scan: 56 | return 57 | scan.status = models.DomainScan.ScanStatus.RUNNING 58 | scan.save() 59 | 60 | command = subprocess.run( 61 | scan.command, shell=True, capture_output=True, encoding="utf8" 62 | ) 63 | 64 | if command.returncode == 0: 65 | scan.status = models.DomainScan.ScanStatus.SUCCESS 66 | _globals = {"__scan__": {"vars": scan.command_tpl_vars}, "__result__": {}} 67 | exec(scan.command_tpl.parser, _globals) 68 | for v in _globals.get("__result__", []): 69 | domain, _ = models.DiscoveredDomain.objects.get_or_create(value=v.strip()) 70 | domain.seeds.add(scan.domain) 71 | else: 72 | scan.status = models.DomainScan.ScanStatus.ERROR 73 | scan.error = command.stderr 74 | 75 | scan.exit_code = command.returncode 76 | scan.finished_at = timezone.now() 77 | scan.save() 78 | -------------------------------------------------------------------------------- /monitorizer/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | """Run administrative tasks.""" 9 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "monitorizer.server.settings") 10 | try: 11 | from django.core.management import execute_from_command_line 12 | except ImportError as exc: 13 | raise ImportError( 14 | "Couldn't import Django. Are you sure it's installed and " 15 | "available on your PYTHONPATH environment variable? Did you " 16 | "forget to activate a virtual environment?" 17 | ) from exc 18 | execute_from_command_line(sys.argv) 19 | 20 | 21 | if __name__ == "__main__": 22 | main() 23 | -------------------------------------------------------------------------------- /monitorizer/report/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitTheByte/Monitorizer/8cf2a8c60076bdb7bcd4d9a7cefe2615c676f8c5/monitorizer/report/__init__.py -------------------------------------------------------------------------------- /monitorizer/report/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from unfold.admin import ModelAdmin 3 | 4 | from monitorizer.report.models import * 5 | 6 | 7 | @admin.register(TelegramReport) 8 | class TelegramReportAdmin(ModelAdmin): 9 | list_display = ["id", "chat_id", "api_id", "interval", "last_executed"] 10 | readonly_fields = ["last_executed"] 11 | list_filter = ["enabled"] 12 | 13 | 14 | @admin.register(WebHookReport) 15 | class WebHookReportAdmin(ModelAdmin): 16 | list_display = ["id", "url", "interval", "last_executed"] 17 | list_filter = ["enabled"] 18 | readonly_fields = ["last_executed"] 19 | -------------------------------------------------------------------------------- /monitorizer/report/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class ReportConfig(AppConfig): 5 | default_auto_field = "django.db.models.BigAutoField" 6 | name = "monitorizer.report" 7 | -------------------------------------------------------------------------------- /monitorizer/report/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.0.4 on 2024-04-11 05:19 2 | 3 | import datetime 4 | import uuid 5 | 6 | import django.db.models.deletion 7 | from django.db import migrations, models 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | initial = True 13 | 14 | dependencies = [ 15 | ("django_celery_beat", "0018_improve_crontab_helptext"), 16 | ] 17 | 18 | operations = [ 19 | migrations.CreateModel( 20 | name="TelegramReport", 21 | fields=[ 22 | ( 23 | "id", 24 | models.UUIDField( 25 | default=uuid.uuid4, 26 | editable=False, 27 | primary_key=True, 28 | serialize=False, 29 | ), 30 | ), 31 | ("created_at", models.DateTimeField(auto_now_add=True)), 32 | ("updated_at", models.DateTimeField(auto_now=True)), 33 | ("enabled", models.BooleanField(default=True)), 34 | ( 35 | "filter_type", 36 | models.CharField( 37 | choices=[ 38 | ("value__startswith", "Starts with"), 39 | ("value__endswith", "Ends with"), 40 | ("value__contains", "Contains"), 41 | ], 42 | max_length=32, 43 | ), 44 | ), 45 | ("filter_value", models.CharField()), 46 | ( 47 | "last_executed", 48 | models.DateTimeField(default=datetime.datetime(1970, 1, 1, 0, 0)), 49 | ), 50 | ("api_id", models.BigIntegerField()), 51 | ("api_hash", models.CharField(max_length=1024)), 52 | ("chat_id", models.BigIntegerField()), 53 | ( 54 | "interval", 55 | models.ForeignKey( 56 | null=True, 57 | on_delete=django.db.models.deletion.SET_NULL, 58 | to="django_celery_beat.intervalschedule", 59 | ), 60 | ), 61 | ], 62 | options={ 63 | "abstract": False, 64 | }, 65 | ), 66 | migrations.CreateModel( 67 | name="WebHookReport", 68 | fields=[ 69 | ( 70 | "id", 71 | models.UUIDField( 72 | default=uuid.uuid4, 73 | editable=False, 74 | primary_key=True, 75 | serialize=False, 76 | ), 77 | ), 78 | ("created_at", models.DateTimeField(auto_now_add=True)), 79 | ("updated_at", models.DateTimeField(auto_now=True)), 80 | ("enabled", models.BooleanField(default=True)), 81 | ( 82 | "filter_type", 83 | models.CharField( 84 | choices=[ 85 | ("value__startswith", "Starts with"), 86 | ("value__endswith", "Ends with"), 87 | ("value__contains", "Contains"), 88 | ], 89 | max_length=32, 90 | ), 91 | ), 92 | ("filter_value", models.CharField()), 93 | ( 94 | "last_executed", 95 | models.DateTimeField(default=datetime.datetime(1970, 1, 1, 0, 0)), 96 | ), 97 | ("url", models.URLField()), 98 | ("message_param", models.CharField(default="content")), 99 | ("file_param", models.CharField(default="file")), 100 | ( 101 | "interval", 102 | models.ForeignKey( 103 | null=True, 104 | on_delete=django.db.models.deletion.SET_NULL, 105 | to="django_celery_beat.intervalschedule", 106 | ), 107 | ), 108 | ], 109 | options={ 110 | "abstract": False, 111 | }, 112 | ), 113 | ] 114 | -------------------------------------------------------------------------------- /monitorizer/report/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BitTheByte/Monitorizer/8cf2a8c60076bdb7bcd4d9a7cefe2615c676f8c5/monitorizer/report/migrations/__init__.py -------------------------------------------------------------------------------- /monitorizer/report/models.py: -------------------------------------------------------------------------------- 1 | import json 2 | from datetime import datetime 3 | 4 | from django.contrib.contenttypes.models import ContentType 5 | from django.db import models 6 | from django.db.models.signals import post_delete 7 | from django.dispatch import receiver 8 | from django_celery_beat.models import IntervalSchedule, PeriodicTask 9 | 10 | from monitorizer.server.models import BaseModel 11 | 12 | 13 | class Report(BaseModel): 14 | class FilterTypes(models.TextChoices): 15 | STARTS_WITH = "value__startswith", "Starts with" 16 | ENDS_WITH = "value__endswith", "Ends with" 17 | CONTAINS = "value__contains", "Contains" 18 | 19 | enabled = models.BooleanField(default=True) 20 | interval = models.ForeignKey(IntervalSchedule, on_delete=models.SET_NULL, null=True) 21 | filter_type = models.CharField(choices=FilterTypes.choices, max_length=32) 22 | filter_value = models.CharField() 23 | last_executed = models.DateTimeField(default=datetime(1970, 1, 1)) 24 | 25 | class Meta: 26 | abstract = True 27 | 28 | @property 29 | def task_name(self): 30 | return f"report_{self.__class__.__name__}_{self.pk}_schedule" 31 | 32 | def save(self, *args, **kwargs): 33 | result = super().save(*args, **kwargs) 34 | PeriodicTask.objects.update_or_create( 35 | name=self.task_name, 36 | task="monitorizer.report.tasks.send_report_result", 37 | defaults={ 38 | "kwargs": json.dumps( 39 | { 40 | "report_channel_type": self.__class__.__name__, 41 | "channel_pk": str(self.pk), 42 | } 43 | ), 44 | "interval": self.interval, 45 | "enabled": self.enabled, 46 | }, 47 | ) 48 | return result 49 | 50 | 51 | class TelegramReport(Report): 52 | api_id = models.BigIntegerField() 53 | api_hash = models.CharField(max_length=1024) 54 | chat_id = models.BigIntegerField() 55 | 56 | 57 | class WebHookReport(Report): 58 | url = models.URLField() 59 | message_param = models.CharField(default="content") 60 | file_param = models.CharField(default="file") 61 | 62 | 63 | @receiver(post_delete, sender=TelegramReport) 64 | @receiver(post_delete, sender=WebHookReport) 65 | def on_delete(sender, instance, **kwargs): 66 | PeriodicTask.objects.filter(name=instance.task_name).delete() 67 | -------------------------------------------------------------------------------- /monitorizer/report/tasks.py: -------------------------------------------------------------------------------- 1 | import os 2 | import uuid 3 | 4 | from django.utils import timezone 5 | 6 | from monitorizer.inventory import models as inventory_models 7 | from monitorizer.report import models 8 | from monitorizer.server import celery_app 9 | 10 | MESSAGE_TEMPLATE = """ 11 | Monitorizer has discovered new %i domain(s) since %s 12 | Filter Type: %s 13 | Filter Value: %s 14 | """ 15 | 16 | 17 | @celery_app.task 18 | def send_report_result(report_channel_type, channel_pk): 19 | report_path = f"/tmp/{uuid.uuid4()}" 20 | report = getattr(models, report_channel_type).objects.get(pk=channel_pk) 21 | 22 | if not report or not report.enabled: 23 | return 24 | 25 | domains = [ 26 | d[0] 27 | for d in inventory_models.DiscoveredDomain.objects.filter( 28 | **{report.filter_type: report.filter_value}, 29 | created_at__gte=report.last_executed, 30 | ).values_list("value") 31 | ] 32 | message = MESSAGE_TEMPLATE % ( 33 | len(domains), 34 | str(report.last_executed), 35 | report.filter_type.get_display(), 36 | report.filter_value, 37 | ) 38 | 39 | report.last_executed = timezone.now() 40 | if not domains: 41 | report.save() 42 | return 43 | 44 | open(report_path, "w").write("\n".join(domains)) 45 | 46 | if isinstance(report, models.TelegramReport): 47 | from telebot import TeleBot 48 | 49 | bot = TeleBot(token="%s:%s" % (report.api_id, report.api_hash)) 50 | bot.send_document( 51 | chat_id=report.chat_id, caption=message, document=open(report_path) 52 | ) 53 | 54 | if isinstance(report, models.WebHookReport): 55 | import requests 56 | 57 | response = requests.post( 58 | report.url, 59 | verify=False, 60 | timeout=60, 61 | data={report.message_param: message}, 62 | files={report.file_param: open(report_path)}, 63 | ) 64 | response.raise_for_status() 65 | 66 | if os.path.exists(report_path): 67 | os.unlink(report_path) 68 | report.save() 69 | -------------------------------------------------------------------------------- /monitorizer/server/__init__.py: -------------------------------------------------------------------------------- 1 | from monitorizer.server.celery import app as celery_app 2 | 3 | __all__ = ("celery_app",) 4 | -------------------------------------------------------------------------------- /monitorizer/server/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.contrib.auth import admin as auth_admin 3 | from django.contrib.auth import models as auth_models 4 | from django.contrib.auth.admin import UserAdmin as BaseUserAdmin 5 | from django_celery_beat import admin as celery_beat_admin 6 | from django_celery_beat import models as celery_beat_models 7 | from unfold.admin import ModelAdmin 8 | from unfold.forms import AdminPasswordChangeForm, UserChangeForm, UserCreationForm 9 | 10 | 11 | class UserAdmin(BaseUserAdmin, ModelAdmin): 12 | form = UserChangeForm 13 | add_form = UserCreationForm 14 | change_password_form = AdminPasswordChangeForm 15 | 16 | 17 | REGISTER = { 18 | auth_models.User: UserAdmin, 19 | auth_models.Group: auth_admin.GroupAdmin, 20 | celery_beat_models.PeriodicTask: celery_beat_admin.PeriodicTaskAdmin, 21 | celery_beat_models.ClockedSchedule: celery_beat_admin.ClockedScheduleAdmin, 22 | celery_beat_models.CrontabSchedule: celery_beat_admin.CrontabScheduleAdmin, 23 | celery_beat_models.SolarSchedule: None, 24 | celery_beat_models.IntervalSchedule: None, 25 | } 26 | 27 | for model, model_admin in REGISTER.items(): 28 | admin.site.unregister(model) 29 | 30 | if model_admin: 31 | 32 | class UnfoldAdmin(model_admin, ModelAdmin): 33 | pass 34 | 35 | admin.site.register(model, UnfoldAdmin) 36 | else: 37 | admin.site.register(model, ModelAdmin) 38 | -------------------------------------------------------------------------------- /monitorizer/server/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class ServerConfig(AppConfig): 5 | default_auto_field = "django.db.models.BigAutoField" 6 | name = "monitorizer.server" 7 | -------------------------------------------------------------------------------- /monitorizer/server/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for asmcore project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/5.0/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "monitorizer.server.settings") 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /monitorizer/server/celery.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from celery import Celery 4 | from kombu import Queue 5 | 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "monitorizer.server.settings") 7 | 8 | app = Celery("asmcore") 9 | 10 | # Using a string here means the worker doesn't have to serialize 11 | # the configuration object to child processes. 12 | # - namespace='CELERY' means all celery-related configuration keys 13 | # should have a `CELERY_` prefix. 14 | app.config_from_object("django.conf:settings", namespace="CELERY") 15 | 16 | # Load task modules from all registered Django apps. 17 | app.autodiscover_tasks() 18 | app.conf.task_routes = { 19 | "monitorizer.inventory.tasks.*": {"queue": "default"}, 20 | "monitorizer.report.tasks.*": {"queue": "reports"}, 21 | } 22 | app.conf.task_queues = ( 23 | Queue("default", routing_key="task.#"), 24 | Queue("reports", routing_key="reports.#"), 25 | ) 26 | -------------------------------------------------------------------------------- /monitorizer/server/management/commands/openresty.conf.jinja: -------------------------------------------------------------------------------- 1 | upstream 'asmcore' { 2 | server {{ gateway }}; 3 | keepalive 300; 4 | } 5 | 6 | server { 7 | listen 8000 default_server; 8 | server_name _; 9 | 10 | location /health { 11 | proxy_set_header X-Original-Host $host; 12 | proxy_pass http://asmcore/health; 13 | } 14 | 15 | location / { 16 | proxy_ssl_server_name on; 17 | proxy_http_version 1.1; 18 | proxy_set_header Host $http_host; 19 | proxy_set_header X-Real-IP $remote_addr; 20 | proxy_set_header X-Scheme $scheme; 21 | proxy_set_header X-Original-URI $request_uri; 22 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 23 | proxy_set_header X-Forwarded-Proto $scheme; 24 | proxy_set_header Upgrade $http_upgrade; 25 | proxy_set_header Connection $connection_upgrade; 26 | proxy_set_header Origin ""; 27 | proxy_redirect off; 28 | proxy_request_buffering on; 29 | proxy_buffering off; 30 | proxy_buffer_size 4k; 31 | proxy_buffers 4 4k; 32 | proxy_max_temp_file_size 1024m; 33 | proxy_read_timeout 120; 34 | proxy_connect_timeout 120; 35 | proxy_send_timeout 120; 36 | send_timeout 120; 37 | proxy_pass http://asmcore/; 38 | } 39 | 40 | location {{ static_path }} { 41 | alias {{ static_root }}; 42 | } 43 | } -------------------------------------------------------------------------------- /monitorizer/server/management/commands/openresty_template.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from django.conf import settings 4 | from django.core.management.base import BaseCommand 5 | from jinja2 import Template 6 | 7 | 8 | class Command(BaseCommand): 9 | help = "Generate openresty config template" 10 | 11 | def add_arguments(self, parser): 12 | parser.add_argument("--gateway", required=True, type=str) 13 | 14 | def handle(self, *args, **options): 15 | template = Template( 16 | (Path(__file__).parent / "openresty.conf.jinja").open("r").read() 17 | ) 18 | return template.render( 19 | gateway=options["gateway"], 20 | static_root=str(settings.STATIC_ROOT).removesuffix("/") + "/", 21 | static_path=settings.STATIC_URL, 22 | ) 23 | -------------------------------------------------------------------------------- /monitorizer/server/models.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from django.db import models 4 | 5 | 6 | class UUIDPrimaryKey(models.Model): 7 | id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) 8 | 9 | class Meta: 10 | abstract = True 11 | 12 | 13 | class TimedModel(models.Model): 14 | created_at = models.DateTimeField(auto_now_add=True) 15 | updated_at = models.DateTimeField(auto_now=True) 16 | 17 | class Meta: 18 | abstract = True 19 | ordering = ["-created_at"] 20 | 21 | 22 | class BaseModel(TimedModel, UUIDPrimaryKey): 23 | class Meta: 24 | abstract = True 25 | -------------------------------------------------------------------------------- /monitorizer/server/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for asmcore project. 3 | 4 | Generated by 'django-admin startproject' using Django 5.0.3. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/5.0/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/5.0/ref/settings/ 11 | """ 12 | 13 | import os 14 | from pathlib import Path 15 | 16 | from django.templatetags.static import static 17 | from django.urls import reverse_lazy 18 | 19 | UNFOLD = { 20 | "SITE_TITLE": "Monitorizer", 21 | "SITE_HEADER": "Monitorizer", 22 | "DASHBOARD_CALLBACK": "monitorizer.inventory.callback.dashboard_callback", 23 | "ENVIRONMENT": "monitorizer.inventory.callback.environment_callback", 24 | "LOGIN": { 25 | "image": lambda request: "https://c0.wallpaperflare.com/preview/1/703/828/fighter-jet-airshow.jpg", 26 | }, 27 | "TABS": [ 28 | { 29 | "models": ["inventory.seeddomain", "inventory.discovereddomain"], 30 | "items": [ 31 | { 32 | "title": "Seed", 33 | "link": reverse_lazy("admin:inventory_seeddomain_changelist"), 34 | }, 35 | { 36 | "title": "Discovered", 37 | "link": reverse_lazy("admin:inventory_discovereddomain_changelist"), 38 | }, 39 | ], 40 | }, 41 | { 42 | "models": [ 43 | "django_celery_beat.periodictask", 44 | "django_celery_beat.crontabschedule", 45 | "django_celery_beat.intervalschedule", 46 | "django_celery_beat.solarschedule", 47 | "django_celery_beat.clockedschedule", 48 | ], 49 | "items": [ 50 | { 51 | "title": "Periodic tasks", 52 | "icon": "task", 53 | "link": reverse_lazy( 54 | "admin:django_celery_beat_periodictask_changelist" 55 | ), 56 | }, 57 | { 58 | "title": "Crontabs", 59 | "icon": "update", 60 | "link": reverse_lazy( 61 | "admin:django_celery_beat_crontabschedule_changelist" 62 | ), 63 | }, 64 | { 65 | "title": "Intervals", 66 | "icon": "arrow_range", 67 | "link": reverse_lazy( 68 | "admin:django_celery_beat_intervalschedule_changelist" 69 | ), 70 | }, 71 | { 72 | "title": "Solar events", 73 | "icon": "event", 74 | "link": reverse_lazy( 75 | "admin:django_celery_beat_solarschedule_changelist" 76 | ), 77 | }, 78 | { 79 | "title": "Clocked", 80 | "icon": "hourglass_bottom", 81 | "link": reverse_lazy( 82 | "admin:django_celery_beat_clockedschedule_changelist" 83 | ), 84 | }, 85 | ], 86 | }, 87 | ], 88 | "COLORS": { 89 | "primary": { 90 | # seed: #1f85e5 91 | "50": "#eff9ff", 92 | "100": "#dcf1fd", 93 | "200": "#c1e7fc", 94 | "300": "#96d9fa", 95 | "400": "#64c2f6", 96 | "500": "#3fa7f2", 97 | "600": "#1f85e5", 98 | "700": "#2174d4", 99 | "800": "#215eac", 100 | "900": "#215087", 101 | "950": "#183253", 102 | }, 103 | }, 104 | "SIDEBAR": { 105 | "show_search": True, # Search in applications and models names 106 | "show_all_applications": True, # Dropdown with all applications and models 107 | "navigation": [ 108 | { 109 | "title": "Navigation", 110 | "separator": True, 111 | "items": [ 112 | { 113 | "title": "Dashboard", 114 | "icon": "dashboard", 115 | "link": reverse_lazy("admin:index"), 116 | }, 117 | { 118 | "title": "Domains", 119 | "icon": "public", 120 | "link": reverse_lazy("admin:inventory_seeddomain_changelist"), 121 | }, 122 | { 123 | "title": "Scans", 124 | "icon": "radar", 125 | "link": reverse_lazy("admin:inventory_domainscan_changelist"), 126 | }, 127 | { 128 | "title": "Commands", 129 | "icon": "handyman", 130 | "link": reverse_lazy( 131 | "admin:inventory_commandtemplate_changelist" 132 | ), 133 | }, 134 | ], 135 | }, 136 | { 137 | "title": "Report", 138 | "separator": True, 139 | "items": [ 140 | { 141 | "title": "Telegram", 142 | "link": reverse_lazy("admin:report_telegramreport_changelist"), 143 | }, 144 | { 145 | "title": "Webhook", 146 | "link": reverse_lazy("admin:report_webhookreport_changelist"), 147 | }, 148 | ], 149 | }, 150 | { 151 | "title": "Settings", 152 | "separator": True, 153 | "items": [ 154 | { 155 | "title": "Users", 156 | "icon": "person", 157 | "link": reverse_lazy("admin:auth_user_changelist"), 158 | }, 159 | { 160 | "title": "Groups", 161 | "icon": "group", 162 | "link": reverse_lazy("admin:auth_group_changelist"), 163 | }, 164 | { 165 | "title": "Tasks", 166 | "icon": "task_alt", 167 | "link": reverse_lazy( 168 | "admin:django_celery_beat_periodictask_changelist" 169 | ), 170 | }, 171 | ], 172 | }, 173 | ], 174 | }, 175 | } 176 | # Build paths inside the project like this: BASE_DIR / 'subdir'. 177 | BASE_DIR = Path(__file__).resolve().parent.parent 178 | 179 | 180 | # Quick-start development settings - unsuitable for production 181 | # See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ 182 | 183 | # SECURITY WARNING: keep the secret key used in production secret! 184 | SECRET_KEY = "django-insecure-!of9a9=_^at8&*oa#$fy0%)d)k&91qchx7-1*4ddz8(ugs12_d" 185 | 186 | # SECURITY WARNING: don't run with debug turned on in production! 187 | DEBUG = bool(os.environ.get("DEBUG", False)) 188 | 189 | ALLOWED_HOSTS = ["*"] 190 | # APPEND_SLASH = True 191 | 192 | # Celery Configuration Options 193 | CELERY_TIMEZONE = "UTC" 194 | CELERY_ENABLE_UTC = True 195 | CELERY_TASK_TRACK_STARTED = True 196 | CELERY_TASK_TIME_LIMIT = 30 * 60 197 | CELERY_BROKER_URL = os.environ.get( 198 | "CELERY_BROKER_URL", "amqp://guest:guest@127.0.0.1:5672" 199 | ) 200 | CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True 201 | CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" 202 | CELERY_IGNORE_RESULT = True 203 | CACHES = { 204 | "default": { 205 | "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", 206 | "LOCATION": "/var/tmp/django_cache", 207 | } 208 | } 209 | 210 | # Application definition 211 | INSTALLED_APPS = [ 212 | "unfold", 213 | "unfold.contrib.filters", 214 | "unfold.contrib.forms", 215 | "unfold.contrib.import_export", 216 | "debug_toolbar", 217 | "django_ace", 218 | "import_export", 219 | "django_celery_beat", 220 | "django.contrib.admin", 221 | "django.contrib.auth", 222 | "django.contrib.contenttypes", 223 | "django.contrib.sessions", 224 | "django.contrib.messages", 225 | "django.contrib.staticfiles", 226 | ] + ["monitorizer.server", "monitorizer.inventory", "monitorizer.report"] 227 | 228 | INTERNAL_IPS = [ 229 | "127.0.0.1", 230 | ] 231 | 232 | MIDDLEWARE = [ 233 | "django.middleware.security.SecurityMiddleware", 234 | "django.contrib.sessions.middleware.SessionMiddleware", 235 | "django.middleware.common.CommonMiddleware", 236 | "django.middleware.csrf.CsrfViewMiddleware", 237 | "debug_toolbar.middleware.DebugToolbarMiddleware", 238 | "django.contrib.auth.middleware.AuthenticationMiddleware", 239 | "django.contrib.messages.middleware.MessageMiddleware", 240 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 241 | ] 242 | 243 | ROOT_URLCONF = "monitorizer.server.urls" 244 | DATA_UPLOAD_MAX_NUMBER_FIELDS = 10_000 245 | 246 | TEMPLATES = [ 247 | { 248 | "BACKEND": "django.template.backends.django.DjangoTemplates", 249 | "DIRS": [BASE_DIR / "templates"], 250 | "APP_DIRS": True, 251 | "OPTIONS": { 252 | "builtins": [ 253 | "unfold.templatetags.unfold", 254 | "unfold.templatetags.unfold_list", 255 | ], 256 | "context_processors": [ 257 | "django.template.context_processors.debug", 258 | "django.template.context_processors.request", 259 | "django.contrib.auth.context_processors.auth", 260 | "django.contrib.messages.context_processors.messages", 261 | ], 262 | }, 263 | }, 264 | ] 265 | 266 | WSGI_APPLICATION = "monitorizer.server.wsgi.application" 267 | 268 | 269 | # Database 270 | # https://docs.djangoproject.com/en/5.0/ref/settings/#databases 271 | 272 | DATABASES = { 273 | "default": { 274 | "ENGINE": "django.db.backends.postgresql", 275 | "NAME": os.environ.get("POSTGRES_DB", "postgres"), 276 | "USER": os.environ.get("POSTGRES_USER", "postgres"), 277 | "PASSWORD": os.environ.get("POSTGRES_PASSWORD", "postgres"), 278 | "HOST": os.environ.get("POSTGRES_HOST", "127.0.0.1"), 279 | "PORT": int(os.environ.get("POSTGRES_PORT", 5432)), 280 | "AUTOCOMMIT": True, 281 | "DISABLE_SERVER_SIDE_CURSORS": True, 282 | } 283 | } 284 | 285 | 286 | # Password validation 287 | # https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators 288 | 289 | AUTH_PASSWORD_VALIDATORS = [ 290 | { 291 | "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", 292 | }, 293 | { 294 | "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", 295 | }, 296 | { 297 | "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", 298 | }, 299 | { 300 | "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", 301 | }, 302 | ] 303 | 304 | 305 | # Internationalization 306 | # https://docs.djangoproject.com/en/5.0/topics/i18n/ 307 | 308 | LANGUAGE_CODE = "en-us" 309 | 310 | TIME_ZONE = "UTC" 311 | 312 | USE_I18N = True 313 | 314 | USE_TZ = True 315 | 316 | 317 | # Static files (CSS, JavaScript, Images) 318 | # https://docs.djangoproject.com/en/5.0/howto/static-files/ 319 | 320 | STATIC_URL = "static/" 321 | 322 | # Default primary key field type 323 | # https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field 324 | 325 | DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" 326 | STATIC_ROOT = BASE_DIR / "staticfiles" 327 | -------------------------------------------------------------------------------- /monitorizer/server/urls.py: -------------------------------------------------------------------------------- 1 | """ 2 | URL configuration for asmcore project. 3 | 4 | The `urlpatterns` list routes URLs to views. For more information please see: 5 | https://docs.djangoproject.com/en/5.0/topics/http/urls/ 6 | Examples: 7 | Function views 8 | 1. Add an import: from my_app import views 9 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 10 | Class-based views 11 | 1. Add an import: from other_app.views import Home 12 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 13 | Including another URLconf 14 | 1. Import the include() function: from django.urls import include, path 15 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 16 | """ 17 | 18 | from django.contrib import admin 19 | from django.urls import include, path 20 | from django.views.generic.base import RedirectView 21 | 22 | urlpatterns = [ 23 | path("", RedirectView.as_view(url="dashboard", permanent=False)), 24 | path("dashboard/", admin.site.urls), 25 | path("__debug__/", include("debug_toolbar.urls")), 26 | ] 27 | -------------------------------------------------------------------------------- /monitorizer/server/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for asmcore project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/5.0/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "monitorizer.server.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /monitorizer/settings.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | HOME = Path("/home/.monitorizer") 4 | HOME.mkdir(exist_ok=True) 5 | 6 | TEMP = HOME / "tmp" 7 | TEMP.mkdir(exist_ok=True) 8 | 9 | WORDLIST = HOME / "wordlist" 10 | WORDLIST.mkdir(exist_ok=True) 11 | -------------------------------------------------------------------------------- /monitorizer/templates/admin/components/chart/pie.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
-------------------------------------------------------------------------------- /monitorizer/templates/admin/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'unfold/layouts/base_simple.html' %} 2 | 3 | {% load cache i18n %} 4 | 5 | {% block breadcrumbs %}{% endblock %} 6 | 7 | {% block title %} 8 | {% if subtitle %} 9 | {{ subtitle }} | 10 | {% endif %} 11 | 12 | {{ title }} | {{ site_title|default:_('Django site admin') }} 13 | {% endblock %} 14 | 15 | {% block branding %} 16 |

17 | 18 | {{ site_header|default:_('Django administration') }} 19 | 20 |

21 | {% endblock %} 22 | 23 | {% block content %} 24 | 25 | {% component "unfold/components/container.html" %} 26 | {% component "unfold/components/flex.html" with class="gap-4" %} 27 | {% component "unfold/components/navigation.html" with items=navigation %}{% endcomponent %} 28 | {% component "unfold/components/navigation.html" with class="ml-auto" items=filters %}{% endcomponent %} 29 | {% endcomponent %} 30 | 31 | {% component "unfold/components/flex.html" with class="gap-8 mb-8 flex-col lg:flex-row" %} 32 | {%for card in cards%} 33 | {% component "unfold/components/card.html" with class="lg:w-1/3" label=card.label %} 34 | {% component "unfold/components/text.html" %} 35 | {{ card.title }} 36 | {% endcomponent %} 37 | {% component "unfold/components/title.html" %} 38 | {{ card.value }} 39 | {% endcomponent %} 40 | {% endcomponent %} 41 | {% endfor %} 42 | {% endcomponent %} 43 | 44 | {% component "unfold/components/flex.html" with class="gap-8 mb-8 flex-col lg:flex-row" %} 45 | {% component "unfold/components/card.html" with class="lg:w-1/2" title="Scans Activity Per Day" %} 46 | {% component "unfold/components/chart/bar.html" with data=activity_per_day_chart height=320 options=DEFAULT_CHART_OPTIONS %}{% endcomponent %} 47 | {% endcomponent %} 48 | {% component "unfold/components/card.html" with class="lg:w-1/2" title="Top Discovered Domains" %} 49 | {% component "unfold/components/flex.html" with col=1 class="gap-8" %} 50 | {% component "admin/components/chart/pie.html" with data=discovered_breakdown options=DEFAULT_CHART_OPTIONS %}{% endcomponent %} 51 | {% endcomponent %} 52 | {% endcomponent %} 53 | {% endcomponent %} 54 | 55 | {% endcomponent %} 56 | 57 | 58 | {% endblock %} -------------------------------------------------------------------------------- /monitorizer/utils/engine.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from monitorizer.settings import TEMP 4 | 5 | 6 | def tmp_file(name: str = None): 7 | return str(TEMP / str(uuid.uuid4())) 8 | -------------------------------------------------------------------------------- /monitorizerv3.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 29 | 30 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "monitorizer" 3 | version = "3.0.0" 4 | description = "" 5 | authors = ["Ahmed Ezzat "] 6 | readme = "README.md" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.11" 10 | django = "^5.0.3" 11 | celery = {extras = ["amqp"], version = "^5.4.0"} 12 | django-unfold = "^0.21.1" 13 | django-celery-beat = "^2.6.0" 14 | django-import-export = "^3.3.7" 15 | requests = "^2.31.0" 16 | markdown = "^3.6" 17 | gunicorn = "^21.2.0" 18 | psycopg = "^3.1.18" 19 | django-debug-toolbar = "^4.3.0" 20 | pytelegrambotapi = "^4.17.0" 21 | pyyaml = "^6.0.1" 22 | jinja2 = "^3.1.3" 23 | django-ace = "^1.32.4" 24 | 25 | [tool.poetry.group.dev.dependencies] 26 | black = "^24.3.0" 27 | isort = "^5.13.2" 28 | 29 | [build-system] 30 | requires = ["poetry-core"] 31 | build-backend = "poetry.core.masonry.api" 32 | -------------------------------------------------------------------------------- /webserver-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | CORE_COUNT=$(grep ^cpu\\scores /proc/cpuinfo | uniq | awk '{print $4}') 5 | WORKERS=${GUNICORN_WORKERS:-$(($CORE_COUNT))} 6 | THREADS=${GUNICORN_THREADS:-$((2 * $WORKERS))} 7 | TIMEOUT=${GUNICORN_TIMEOUT:-500} 8 | KEEPALIVE=${GUNICORN_KEEPALIVE:-300} 9 | SOCKET=unix:/etc/openresty/monitorizer.gunicorn.sock 10 | 11 | python3 -m monitorizer.manage migrate 12 | python3 -m monitorizer.configure 13 | python3 -m monitorizer.manage collectstatic --no-input 14 | 15 | mkdir -p /var/log/openresty/ /etc/openresty/conf.d/ 16 | python3 -m monitorizer.manage openresty_template --gateway $SOCKET > /etc/openresty/conf.d/server.conf 17 | 18 | openresty -t 19 | service openresty start 20 | openresty -s reload 21 | 22 | gunicorn monitorizer.server.wsgi:application\ 23 | -b $SOCKET\ 24 | --capture-output\ 25 | --enable-stdio-inheritance\ 26 | --workers $WORKERS\ 27 | --threads $THREADS\ 28 | --keep-alive $KEEPALIVE\ 29 | --worker-class gthread\ 30 | --error-logfile '-'\ 31 | --max-requests 20000\ 32 | --timeout $TIMEOUT $@ --------------------------------------------------------------------------------