├── .github └── workflows │ └── main.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .travis.yml ├── Dockerfile ├── Procfile ├── README.md ├── agent ├── __init__.py ├── analytics.py ├── app.py ├── base.py ├── bench.py ├── builder.py ├── callbacks.py ├── cli.py ├── database.py ├── database_optimizer.py ├── database_physical_backup.py ├── database_physical_restore.py ├── database_server.py ├── docker_cache_utils.py ├── exceptions.py ├── exporter.py ├── job.py ├── minio.py ├── monitor.py ├── nginx_reload_manager.py ├── pages │ ├── deactivated.html │ ├── exceeded.html │ ├── home.html │ ├── suspended.html │ └── suspended_saas.html ├── patch_handler.py ├── patches.txt ├── patches │ ├── __init__.py │ ├── add_agent_id_field.py │ ├── add_index_on_agent_job_id_field.py │ ├── add_index_on_jobmodel_id_field.py │ └── create_binlog_index_folder.py ├── proxy.py ├── proxysql.py ├── security.py ├── server.py ├── site.py ├── ssh.py ├── templates │ ├── agent │ │ ├── nginx.conf.jinja2 │ │ ├── redis.conf.jinja2 │ │ └── supervisor.conf.jinja2 │ ├── alertmanager │ │ └── routes.yml │ ├── bench │ │ ├── docker-compose.yml.jinja2 │ │ ├── nginx.conf.jinja2 │ │ └── supervisor.conf │ ├── nginx │ │ └── nginx.conf.jinja2 │ ├── prometheus │ │ ├── domains.yml │ │ ├── rules.yml │ │ ├── servers.yml │ │ ├── sites.yml │ │ └── tls.yml │ └── proxy │ │ └── nginx.conf.jinja2 ├── tests │ ├── __init__.py │ ├── test_database.py │ ├── test_database_optimizer.py │ ├── test_proxy.py │ └── test_site.py ├── usage.py ├── utils.py └── web.py ├── dev-requirements.txt ├── docker-compose.yml ├── license.txt ├── redis.conf ├── requirements.txt ├── ruff.toml ├── setup.py └── wait-for-it.sh /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Agent Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - "*" 10 | workflow_dispatch: 11 | 12 | jobs: 13 | lint-and-format: 14 | name: "Lint and Format" 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Checkout code 19 | uses: actions/checkout@v4 20 | 21 | - name: Set up Python 22 | uses: actions/setup-python@v4 23 | with: 24 | python-version: "3.8" 25 | 26 | - name: Install dependencies 27 | run: | 28 | pip install ruff 29 | 30 | - name: Lint Check 31 | run: | 32 | ruff check --output-format github 33 | 34 | - name: Format Check 35 | run: | 36 | ruff format --check 37 | 38 | unit-tests: 39 | name: "Unit Tests" 40 | runs-on: ubuntu-latest 41 | 42 | steps: 43 | - name: Checkout code 44 | uses: actions/checkout@v4 45 | 46 | - name: Set up Python 47 | uses: actions/setup-python@v4 48 | with: 49 | python-version: "3.8" 50 | 51 | - name: Install agent 52 | run: | 53 | python -m venv env 54 | env/bin/pip install -e . 55 | 56 | - name: Install development packages 57 | run: | 58 | env/bin/pip install -r dev-requirements.txt 59 | 60 | - name: Setup agent 61 | run: | 62 | source env/bin/activate 63 | agent setup config --name test.frappe.agent --user abdul --workers 1 64 | agent setup authentication --password password 65 | 66 | - name: Run Tests 67 | run: | 68 | source env/bin/activate 69 | python --version 70 | python -m unittest discover 71 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # VSCode 132 | .vscode/ 133 | 134 | # PyCharm 135 | .idea/ 136 | 137 | # Vim 138 | .vim/ 139 | 140 | .DS_Store -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: 'node_modules|.git' 2 | default_stages: [commit] 3 | fail_fast: false 4 | 5 | repos: 6 | - repo: https://github.com/pre-commit/pre-commit-hooks 7 | rev: v4.4.0 8 | hooks: 9 | - id: debug-statements 10 | - id: trailing-whitespace 11 | files: 'agent.*' 12 | exclude: '.*json$|.*txt$|.*csv|.*md|.*svg' 13 | - id: check-merge-conflict 14 | - id: check-ast 15 | - id: check-json 16 | - id: check-toml 17 | - id: check-yaml 18 | 19 | - repo: https://github.com/astral-sh/ruff-pre-commit 20 | # Ruff version. 21 | rev: v0.6.5 22 | hooks: 23 | # Run the linter. 24 | - id: ruff 25 | args: [--fix] 26 | # Run the formatter. 27 | - id: ruff-format 28 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | os: linux 2 | language: python 3 | dist: bionic 4 | 5 | git: 6 | depth: 1 7 | 8 | python: 9 | - 3.7 10 | 11 | cache: 12 | - pip 13 | 14 | before_install: 15 | - mkdir /home/travis/agent 16 | - cd /home/travis/agent 17 | - virtualenv -p python3 env 18 | - cp -R $TRAVIS_BUILD_DIR repo 19 | - source /home/travis/agent/env/bin/activate 20 | 21 | install: 22 | - pip install -e /home/travis/agent/repo 23 | - cd /home/travis/agent 24 | 25 | jobs: 26 | include: 27 | - name: Agent Setup 28 | addons: 29 | apt: 30 | packages: 31 | - supervisor 32 | hosts: 33 | - test.frappe.agent 34 | 35 | script: 36 | - sudo ln -s /home/travis/agent/supervisor.conf /etc/supervisor/conf.d/agent.conf 37 | - mkdir /home/travis/agent/logs 38 | - agent setup config --name test.frappe.agent --user travis --workers 1 39 | - agent setup authentication --password password 40 | - agent setup supervisor 41 | - agent ping-server --password password 42 | - pip install black flake8 43 | - black -l 79 repo --check --diff 44 | - flake8 repo 45 | 46 | - name: Run Tests 47 | script: 48 | - python -m unittest discover repo 49 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-slim 2 | 3 | RUN useradd -ms /bin/bash frappe 4 | USER frappe 5 | ENV HOME /home/frappe 6 | ENV PATH $PATH:$HOME/.local/bin 7 | 8 | RUN mkdir /home/frappe/agent && \ 9 | mkdir /home/frappe/repo && \ 10 | chown -R frappe:frappe /home/frappe 11 | 12 | COPY --chown=frappe:frappe requirements.txt /home/frappe/repo/ 13 | RUN pip install --user --requirement /home/frappe/repo/requirements.txt 14 | 15 | COPY --chown=frappe:frappe . /home/frappe/repo/ 16 | RUN pip install --user --editable /home/frappe/repo 17 | 18 | WORKDIR /home/frappe/agent 19 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: PYTHONUNBUFFERED=1 FLASK_ENV=development FLASK_DEBUG=1 FLASK_APP=agent.web:application SENTRY_DSN="https://e9b2a2274f2245daab48c6245fb69431@trace.frappe.cloud/17" ./env/bin/flask run -p 25052 2 | redis: redis-server redis.conf 3 | worker_1: PYTHONUNBUFFERED=1 ./repo/wait-for-it.sh redis://127.0.0.1:25025 && ./env/bin/rq worker --url redis://127.0.0.1:25025 high default low --sentry-dsn 'https://e9b2a2274f2245daab48c6245fb69431@trace.frappe.cloud/17' 4 | worker_2: PYTHONUNBUFFERED=1 ./repo/wait-for-it.sh redis://127.0.0.1:25025 && ./env/bin/rq worker --url redis://127.0.0.1:25025 high default low --sentry-dsn 'https://e9b2a2274f2245daab48c6245fb69431@trace.frappe.cloud/17' 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Agent 2 | 3 | ## Installation 4 | 5 | ``` 6 | mkdir agent && cd agent 7 | git clone https://github.com/frappe/agent repo 8 | virtualenv env 9 | source env/bin/activate 10 | pip install -e ./repo 11 | cp repo/redis.conf . 12 | cp repo/Procfile . 13 | ``` 14 | 15 | ## Running 16 | 17 | ``` 18 | honcho start 19 | ``` 20 | 21 | ## CLI 22 | 23 | Agent has a CLI 24 | ([ref](https://github.com/frappe/agent/blob/master/agent/cli.py)). You can 25 | access this by activating the env: 26 | 27 | ```bash 28 | # Path to your agent's Python env might be different 29 | source ./agent/env/bin/activate 30 | 31 | agent --help 32 | ``` 33 | 34 | Once you have activated the env, you can access the iPython console: 35 | 36 | ```bash 37 | agent console 38 | ``` 39 | 40 | This should have the server object instantiated if it was able to find the 41 | `config.json` file. If not you can specify the path (check `agent console --help`). 42 | -------------------------------------------------------------------------------- /agent/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import sys 4 | 5 | from agent.cli import cli 6 | 7 | if __name__ == "__main__": 8 | if getattr(sys, "frozen", False): 9 | cli(sys.argv[1:]) 10 | -------------------------------------------------------------------------------- /agent/analytics.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | import sys 5 | import traceback 6 | from datetime import datetime 7 | 8 | from agent.server import Server 9 | 10 | if __name__ == "__main__": 11 | info = [] 12 | server = Server() 13 | for bench in server.benches.values(): 14 | for site in bench.sites.values(): 15 | try: 16 | timestamp = str(datetime.utcnow()) 17 | info = { 18 | "timestamp": timestamp, 19 | "analytics": site.get_analytics(), 20 | } 21 | with open(site.analytics_file, "w") as f: 22 | json.dump(info, f, indent=1) 23 | except Exception: 24 | exception = traceback.format_exc() 25 | error_log = f"ERROR [{site.name}:{timestamp}]: {exception}" 26 | print(error_log, file=sys.stderr) 27 | -------------------------------------------------------------------------------- /agent/app.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | 5 | from agent.base import Base 6 | 7 | 8 | class App(Base): 9 | def __init__(self, name, bench): 10 | super.__init__() 11 | 12 | self.name = name 13 | self.directory = os.path.join(bench.directory, "apps", name) 14 | if not os.path.isdir(self.directory): 15 | raise Exception 16 | self.execute("git rev-parse --is-inside-work-tree") 17 | 18 | def dump(self): 19 | return {"name": self.name} 20 | 21 | def execute(self, command): 22 | return super().execute(command, directory=self.directory) 23 | 24 | def reset(self, abbreviation="HEAD"): 25 | return self.execute(f"git reset --hard {abbreviation}") 26 | 27 | def fetch(self): 28 | # Automatically unshallow repository while fetching 29 | shallow = self.execute("git rev-parse --is-shallow-repository")["output"] 30 | unshallow = "--unshallow" if shallow == "true" else "" 31 | return self.execute(f"git fetch {self.remote} {unshallow}") 32 | 33 | def fetch_ref(self, ref): 34 | return self.execute(f"git fetch --progress --depth 1 {self.remote} {ref}") 35 | 36 | def checkout(self, ref): 37 | return self.execute(f"git checkout {ref}") 38 | 39 | @property 40 | def remote(self): 41 | remotes = self.execute("git remote")["output"].split("\n") 42 | if "upstream" in remotes: 43 | return "upstream" 44 | if "origin" in remotes: 45 | return "origin" 46 | raise Exception(f"Invalid remote for {self.directory}") 47 | -------------------------------------------------------------------------------- /agent/base.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | import os 5 | import shutil 6 | import subprocess 7 | import tempfile 8 | import traceback 9 | from contextlib import suppress 10 | from datetime import datetime 11 | from functools import partial 12 | from typing import TYPE_CHECKING 13 | 14 | import filelock 15 | import redis 16 | 17 | from agent.exceptions import AgentException 18 | from agent.job import connection 19 | from agent.utils import get_execution_result 20 | 21 | if TYPE_CHECKING: 22 | from typing import Any 23 | 24 | from agent.job import Job, Step 25 | 26 | 27 | class Base: 28 | if TYPE_CHECKING: 29 | job_record: Job | None 30 | step_record: Step | None 31 | 32 | def __init__(self): 33 | self.directory = None 34 | self.config_file = None 35 | self.name = None 36 | self.data: dict[str, str] = {} 37 | 38 | # internal 39 | self._config_file_lock: filelock.SoftFileLock | None = None 40 | 41 | def __repr__(self): 42 | return f"{self.__class__.__name__}({self.name})" 43 | 44 | def execute( 45 | self, 46 | command, 47 | directory=None, 48 | input=None, 49 | skip_output_log=False, 50 | executable=None, 51 | non_zero_throw=True, 52 | ): 53 | directory = directory or self.directory 54 | start = datetime.now() 55 | self.skip_output_log = skip_output_log 56 | self.data = get_execution_result(command, directory, start) 57 | self.log() 58 | output = "" 59 | try: 60 | output, returncode = self.run_subprocess( 61 | command, 62 | directory, 63 | input, 64 | executable, 65 | non_zero_throw, 66 | ) 67 | except subprocess.CalledProcessError as e: 68 | output = str(e.output or "") 69 | returncode = e.returncode 70 | self.data.update( 71 | { 72 | "status": "Failure", 73 | "traceback": "".join(traceback.format_exc()), 74 | } 75 | ) 76 | raise AgentException(self.data) from e 77 | else: 78 | self.data.update({"status": "Success"}) 79 | finally: 80 | end = datetime.now() 81 | self.data.update( 82 | { 83 | "returncode": returncode, 84 | "duration": end - start, 85 | "end": end, 86 | "output": output, 87 | } 88 | ) 89 | self.log() 90 | return self.data 91 | 92 | def run_subprocess(self, command, directory, input, executable, non_zero_throw=True): 93 | # Start a child process and start reading output immediately 94 | with subprocess.Popen( 95 | command, 96 | stdout=subprocess.PIPE, 97 | stderr=subprocess.STDOUT, 98 | stdin=subprocess.PIPE if input else None, 99 | cwd=directory, 100 | shell=True, 101 | executable=executable, 102 | ) as process: 103 | if input: 104 | process._stdin_write(input.encode()) 105 | 106 | output = self.parse_output(process) 107 | returncode = process.poll() or 0 108 | # This is equivalent of check=True 109 | # Raise an exception if the process returns a non-zero return code 110 | if non_zero_throw and returncode: 111 | raise subprocess.CalledProcessError(returncode, command, output=output) 112 | return output, returncode 113 | 114 | def parse_output(self, process) -> str: 115 | if not process.stdout: 116 | return "" 117 | 118 | line = b"" 119 | lines = [] 120 | # This is equivalent of remove_crs 121 | # Make sure output matches what'll be shown in the terminal 122 | # This won't work for top, htop etc, but good enough to handle progress bars 123 | for char in iter(partial(process.stdout.read, 1), b""): 124 | if char == b"" and process.poll() is not None: 125 | break 126 | if char == b"\r": 127 | # Publish output and then wipe current line. 128 | # Include the overwritten line in the output 129 | self.publish_lines([*lines, line.decode(errors="replace")]) 130 | line = b"" 131 | elif char == b"\n": 132 | lines.append(line.decode(errors="replace")) 133 | line = b"" 134 | self.publish_lines(lines) 135 | else: 136 | line += char 137 | 138 | if line: 139 | lines.append(line.decode(errors="replace")) 140 | self.publish_lines(lines) 141 | return "\n".join(lines) 142 | 143 | def publish_lines(self, lines: list[str]): 144 | output = "\n".join(lines) 145 | self.data.update({"output": output}) 146 | self.update_redis() 147 | 148 | def publish_data(self, data: Any): 149 | if not isinstance(data, str): 150 | data = json.dumps(data, default=str) 151 | 152 | self.data.update({"output": data}) 153 | self.update_redis() 154 | 155 | def update_redis(self): 156 | if not (redis_key := self.get_redis_key()): 157 | return 158 | 159 | value = json.dumps(self.data, default=str) 160 | self.push_redis_value(redis_key, value) 161 | self.redis.expire(redis_key, 60 * 60 * 6) 162 | 163 | def push_redis_value(self, key: str, value: str): 164 | if "output" not in self.data: 165 | self.redis.rpush(key, value) 166 | 167 | try: 168 | self.redis.lset(key, -1, value) 169 | except redis.exceptions.ResponseError as e: 170 | if "no such key" in str(e): 171 | self.redis.rpush(key, value) 172 | 173 | def get_redis_key(self): 174 | if not self.job_record: 175 | return None 176 | 177 | if not hasattr(self.job_record, "model"): 178 | return None 179 | 180 | key = f"agent:job:{self.job_record.model.id}" 181 | if self.step_record and hasattr(self.step_record, "model"): 182 | return f"{key}:step:{self.step_record.model.id}" 183 | 184 | return key 185 | 186 | @property 187 | def redis(self): 188 | return connection() 189 | 190 | @property 191 | def config(self) -> dict: 192 | """ 193 | This should be used where we know that, 194 | the config file isn't going to be frequently updated. 195 | """ 196 | return self.get_config() 197 | 198 | def get_config(self, for_update: bool = False) -> dict: 199 | """ 200 | If we are fetching the config for updating some part of it. 201 | It's better to acquire the lock to avoid race conditions and dirty reads. 202 | """ 203 | if for_update: 204 | if not self._config_file_lock: 205 | self._config_file_lock = filelock.SoftFileLock(self.config_file + ".lock") 206 | self._config_file_lock.acquire() 207 | 208 | with open(self.config_file, "r") as f: 209 | return json.load(f) 210 | 211 | def set_config(self, value: dict, indent=1, release_lock: bool = True): 212 | """ 213 | Args: 214 | value (dict): Config to be set 215 | indent (int, optional): Indent for the config file. Defaults to 1. 216 | release_lock (bool, optional): Release the lock after setting the config. Defaults to True. 217 | If we have more writes to do, then we should not release the lock. 218 | 219 | To prevent partial writes, we need to first write the config to a temporary file, 220 | then rename it to the original file. 221 | """ 222 | with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: 223 | json.dump(value, temp_file, indent=indent, sort_keys=True) 224 | temp_file.flush() 225 | os.fsync(temp_file.fileno()) 226 | temp_file.close() 227 | 228 | shutil.copy2(temp_file.name, self.config_file) 229 | os.remove(temp_file.name) 230 | 231 | if release_lock and self._config_file_lock: 232 | self._config_file_lock.release() 233 | 234 | def log(self): 235 | data = self.data.copy() 236 | if self.skip_output_log: 237 | data.update({"output": ""}) 238 | print(json.dumps(data, default=str)) 239 | self.update_redis() 240 | 241 | @property 242 | def logs(self): 243 | def path(file): 244 | return os.path.join(self.logs_directory, file) 245 | 246 | def modified_time(file): 247 | return os.path.getctime(path(file)) 248 | 249 | try: 250 | log_files = sorted( 251 | os.listdir(self.logs_directory), 252 | key=modified_time, 253 | reverse=True, 254 | ) 255 | payload = [] 256 | 257 | for x in log_files: 258 | stats = os.stat(path(x)) 259 | payload.append( 260 | { 261 | "name": x, 262 | "size": stats.st_size / 1000, 263 | "created": str(datetime.fromtimestamp(stats.st_ctime)), 264 | "modified": str(datetime.fromtimestamp(stats.st_mtime)), 265 | } 266 | ) 267 | 268 | return payload 269 | 270 | except FileNotFoundError: 271 | return [] 272 | 273 | def retrieve_log(self, name): 274 | if name not in {x["name"] for x in self.logs}: 275 | return "" 276 | log_file = os.path.join(self.logs_directory, name) 277 | with open(log_file) as lf: 278 | return lf.read() 279 | 280 | def __del__(self): 281 | # Release lock at the end of the object's lifetime 282 | if self._config_file_lock: 283 | with suppress(Exception): 284 | self._config_file_lock.release() 285 | -------------------------------------------------------------------------------- /agent/builder.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | import shlex 5 | import subprocess 6 | from datetime import datetime 7 | from subprocess import Popen 8 | from typing import TYPE_CHECKING 9 | 10 | import docker 11 | 12 | from agent.base import Base 13 | from agent.job import Job, Step, job, step 14 | 15 | if TYPE_CHECKING: 16 | from typing import Literal 17 | 18 | OutputKey = Literal["build", "push"] 19 | Output = dict[OutputKey, list[str]] 20 | 21 | 22 | class ImageBuilder(Base): 23 | output: Output 24 | 25 | def __init__( 26 | self, 27 | filename: str, 28 | image_repository: str, 29 | image_tag: str, 30 | no_cache: bool, 31 | no_push: bool, 32 | registry: dict, 33 | platform: str, 34 | ) -> None: 35 | super().__init__() 36 | 37 | # Image push params 38 | self.image_repository = image_repository 39 | self.image_tag = image_tag 40 | self.registry = registry 41 | self.platform = platform 42 | 43 | # Build context, params 44 | self.filename = filename 45 | self.filepath = os.path.join( 46 | get_image_build_context_directory(), 47 | self.filename, 48 | ) 49 | self.no_cache = no_cache 50 | self.no_push = no_push 51 | self.last_published = datetime.now() 52 | self.build_failed = False 53 | 54 | cwd = os.getcwd() 55 | self.config_file = os.path.join(cwd, "config.json") 56 | 57 | # Lines from build and push are sent to press for processing 58 | # and updating the respective Deploy Candidate 59 | self.output = { 60 | "build": [], 61 | "push": [], 62 | } 63 | self.push_output_lines = [] 64 | 65 | self.job = None 66 | self.step = None 67 | 68 | @property 69 | def job_record(self): 70 | if self.job is None: 71 | self.job = Job() 72 | return self.job 73 | 74 | @property 75 | def step_record(self): 76 | if self.step is None: 77 | self.step = Step() 78 | return self.step 79 | 80 | @step_record.setter 81 | def step_record(self, value): 82 | self.step = value 83 | 84 | @job("Run Remote Builder") 85 | def run_remote_builder(self): 86 | try: 87 | return self._build_and_push() 88 | finally: 89 | self._cleanup_context() 90 | 91 | def _build_and_push(self): 92 | self._build_image() 93 | if not self.build_failed and not self.no_push: 94 | self._push_docker_image() 95 | return self.data 96 | 97 | @step("Build Image") 98 | def _build_image(self): 99 | # Note: build command and environment are different from when 100 | # build runs on the press server. 101 | command = self._get_build_command() 102 | environment = self._get_build_environment() 103 | result = self._run( 104 | command=command, 105 | environment=environment, 106 | input_filepath=self.filepath, 107 | ) 108 | self.output["build"] = [] 109 | self._publish_docker_build_output(result) 110 | return {"output": self.output["build"]} 111 | 112 | def _get_build_command(self) -> str: 113 | command = f"docker buildx build --platform {self.platform}" 114 | command = f"{command} -t {self._get_image_name()}" 115 | 116 | if self.no_cache: 117 | command = f"{command} --no-cache" 118 | 119 | return f"{command} - " 120 | 121 | def _get_build_environment(self) -> dict: 122 | environment = os.environ.copy() 123 | environment.update( 124 | { 125 | "DOCKER_BUILDKIT": "1", 126 | "BUILDKIT_PROGRESS": "plain", 127 | "PROGRESS_NO_TRUNC": "1", 128 | } 129 | ) 130 | return environment 131 | 132 | def _publish_docker_build_output(self, result): 133 | for line in result: 134 | self.output["build"].append(line) 135 | self._publish_throttled_output(False) 136 | self._publish_throttled_output(True) 137 | 138 | @step("Push Docker Image") 139 | def _push_docker_image(self): 140 | environment = os.environ.copy() 141 | client = docker.from_env(environment=environment, timeout=5 * 60) 142 | auth_config = { 143 | "username": self.registry["username"], 144 | "password": self.registry["password"], 145 | "serveraddress": self.registry["url"], 146 | } 147 | try: 148 | for line in client.images.push( 149 | self.image_repository, 150 | self.image_tag, 151 | stream=True, 152 | decode=True, 153 | auth_config=auth_config, 154 | ): 155 | self.output["push"].append(line) 156 | self._publish_throttled_output(False) 157 | except Exception: 158 | self._publish_throttled_output(True) 159 | raise 160 | return self.output["push"] 161 | 162 | def _publish_throttled_output(self, flush: bool): 163 | if flush: 164 | self.publish_data(self.output) 165 | return 166 | 167 | now = datetime.now() 168 | if (now - self.last_published).total_seconds() <= 1: 169 | return 170 | 171 | self.last_published = now 172 | self.publish_data(self.output) 173 | 174 | def _get_image_name(self): 175 | return f"{self.image_repository}:{self.image_tag}" 176 | 177 | def _run( 178 | self, 179 | command: str, 180 | environment: dict, 181 | input_filepath: str, 182 | ): 183 | with open(input_filepath, "rb") as input_file: 184 | process = Popen( 185 | shlex.split(command), 186 | stdin=input_file, 187 | stdout=subprocess.PIPE, 188 | stderr=subprocess.STDOUT, 189 | env=environment, 190 | universal_newlines=True, 191 | ) 192 | 193 | yield from process.stdout 194 | 195 | process.stdout.close() 196 | input_file.close() 197 | 198 | return_code = process.wait() 199 | self._publish_throttled_output(True) 200 | 201 | self.build_failed = return_code != 0 202 | self.data.update({"build_failed": self.build_failed}) 203 | 204 | @step("Cleanup Context") 205 | def _cleanup_context(self): 206 | if not os.path.exists(self.filepath): 207 | return {"cleanup": False} 208 | 209 | os.remove(self.filepath) 210 | return {"cleanup": True} 211 | 212 | 213 | def get_image_build_context_directory(): 214 | path = os.path.join(os.getcwd(), "build_context") 215 | if not os.path.exists(path): 216 | os.makedirs(path) 217 | return path 218 | -------------------------------------------------------------------------------- /agent/callbacks.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | 4 | def callback(job, connection, result, *args, **kwargs): 5 | from agent.server import Server 6 | 7 | press_url = Server().press_url 8 | requests.post(url=f"{press_url}/api/method/press.api.callbacks.callback", data={"job_id": job.id}) 9 | -------------------------------------------------------------------------------- /agent/cli.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | import os 5 | import shutil 6 | import sys 7 | from pathlib import Path 8 | from typing import TYPE_CHECKING 9 | 10 | import click 11 | import requests 12 | 13 | from agent.proxy import Proxy 14 | from agent.server import Server 15 | from agent.utils import get_timestamp 16 | 17 | if TYPE_CHECKING: 18 | from IPython.terminal.embed import InteractiveShellEmbed 19 | 20 | 21 | @click.group() 22 | def cli(): 23 | pass 24 | 25 | 26 | @cli.group() 27 | def setup(): 28 | pass 29 | 30 | 31 | @cli.command() 32 | @click.option("--restart-web-workers", default=True) 33 | @click.option("--restart-rq-workers", default=True) 34 | @click.option("--restart-redis", default=True) 35 | @click.option("--skip-repo-setup", default=False) 36 | @click.option("--skip-patches", default=False) 37 | def update(restart_web_workers, restart_rq_workers, restart_redis, skip_repo_setup, skip_patches): 38 | Server().update_agent_cli( 39 | restart_redis=restart_redis, 40 | restart_rq_workers=restart_rq_workers, 41 | restart_web_workers=restart_web_workers, 42 | skip_repo_setup=skip_repo_setup, 43 | skip_patches=skip_patches, 44 | ) 45 | 46 | 47 | @cli.command() 48 | def run_patches(): 49 | from agent.patch_handler import run_patches 50 | 51 | run_patches() 52 | 53 | 54 | @cli.command() 55 | @click.option("--password", required=True) 56 | def ping_server(password: str): 57 | """Ping web api on localhost and check for pong.""" 58 | res = requests.get( 59 | "http://localhost:25052/ping", 60 | headers={"Authorization": f"bearer {password}"}, 61 | ) 62 | res = res.json() 63 | if res["message"] != "pong": 64 | raise Exception("pong not in response") 65 | print(res) 66 | 67 | 68 | @setup.command() 69 | @click.option("--name", required=True) 70 | @click.option("--user", default="frappe") 71 | @click.option("--workers", required=True, type=int) 72 | @click.option("--proxy-ip", required=False, type=str, default=None) 73 | @click.option("--sentry-dsn", required=False, type=str) 74 | @click.option("--press-url", required=False, type=str) 75 | def config(name, user, workers, proxy_ip=None, sentry_dsn=None, press_url=None): 76 | config = { 77 | "benches_directory": f"/home/{user}/benches", 78 | "name": name, 79 | "tls_directory": f"/home/{user}/agent/tls", 80 | "nginx_directory": f"/home/{user}/agent/nginx", 81 | "redis_port": 25025, 82 | "user": user, 83 | "workers": workers, 84 | "gunicorn_workers": 2, 85 | "web_port": 25052, 86 | "press_url": "https://frappecloud.com", 87 | } 88 | if press_url: 89 | config["press_url"] = press_url 90 | if proxy_ip: 91 | config["proxy_ip"] = proxy_ip 92 | if sentry_dsn: 93 | config["sentry_dsn"] = sentry_dsn 94 | 95 | with open("config.json", "w") as f: 96 | json.dump(config, f, sort_keys=True, indent=4) 97 | 98 | 99 | @setup.command() 100 | def pyspy(): 101 | privileges_line = "frappe ALL = (root) NOPASSWD: /home/frappe/agent/env/bin/py-spy" 102 | with open("/etc/sudoers.d/frappe", "a+") as sudoers: 103 | sudoers.seek(0) 104 | lines = sudoers.read().splitlines() 105 | 106 | if privileges_line not in lines: 107 | sudoers.write(privileges_line + "\n") 108 | 109 | 110 | @setup.command() 111 | @click.option("--password", prompt=True, hide_input=True) 112 | def authentication(password): 113 | Server().setup_authentication(password) 114 | 115 | 116 | @setup.command() 117 | @click.option("--sentry-dsn", required=True) 118 | def sentry(sentry_dsn): 119 | Server().setup_sentry(sentry_dsn) 120 | 121 | 122 | @setup.command() 123 | def supervisor(): 124 | Server().setup_supervisor() 125 | 126 | 127 | @setup.command() 128 | def nginx(): 129 | Server().setup_nginx() 130 | 131 | 132 | @setup.command() 133 | @click.option("--domain") 134 | @click.option("--press-url") 135 | def proxy(domain=None, press_url=None): 136 | proxy = Proxy() 137 | if domain: 138 | config = proxy.get_config(for_update=True) 139 | config["domain"] = domain 140 | config["press_url"] = press_url 141 | proxy.set_config(config, indent=4) 142 | proxy.setup_proxy() 143 | 144 | 145 | @setup.command() 146 | @click.option("--domain") 147 | def standalone(domain=None): 148 | server = Server() 149 | if domain: 150 | config = server.get_config(for_update=True) 151 | config["domain"] = domain 152 | config["standalone"] = True 153 | server.set_config(config, indent=4) 154 | 155 | 156 | @setup.command() 157 | def database(): 158 | from agent.job import JobModel, PatchLogModel, StepModel 159 | from agent.job import agent_database as database 160 | 161 | database.create_tables([JobModel, StepModel, PatchLogModel]) 162 | 163 | 164 | @setup.command() 165 | def site_analytics(): 166 | from crontab import CronTab 167 | 168 | script_directory = os.path.dirname(__file__) 169 | agent_directory = os.path.dirname(os.path.dirname(script_directory)) 170 | logs_directory = os.path.join(agent_directory, "logs") 171 | script = os.path.join(script_directory, "analytics.py") 172 | stdout = os.path.join(logs_directory, "analytics.log") 173 | stderr = os.path.join(logs_directory, "analytics.error.log") 174 | 175 | cron = CronTab(user=True) 176 | command = f"cd {agent_directory} && {sys.executable} {script} 1>> {stdout} 2>> {stderr}" 177 | 178 | if command in str(cron): 179 | cron.remove_all(command=command) 180 | 181 | job = cron.new(command=command) 182 | job.hour.on(23) 183 | job.minute.on(0) 184 | cron.write() 185 | 186 | 187 | @setup.command() 188 | def usage(): 189 | from crontab import CronTab 190 | 191 | script_directory = os.path.dirname(__file__) 192 | agent_directory = os.path.dirname(os.path.dirname(script_directory)) 193 | logs_directory = os.path.join(agent_directory, "logs") 194 | script = os.path.join(script_directory, "usage.py") 195 | stdout = os.path.join(logs_directory, "usage.log") 196 | stderr = os.path.join(logs_directory, "usage.error.log") 197 | 198 | cron = CronTab(user=True) 199 | command = f"cd {agent_directory} && {sys.executable} {script} 1>> {stdout} 2>> {stderr}" 200 | 201 | if command not in str(cron): 202 | job = cron.new(command=command) 203 | job.every(6).hours() 204 | job.minute.on(30) 205 | cron.write() 206 | 207 | 208 | @setup.command() 209 | def registry(): 210 | Server().setup_registry() 211 | 212 | 213 | @setup.command() 214 | @click.option("--url", required=True) 215 | @click.option("--token", required=True) 216 | def monitor(url, token): 217 | from agent.monitor import Monitor 218 | 219 | server = Monitor() 220 | server.update_config({"monitor": True, "press_url": url, "press_token": token}) 221 | server.discover_targets() 222 | 223 | 224 | @setup.command() 225 | def log(): 226 | Server().setup_log() 227 | 228 | 229 | @setup.command() 230 | def analytics(): 231 | Server().setup_analytics() 232 | 233 | 234 | @setup.command() 235 | def trace(): 236 | Server().setup_trace() 237 | 238 | 239 | @setup.command() 240 | @click.option("--password", prompt=True, hide_input=True) 241 | def proxysql(password): 242 | Server().setup_proxysql(password) 243 | 244 | 245 | @cli.group() 246 | def run(): 247 | pass 248 | 249 | 250 | @run.command() 251 | def web(): 252 | executable = shutil.which("gunicorn") 253 | port = Server().config["web_port"] 254 | arguments = [ 255 | executable, 256 | "--bind", 257 | f"127.0.0.1:{port}", 258 | "--reload", 259 | "--preload", 260 | "agent.web:application", 261 | ] 262 | os.execv(executable, arguments) 263 | 264 | 265 | @run.command() 266 | def worker(): 267 | executable = shutil.which("rq") 268 | port = Server().config["redis_port"] 269 | arguments = [ 270 | executable, 271 | "worker", 272 | "--url", 273 | f"redis://127.0.0.1:{port}", 274 | ] 275 | os.execv(executable, arguments) 276 | 277 | 278 | @cli.command() 279 | def discover(): 280 | from agent.monitor import Monitor 281 | 282 | Monitor().discover_targets() 283 | 284 | 285 | @cli.group() 286 | def bench(): 287 | pass 288 | 289 | 290 | @bench.command() 291 | @click.argument("bench", nargs=-1) 292 | def start(bench: tuple[str]): 293 | server = Server() 294 | 295 | if bench: 296 | for b in bench: 297 | server.benches[b].start() 298 | else: 299 | server.start_all_benches() 300 | 301 | 302 | @bench.command() 303 | @click.argument("bench", required=False) 304 | def stop(bench): 305 | if bench: 306 | return Server().benches[bench].stop() 307 | return Server().stop_all_benches() 308 | 309 | 310 | @cli.command(help="Run iPython console.") 311 | @click.option( 312 | "--config-path", 313 | required=False, 314 | type=str, 315 | help="Path to agent config.json.", 316 | ) 317 | def console(config_path): 318 | from atexit import register 319 | 320 | from IPython.terminal.embed import InteractiveShellEmbed 321 | 322 | terminal = InteractiveShellEmbed.instance() 323 | 324 | config_dir = get_config_dir(config_path) 325 | if config_dir: 326 | try: 327 | locals()["server"] = Server(config_dir) 328 | print(f"In namespace:\nserver = agent.server.Server('{config_dir}')") 329 | except Exception: 330 | print(f"Could not initialize agent.server.Server('{config_dir}')") 331 | 332 | elif config_path: 333 | print(f"Could not find config.json at '{config_path}'") 334 | else: 335 | print("Could not find config.json use --config-path to specify") 336 | 337 | register(store_ipython_logs, terminal, config_dir) 338 | 339 | # ref: https://stackoverflow.com/a/74681224 340 | try: 341 | from IPython.core import ultratb 342 | 343 | ultratb.VerboseTB._tb_highlight = "bg:ansibrightblack" 344 | except Exception: 345 | pass 346 | 347 | terminal.colors = "neutral" 348 | terminal.display_banner = False 349 | terminal() 350 | 351 | 352 | def get_config_dir(config_path: str | None = None) -> str | None: 353 | cwd = os.getcwd() 354 | if config_path is None: 355 | config_path = cwd 356 | 357 | config_dir = Path(config_path) 358 | 359 | if config_dir.suffix == "json" and config_dir.exists(): 360 | return config_dir.parent.as_posix() 361 | 362 | if config_dir.suffix != "": 363 | config_dir = config_dir.parent 364 | 365 | potential = [ 366 | Path("/home/frappe/agent/config.json"), 367 | config_dir / "config.json", 368 | config_dir / ".." / "config.json", 369 | ] 370 | 371 | for p in potential: 372 | if not p.exists(): 373 | continue 374 | try: 375 | return p.parent.relative_to(cwd).as_posix() 376 | except Exception: 377 | return p.parent.as_posix() 378 | return None 379 | 380 | 381 | def store_ipython_logs(terminal: InteractiveShellEmbed, config_dir: str | None): 382 | if not config_dir: 383 | config_dir = os.getcwd() 384 | 385 | log_path = Path(config_dir) / "logs" / "agent_console.log" 386 | log_path.parent.mkdir(exist_ok=True) 387 | 388 | with log_path.open("a") as file: 389 | timestamp = get_timestamp() 390 | 391 | file.write(f"# SESSION BEGIN {timestamp}\n") 392 | for line in terminal.history_manager.get_range(): 393 | file.write(f"{line[2]}\n") 394 | file.write(f"# SESSION END {timestamp}\n\n") 395 | -------------------------------------------------------------------------------- /agent/database_physical_backup.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import contextlib 4 | import json 5 | import os 6 | import subprocess 7 | import time 8 | from random import randint 9 | 10 | import requests 11 | 12 | from agent.database import CustomPeeweeDB 13 | from agent.database_server import DatabaseServer 14 | from agent.job import job, step 15 | from agent.utils import compute_file_hash, decode_mariadb_filename 16 | 17 | 18 | class DatabasePhysicalBackup(DatabaseServer): 19 | def __init__( 20 | self, 21 | databases: list[str], 22 | db_user: str, 23 | db_password: str, 24 | site_backup_name: str, 25 | snapshot_trigger_url: str, 26 | snapshot_request_key: str, 27 | db_host: str = "localhost", 28 | db_port: int = 3306, 29 | db_base_path: str = "/var/lib/mysql", 30 | ): 31 | if not databases: 32 | raise ValueError("At least one database is required") 33 | # Instance variable for internal use 34 | self._db_instances: dict[str, CustomPeeweeDB] = {} 35 | self._db_instances_connection_id: dict[str, int] = {} 36 | self._db_tables_locked: dict[str, bool] = {db: False for db in databases} 37 | 38 | # variables 39 | self.site_backup_name = site_backup_name 40 | self.snapshot_trigger_url = snapshot_trigger_url 41 | self.snapshot_request_key = snapshot_request_key 42 | self.databases = databases 43 | self.db_user = db_user 44 | self.db_password = db_password 45 | self.db_host = db_host 46 | self.db_port = db_port 47 | self.db_base_path = db_base_path 48 | self.db_directories: dict[str, str] = { 49 | db: os.path.join(self.db_base_path, db) for db in self.databases 50 | } 51 | 52 | self.innodb_tables: dict[str, list[str]] = {db: [] for db in self.databases} 53 | self.myisam_tables: dict[str, list[str]] = {db: [] for db in self.databases} 54 | self.sequence_tables: dict[str, list[str]] = {db: [] for db in self.databases} 55 | self.files_metadata: dict[str, dict[str, dict[str, str]]] = {db: {} for db in self.databases} 56 | self.table_schemas: dict[str, str] = {} 57 | 58 | super().__init__() 59 | 60 | @job("Physical Backup Database", priority="low") 61 | def create_backup_job(self): 62 | self.remove_backups_metadata() 63 | self.fetch_table_info() 64 | self.flush_tables() 65 | self.flush_changes_to_disk() 66 | self.validate_exportable_files() 67 | self.export_table_schemas() 68 | self.collect_files_metadata() 69 | self.store_backup_metadata() 70 | self.create_snapshot() # Blocking call 71 | self.unlock_all_tables() 72 | self.remove_backups_metadata() 73 | 74 | def remove_backups_metadata(self): 75 | with contextlib.suppress(Exception): 76 | for db_name in self.databases: 77 | os.remove(get_path_of_physical_backup_metadata(self.db_base_path, db_name)) 78 | 79 | @step("Fetch Database Tables Information") 80 | def fetch_table_info(self): 81 | """ 82 | Store the table names and their engines in the respective dictionaries 83 | """ 84 | for db_name in self.databases: 85 | db_instance = self.get_db(db_name) 86 | query = ( 87 | "SELECT table_name, engine, table_type FROM information_schema.tables " 88 | "WHERE table_schema = DATABASE() AND table_type != 'VIEW' " 89 | "ORDER BY table_name" 90 | ) 91 | data = db_instance.execute_sql(query).fetchall() 92 | for row in data: 93 | table = row[0] 94 | engine = row[1] 95 | table_type = row[2] 96 | if engine == "InnoDB": 97 | self.innodb_tables[db_name].append(table) 98 | elif engine == "MyISAM": 99 | self.myisam_tables[db_name].append(table) 100 | 101 | if table_type == "SEQUENCE": 102 | """ 103 | Sequence table can use any engine 104 | 105 | In mariadb-dump result, sequence table will have specific SQL for creating/dropping 106 | https://mariadb.com/kb/en/create-sequence/ 107 | https://mariadb.com/kb/en/drop-sequence/ 108 | 109 | Store the table references, so that we can handle this in a different manner 110 | during physical restore 111 | """ 112 | self.sequence_tables[db_name].append(table) 113 | 114 | @step("Flush Database Tables") 115 | def flush_tables(self): 116 | for db_name in self.databases: 117 | """ 118 | InnoDB and MyISAM tables 119 | Flush the tables and take read lock 120 | 121 | Ref : https://mariadb.com/kb/en/flush-tables-for-export/#:~:text=If%20FLUSH%20TABLES%20...%20FOR%20EXPORT%20is%20in%20effect%20in%20the%20session%2C%20the%20following%20statements%20will%20produce%20an%20error%20if%20attempted%3A 122 | 123 | FLUSH TABLES ... FOR EXPORT 124 | This will 125 | - Take READ lock on the tables 126 | - Flush the tables 127 | - Will not allow to change table structure (ALTER TABLE, DROP TABLE nothing will work) 128 | """ 129 | tables = self.innodb_tables[db_name] + self.myisam_tables[db_name] 130 | tables = [f"`{table}`" for table in tables] 131 | flush_table_export_query = "FLUSH TABLES {} FOR EXPORT;".format(", ".join(tables)) 132 | self._kill_other_db_connections(db_name) 133 | self.get_db(db_name).execute_sql(flush_table_export_query) 134 | self._db_tables_locked[db_name] = True 135 | 136 | @step("Flush Changes to Disk") 137 | def flush_changes_to_disk(self): 138 | """ 139 | It's important to flush all the disk buffer of files to disk before snapshot. 140 | This will ensure that the snapshot is consistent. 141 | """ 142 | for db_name in self.databases: 143 | files = os.listdir(self.db_directories[db_name]) 144 | for file in files: 145 | file_path = os.path.join(self.db_directories[db_name], file) 146 | """ 147 | Open the file in binary mode and keep buffering disabled(allowed only for binary mode) 148 | https://docs.python.org/3/library/functions.html#open:~:text=buffering%20is%20an%20optional%20integer 149 | 150 | With this change, we don't need to call f.flush() 151 | https://docs.python.org/3/library/os.html#os.fsync 152 | """ 153 | with open(file_path, "rb", buffering=0) as f: 154 | os.fsync(f.fileno()) 155 | 156 | @step("Validate Exportable Files") 157 | def validate_exportable_files(self): 158 | for db_name in self.databases: 159 | # list all the files in the database directory 160 | db_files = os.listdir(self.db_directories[db_name]) 161 | db_files = [decode_mariadb_filename(file) for file in db_files] 162 | """ 163 | InnoDB tables should have the .cfg files to be able to restore it back 164 | 165 | https://mariadb.com/kb/en/innodb-file-per-table-tablespaces/#exporting-transportable-tablespaces-for-non-partitioned-tables 166 | 167 | Additionally, ensure .frm files should be present as well. 168 | If Import tablespace failed for some reason and we need to reconstruct the tables, 169 | we need .frm files to start the database server. 170 | """ 171 | for table in self.innodb_tables[db_name]: 172 | file = table + ".cfg" 173 | if file not in db_files: 174 | raise DatabaseExportFileNotFoundError(f"CFG file for table {table} not found") 175 | file = table + ".frm" 176 | if file not in db_files: 177 | raise DatabaseExportFileNotFoundError(f"FRM file for table {table} not found") 178 | 179 | """ 180 | MyISAM tables should have .MYD and .MYI files at-least to be able to restore it back 181 | """ 182 | for table in self.myisam_tables[db_name]: 183 | table_files = [table + ".MYD", table + ".MYI"] 184 | for table_file in table_files: 185 | if table_file not in db_files: 186 | raise DatabaseExportFileNotFoundError(f"MYD or MYI file for table {table} not found") 187 | 188 | @step("Export Table Schema") 189 | def export_table_schemas(self): 190 | for db_name in self.databases: 191 | """ 192 | Export the database schema 193 | It's important to export the schema only after taking the read lock. 194 | """ 195 | self.table_schemas[db_name] = self.export_table_schema(db_name) 196 | 197 | @step("Collect Files Metadata") 198 | def collect_files_metadata(self): 199 | for db_name in self.databases: 200 | files = os.listdir(self.db_directories[db_name]) 201 | for file in files: 202 | file_extension = os.path.splitext(file)[-1].lower() 203 | if file_extension not in [".cfg", ".frm", ".myd", ".myi", ".ibd"]: 204 | continue 205 | file_path = os.path.join(self.db_directories[db_name], file) 206 | """ 207 | cfg, frm files are too important to restore/reconstruct the database. 208 | This files are small also, so we can take the checksum of these files. 209 | 210 | For IBD, MYD files we will just take the size of the file as a validation during restore, 211 | whether the fsync has happened successfully or not. 212 | 213 | For MariaDB > 10.6.0, it use O_DIRECT as innodb_flush_method, 214 | So, data will be written directly to the disk without buffering. 215 | Only the metadata will be updated via fsync. 216 | 217 | https://mariadb.com/kb/en/innodb-system-variables/#innodb_flush_method 218 | """ 219 | self.files_metadata[db_name][file] = { 220 | "size": os.path.getsize(file_path), 221 | "checksum": compute_file_hash(file_path, raise_exception=True) 222 | if file_extension in [".cfg", ".frm", ".myi"] 223 | else None, 224 | } 225 | 226 | @step("Store Backup Metadata") 227 | def store_backup_metadata(self): 228 | """ 229 | Store the backup metadata in the database 230 | """ 231 | data = {} 232 | for db_name in self.databases: 233 | data = { 234 | "innodb_tables": self.innodb_tables[db_name], 235 | "myisam_tables": self.myisam_tables[db_name], 236 | "sequence_tables": self.sequence_tables[db_name], 237 | "table_schema": self.table_schemas[db_name], 238 | "files_metadata": self.files_metadata[db_name], 239 | } 240 | file_path = get_path_of_physical_backup_metadata(self.db_base_path, db_name) 241 | with open(file_path, "w") as f: 242 | json.dump(data, f) 243 | 244 | os.chmod(file_path, 0o777) 245 | 246 | with open(file_path, "rb", buffering=0) as f: 247 | os.fsync(f.fileno()) 248 | 249 | @step("Create Database Snapshot") 250 | def create_snapshot(self): 251 | """ 252 | Trigger the snapshot creation 253 | """ 254 | retries = 0 255 | 256 | while True: 257 | response = requests.post( 258 | self.snapshot_trigger_url, 259 | json={ 260 | "name": self.site_backup_name, 261 | "key": self.snapshot_request_key, 262 | }, 263 | ) 264 | if response.status_code in [417, 500, 502, 503, 504] and retries <= 10: 265 | retries += 1 266 | time.sleep(15 + randint(2, 8)) 267 | continue 268 | 269 | response.raise_for_status() 270 | break 271 | 272 | @step("Unlock Tables") 273 | def unlock_all_tables(self): 274 | for db_name in self.databases: 275 | self._unlock_tables(db_name) 276 | 277 | def export_table_schema(self, db_name: str) -> str: 278 | self._kill_other_db_connections(db_name) 279 | command = [ 280 | "mariadb-dump", 281 | "-u", 282 | self.db_user, 283 | "-p" + self.db_password, 284 | "--no-data", 285 | db_name, 286 | ] 287 | try: 288 | output = subprocess.check_output(command) 289 | except subprocess.CalledProcessError as e: 290 | raise DatabaseSchemaExportError(e.output) # noqa: B904 291 | 292 | return output.decode("utf-8") 293 | 294 | def _unlock_tables(self, db_name): 295 | self.get_db(db_name).execute_sql("UNLOCK TABLES;") 296 | self._db_tables_locked[db_name] = False 297 | """ 298 | Anyway, if the db connection gets closed or db thread dies, 299 | the tables will be unlocked automatically 300 | """ 301 | 302 | def get_db(self, db_name: str) -> CustomPeeweeDB: 303 | instance = self._db_instances.get(db_name, None) 304 | if instance is not None: 305 | if not instance.is_connection_usable(): 306 | raise DatabaseConnectionClosedWithDatabase( 307 | f"Database connection closed with database {db_name}" 308 | ) 309 | return instance 310 | if db_name not in self.databases: 311 | raise ValueError(f"Database {db_name} not found") 312 | self._db_instances[db_name] = CustomPeeweeDB( 313 | db_name, 314 | user=self.db_user, 315 | password=self.db_password, 316 | host=self.db_host, 317 | port=self.db_port, 318 | ) 319 | self._db_instances[db_name].connect() 320 | # Set session wait timeout to 4 hours [EXPERIMENTAL] 321 | self._db_instances[db_name].execute_sql("SET SESSION wait_timeout = 14400;") 322 | # Fetch the connection id 323 | self._db_instances_connection_id[db_name] = int( 324 | self._db_instances[db_name].execute_sql("SELECT CONNECTION_ID();").fetchone()[0] 325 | ) 326 | return self._db_instances[db_name] 327 | 328 | def _kill_other_db_connections(self, db_name: str): 329 | kill_other_db_connections(self.get_db(db_name), [self._db_instances_connection_id[db_name]]) 330 | 331 | def __del__(self): 332 | for db_name in self.databases: 333 | if self._db_tables_locked[db_name]: 334 | self._unlock_tables(db_name) 335 | 336 | for db_name in self.databases: 337 | self.get_db(db_name).close() 338 | 339 | 340 | class DatabaseSchemaExportError(Exception): 341 | pass 342 | 343 | 344 | class DatabaseExportFileNotFoundError(Exception): 345 | pass 346 | 347 | 348 | class DatabaseConnectionClosedWithDatabase(Exception): 349 | pass 350 | 351 | 352 | def get_path_of_physical_backup_metadata(db_base_path: str, database_name: str) -> str: 353 | return os.path.join(db_base_path, database_name, "physical_backup_meta.json") 354 | 355 | 356 | def is_db_connection_usable(db: CustomPeeweeDB) -> bool: 357 | try: 358 | if not db.is_connection_usable(): 359 | return False 360 | db.execute_sql("SELECT 1;") 361 | return True 362 | except Exception: 363 | return False 364 | 365 | 366 | def run_sql_query(db: CustomPeeweeDB, query: str) -> list[str]: 367 | """ 368 | Return the result of the query as a list of rows 369 | """ 370 | cursor = db.execute_sql(query) 371 | if not cursor.description: 372 | return [] 373 | rows = cursor.fetchall() 374 | return [row for row in rows] 375 | 376 | 377 | def kill_other_db_connections(db: CustomPeeweeDB, thread_ids: list[int]): 378 | """ 379 | We deactivate site before backup/restore and activate site after backup/restore. 380 | But, connection through ProxySQL or Frappe Cloud devtools can still be there. 381 | 382 | it's important to kill all the connections except current threads. 383 | """ 384 | 385 | # Get process list 386 | thread_ids_str = ",".join([str(thread_id) for thread_id in thread_ids]) 387 | query = ( 388 | "SELECT ID from INFORMATION_SCHEMA.PROCESSLIST " 389 | "where DB=DATABASE() AND USER!='system user' " 390 | f"AND ID NOT IN ({thread_ids_str});" 391 | ) 392 | 393 | rows = run_sql_query(db, query) 394 | db_pids = [row[0] for row in rows] 395 | if not db_pids: 396 | return 397 | 398 | # Kill the processes 399 | for pid in db_pids: 400 | with contextlib.suppress(Exception): 401 | run_sql_query(db, f"KILL {pid};") 402 | -------------------------------------------------------------------------------- /agent/docker_cache_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | # Code below copied mostly verbatim from press, this is tentative and 4 | # will be removed once build code has been moved out of press. 5 | # 6 | # Primary source: 7 | # https://github.com/frappe/press/blob/40859becf2976a3b6a5ac0ff79e2dff8cd2c46af/press/press/doctype/deploy_candidate/cache_utils.py 8 | import os 9 | import platform 10 | import random 11 | import re 12 | import shlex 13 | import shutil 14 | import subprocess 15 | from datetime import datetime 16 | from pathlib import Path 17 | from textwrap import dedent 18 | from typing import TYPE_CHECKING 19 | 20 | if TYPE_CHECKING: 21 | from typing import TypedDict 22 | 23 | class CommandOutput(TypedDict): 24 | cwd: str 25 | image_tag: str 26 | returncode: int 27 | output: str 28 | 29 | 30 | def copy_file_from_docker_cache( 31 | container_source: str, 32 | host_dest: str = ".", 33 | cache_target: str = "/home/frappe/.cache", 34 | ) -> CommandOutput: 35 | """ 36 | Function is used to copy files from docker cache i.e. `cache_target/container_source` 37 | to the host system i.e `host_dest`. 38 | 39 | This function is required cause cache files may be available only during docker build. 40 | 41 | This works by: 42 | - copy the file from mount cache (image) to another_folder (image) 43 | - create a container from image 44 | - copy file from another_folder (container) to host system (using docker cp) 45 | - remove container and then image 46 | """ 47 | filename = Path(container_source).name 48 | container_dest_dirpath = Path(cache_target).parent / "container_dest" 49 | container_dest_filepath = container_dest_dirpath / filename 50 | command = f"mkdir -p {container_dest_dirpath} && " + f"cp {container_source} {container_dest_filepath}" 51 | output = run_command_in_docker_cache( 52 | command, 53 | cache_target, 54 | False, 55 | ) 56 | 57 | if output["returncode"] == 0: 58 | container_id = create_container(output["image_tag"]) 59 | copy_file_from_container( 60 | container_id, 61 | container_dest_filepath, 62 | Path(host_dest), 63 | ) 64 | remove_container(container_id) 65 | 66 | run_image_rm(output["image_tag"]) 67 | return output 68 | 69 | 70 | def run_command_in_docker_cache( 71 | command: str = "ls -A", 72 | cache_target: str = "/home/frappe/.cache", 73 | remove_image: bool = True, 74 | ) -> CommandOutput: 75 | """ 76 | This function works by capturing the output of the given `command` 77 | by running it in the cache dir (`cache_target`) while building a 78 | dummy image. 79 | 80 | The primary purpose is to check the contents of the mounted cache. It's 81 | an incredibly hacky way to achieve this, but afaik the only one. 82 | 83 | Note: The `ARG CACHE_BUST=1` line is used to cause layer cache miss 84 | while running `command` at `cache_target`. This is achieved by changing 85 | `CACHE_BUST` value every run. 86 | 87 | Warning: Takes time to run, use judiciously. 88 | """ 89 | dockerfile = get_cache_check_dockerfile( 90 | command, 91 | cache_target, 92 | ) 93 | df_path = prep_dockerfile_path(dockerfile) 94 | return run_build_command(df_path, remove_image) 95 | 96 | 97 | def get_cache_check_dockerfile(command: str, cache_target: str) -> str: 98 | """ 99 | Note: Mount cache is identified by different attributes, hence it should 100 | be the same as the Dockerfile else it will always result in a cache miss. 101 | 102 | Ref: https://docs.docker.com/engine/reference/builder/#run---mounttypecache 103 | """ 104 | df = f""" 105 | FROM ubuntu:20.04 106 | ARG CACHE_BUST=1 107 | WORKDIR {cache_target} 108 | RUN --mount=type=cache,target={cache_target},uid=1000,gid=1000 {command} 109 | """ 110 | return dedent(df).strip() 111 | 112 | 113 | def create_container(image_tag: str) -> str: 114 | args = shlex.split(f"docker create --platform linux/amd64 {image_tag}") 115 | return subprocess.run( 116 | args, 117 | env=os.environ.copy(), 118 | stdout=subprocess.PIPE, 119 | stderr=subprocess.STDOUT, 120 | text=True, 121 | ).stdout.strip() 122 | 123 | 124 | def copy_file_from_container( 125 | container_id: str, 126 | container_filepath: Path, 127 | host_dest: Path, 128 | ): 129 | container_source = f"{container_id}:{container_filepath}" 130 | args = ["docker", "cp", container_source, host_dest.as_posix()] 131 | proc = subprocess.run( 132 | args, 133 | env=os.environ.copy(), 134 | stdout=subprocess.PIPE, 135 | stderr=subprocess.STDOUT, 136 | text=True, 137 | ) 138 | 139 | if not proc.returncode: 140 | print(f"file copied:\n- from {container_source}\n- to {host_dest.absolute().as_posix()}") 141 | else: 142 | print(proc.stdout) 143 | 144 | 145 | def remove_container(container_id: str) -> str: 146 | args = shlex.split(f"docker rm -v {container_id}") 147 | return subprocess.run( 148 | args, 149 | env=os.environ.copy(), 150 | stdout=subprocess.PIPE, 151 | stderr=subprocess.STDOUT, 152 | text=True, 153 | ).stdout 154 | 155 | 156 | def prep_dockerfile_path(dockerfile: str) -> Path: 157 | dir = Path("cache_check_dockerfile_dir") 158 | if dir.is_dir(): 159 | shutil.rmtree(dir) 160 | 161 | dir.mkdir() 162 | df_path = dir / "Dockerfile" 163 | with open(df_path, "w") as df: 164 | df.write(dockerfile) 165 | 166 | return df_path 167 | 168 | 169 | def run_build_command(df_path: Path, remove_image: bool) -> CommandOutput: 170 | command, image_tag = get_cache_check_build_command() 171 | env = os.environ.copy() 172 | env["DOCKER_BUILDKIT"] = "1" 173 | env["BUILDKIT_PROGRESS"] = "plain" 174 | 175 | output = subprocess.run( 176 | shlex.split(command), 177 | env=env, 178 | cwd=df_path.parent, 179 | stdout=subprocess.PIPE, 180 | stderr=subprocess.STDOUT, 181 | text=True, 182 | ) 183 | if remove_image: 184 | run_image_rm(image_tag) 185 | return dict( 186 | cwd=df_path.parent.absolute().as_posix(), 187 | image_tag=image_tag, 188 | returncode=output.returncode, 189 | output=strip_build_output(output.stdout), 190 | ) 191 | 192 | 193 | def get_cache_check_build_command() -> tuple[str, str]: 194 | command = "docker build" 195 | if platform.machine() == "arm64" and platform.system() == "Darwin" and platform.processor() == "arm": 196 | command += "x build --platform linux/amd64" 197 | 198 | now_ts = datetime.timestamp(datetime.today()) 199 | command += f" --build-arg CACHE_BUST={now_ts}" 200 | 201 | image_tag = f"cache_check:id-{random.getrandbits(40):x}" 202 | command += f" --tag {image_tag} ." 203 | return command, image_tag 204 | 205 | 206 | def run_image_rm(image_tag: str): 207 | command = f"docker image rm {image_tag}" 208 | subprocess.run( 209 | shlex.split(command), 210 | stdout=subprocess.DEVNULL, 211 | stderr=subprocess.DEVNULL, 212 | ) 213 | 214 | 215 | def strip_build_output(stdout: str) -> str: 216 | output = [] 217 | is_output = False 218 | 219 | line_rx = re.compile(r"^#\d+\s\d+\.\d+\s") 220 | done_rx = re.compile(r"^#\d+\sDONE\s\d+\.\d+s$") 221 | 222 | for line in stdout.split("\n"): 223 | if is_output and (m := line_rx.match(line)): 224 | start = m.end() 225 | output.append(line[start:]) 226 | elif is_output and done_rx.search(line): 227 | break 228 | elif "--mount=type=cache,target=" in line: 229 | is_output = True 230 | return "\n".join(output) 231 | 232 | 233 | def get_cached_apps() -> dict[str, list[str]]: 234 | result = run_command_in_docker_cache( 235 | command="ls -A bench/apps", 236 | cache_target="/home/frappe/.cache", 237 | ) 238 | 239 | apps = dict() 240 | if result["returncode"] != 0: 241 | return apps 242 | 243 | for line in result["output"].split("\n"): 244 | # File Name: app_name-cache_key.ext 245 | splits = line.split("-", 1) 246 | if len(splits) != 2: 247 | continue 248 | 249 | app_name, suffix = splits 250 | suffix_splits = suffix.split(".", 1) 251 | if len(suffix_splits) != 2 or suffix_splits[1] not in ["tar", "tgz"]: 252 | continue 253 | 254 | if app_name not in apps: 255 | apps[app_name] = [] 256 | 257 | app_hash = suffix_splits[0] 258 | apps[app_name].append(app_hash) 259 | return apps 260 | -------------------------------------------------------------------------------- /agent/exceptions.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | 4 | class AgentException(Exception): 5 | def __init__(self, data): 6 | self.data = data 7 | 8 | 9 | class BenchNotExistsException(Exception): 10 | def __init__(self, bench): 11 | self.bench = bench 12 | self.message = f"Bench {bench} does not exist" 13 | 14 | super().__init__(self.message) 15 | 16 | 17 | class SiteNotExistsException(Exception): 18 | def __init__(self, site, bench): 19 | self.site = site 20 | self.bench = bench 21 | self.message = f"Site {site} does not exist on bench {bench}" 22 | 23 | super().__init__(self.message) 24 | 25 | 26 | class InvalidSiteConfigException(AgentException): 27 | def __init__(self, data: dict, site=None): 28 | self.site = site 29 | super().__init__(data) 30 | -------------------------------------------------------------------------------- /agent/exporter.py: -------------------------------------------------------------------------------- 1 | from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily 2 | from prometheus_client.registry import Collector 3 | from redis import Redis 4 | from rq import Queue, Worker 5 | from rq.job import JobStatus 6 | 7 | 8 | def get_metrics(name: str, port: int): 9 | from prometheus_client.exposition import generate_latest 10 | 11 | return generate_latest(RQCollector(name, port)) 12 | 13 | 14 | def get_workers_stats(connection: Redis): 15 | workers = Worker.all(connection=connection) 16 | 17 | return [ 18 | { 19 | "name": w.name, 20 | "queues": w.queue_names(), 21 | "state": w.get_state(), 22 | "successful_job_count": w.successful_job_count, 23 | "failed_job_count": w.failed_job_count, 24 | "total_working_time": w.total_working_time, 25 | } 26 | for w in workers 27 | ] 28 | 29 | 30 | def get_jobs_by_queue(connection: Redis): 31 | return { 32 | queue.name: { 33 | JobStatus.QUEUED: queue.count, 34 | JobStatus.STARTED: queue.started_job_registry.count, 35 | JobStatus.FINISHED: queue.finished_job_registry.count, 36 | JobStatus.FAILED: queue.failed_job_registry.count, 37 | JobStatus.DEFERRED: queue.deferred_job_registry.count, 38 | JobStatus.SCHEDULED: queue.scheduled_job_registry.count, 39 | } 40 | for queue in Queue.all(connection=connection) 41 | } 42 | 43 | 44 | class RQCollector(Collector): 45 | def __init__(self, name: str, port: int): 46 | self.name = name 47 | self.port = port 48 | self.conn = Redis(port=port) 49 | super().__init__() 50 | 51 | def collect(self): 52 | rq_workers = GaugeMetricFamily( 53 | "rq_workers", 54 | "RQ workers", 55 | labels=["bench", "name", "state", "queues"], 56 | ) 57 | rq_workers_success = CounterMetricFamily( 58 | "rq_workers_success", 59 | "RQ workers success count", 60 | labels=["bench", "name", "queues"], 61 | ) 62 | rq_workers_failed = CounterMetricFamily( 63 | "rq_workers_failed", 64 | "RQ workers fail count", 65 | labels=["bench", "name", "queues"], 66 | ) 67 | rq_workers_working_time = CounterMetricFamily( 68 | "rq_workers_working_time", 69 | "RQ workers spent seconds", 70 | labels=["bench", "name", "queues"], 71 | ) 72 | 73 | rq_jobs = GaugeMetricFamily("rq_jobs", "RQ jobs by state", labels=["bench", "queue", "status"]) 74 | 75 | workers = get_workers_stats(self.conn) 76 | for worker in workers: 77 | label_queues = ",".join(worker["queues"]) 78 | rq_workers.add_metric( 79 | [ 80 | self.name, 81 | worker["name"], 82 | worker["state"], 83 | label_queues, 84 | ], 85 | 1, 86 | ) 87 | rq_workers_success.add_metric( 88 | [self.name, worker["name"], label_queues], 89 | worker["successful_job_count"], 90 | ) 91 | rq_workers_failed.add_metric( 92 | [self.name, worker["name"], label_queues], 93 | worker["failed_job_count"], 94 | ) 95 | rq_workers_working_time.add_metric( 96 | [self.name, worker["name"], label_queues], 97 | worker["total_working_time"], 98 | ) 99 | 100 | yield rq_workers 101 | yield rq_workers_success 102 | yield rq_workers_failed 103 | yield rq_workers_working_time 104 | 105 | for queue_name, jobs in get_jobs_by_queue(self.conn).items(): 106 | for status, count in jobs.items(): 107 | rq_jobs.add_metric([self.name, queue_name, status], count) 108 | 109 | yield rq_jobs 110 | -------------------------------------------------------------------------------- /agent/job.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import datetime 4 | import json 5 | import os 6 | import traceback 7 | from typing import TYPE_CHECKING 8 | 9 | import wrapt 10 | from peewee import ( 11 | AutoField, 12 | CharField, 13 | DateTimeField, 14 | ForeignKeyField, 15 | Model, 16 | SqliteDatabase, 17 | TextField, 18 | TimeField, 19 | ) 20 | from redis import Redis 21 | from rq import Queue, get_current_job 22 | from rq.command import send_stop_job_command 23 | from rq.job import Job as RQJob 24 | 25 | from agent.callbacks import callback 26 | 27 | if TYPE_CHECKING: 28 | from agent.base import Base 29 | 30 | 31 | if os.environ.get("SENTRY_DSN"): 32 | try: 33 | import sentry_sdk 34 | 35 | sentry_sdk.init(dsn=os.environ["SENTRY_DSN"]) 36 | except ImportError: 37 | pass 38 | 39 | 40 | agent_database = SqliteDatabase( 41 | "jobs.sqlite3", 42 | timeout=15, 43 | pragmas={ 44 | "journal_mode": "wal", 45 | "synchronous": "normal", 46 | "mmap_size": 2**32 - 1, 47 | "page_size": 8192, 48 | }, 49 | ) 50 | 51 | 52 | def connection(): 53 | from agent.server import Server 54 | 55 | port = Server().config["redis_port"] 56 | return Redis(port=port) 57 | 58 | 59 | def queue(name): 60 | return Queue(name, connection=connection()) 61 | 62 | 63 | @wrapt.decorator 64 | def save(wrapped, instance: Action, args, kwargs): 65 | wrapped(*args, **kwargs) 66 | instance.model.save() 67 | 68 | 69 | class Action: 70 | if TYPE_CHECKING: 71 | model: Model | None 72 | 73 | def success(self, data): 74 | self.model.status = "Success" 75 | self.model.data = json.dumps(data, default=str) 76 | self.end() 77 | 78 | def failure(self, data): 79 | self.model.data = json.dumps(data, default=str) 80 | self.model.status = "Failure" 81 | self.end() 82 | 83 | @save 84 | def end(self): 85 | self.model.end = datetime.datetime.now() 86 | self.model.duration = self.model.end - self.model.start 87 | 88 | 89 | class Step(Action): 90 | if TYPE_CHECKING: 91 | model: StepModel | None 92 | 93 | @save 94 | def start(self, name, job): 95 | self.model = StepModel() 96 | self.model.name = name 97 | self.model.job = job 98 | self.model.start = datetime.datetime.now() 99 | self.model.status = "Running" 100 | 101 | 102 | class Job(Action): 103 | if TYPE_CHECKING: 104 | model: JobModel | None 105 | 106 | def __init__(self, id=None): 107 | super().__init__() 108 | if id: 109 | self.model = JobModel.get(JobModel.id == id) 110 | self.redis = connection() 111 | self.job = RQJob.fetch(str(self.model.id), connection=self.redis) 112 | 113 | @save 114 | def start(self): 115 | self.model.start = datetime.datetime.now() 116 | self.model.status = "Running" 117 | 118 | @save 119 | def enqueue(self, name, function, args, kwargs, agent_job_id=None): 120 | self.model = JobModel() 121 | self.model.name = name 122 | self.model.status = "Pending" 123 | self.model.enqueue = datetime.datetime.now() 124 | self.model.data = json.dumps( 125 | { 126 | "function": function.__func__.__name__, 127 | "args": args, 128 | "kwargs": kwargs, 129 | }, 130 | default=str, 131 | sort_keys=True, 132 | indent=4, 133 | ) 134 | self.model.agent_job_id = agent_job_id 135 | 136 | @save 137 | def cancel(self): 138 | self.job.cancel() 139 | self.model.status = "Failure" 140 | 141 | @save 142 | def stop(self): 143 | send_stop_job_command(self.redis, self.job.get_id()) 144 | self.job.refresh() 145 | self.model.data = json.dumps(self.job.to_dict(), default=str) 146 | self.model.status = "Failure" 147 | self.end() 148 | 149 | def cancel_or_stop(self): 150 | if self.job.is_started: 151 | self.stop() 152 | else: 153 | self.cancel() 154 | 155 | 156 | def step(name): 157 | @wrapt.decorator 158 | def wrapper(wrapped, instance: Base, args, kwargs): 159 | from agent.base import AgentException 160 | 161 | instance.step_record.start(name, instance.job_record.model.id) 162 | try: 163 | result = wrapped(*args, **kwargs) 164 | except AgentException as e: 165 | instance.step_record.failure(e.data) 166 | raise e 167 | except Exception as e: 168 | instance.step_record.failure({"traceback": "".join(traceback.format_exc())}) 169 | raise e 170 | else: 171 | instance.step_record.success(result) 172 | finally: 173 | instance.step_record = None 174 | return result 175 | 176 | return wrapper 177 | 178 | 179 | def job(name: str, priority="default", on_success=None, on_failure=None): 180 | @wrapt.decorator 181 | def wrapper(wrapped, instance: Base, args, kwargs): 182 | from agent.base import AgentException 183 | 184 | if get_current_job(connection=connection()): 185 | instance.job_record.start() 186 | try: 187 | result = wrapped(*args, **kwargs) 188 | except AgentException as e: 189 | instance.job_record.failure(e.data) 190 | raise e 191 | except Exception as e: 192 | instance.job_record.failure({"traceback": "".join(traceback.format_exc())}) 193 | raise e 194 | else: 195 | instance.job_record.success(result) 196 | return result 197 | agent_job_id = get_agent_job_id() 198 | instance.job_record.enqueue(name, wrapped, args, kwargs, agent_job_id) 199 | queue(priority).enqueue_call( 200 | wrapped, 201 | args=args, 202 | kwargs=kwargs, 203 | timeout=4 * 3600, 204 | result_ttl=24 * 3600, 205 | job_id=str(instance.job_record.model.id), 206 | on_success=on_success or callback, 207 | on_failure=on_failure or callback, 208 | ) 209 | return instance.job_record.model.id 210 | 211 | return wrapper 212 | 213 | 214 | def get_agent_job_id(): 215 | from flask import request 216 | 217 | return request.headers.get("X-Agent-Job-Id") 218 | 219 | 220 | class JobModel(Model): 221 | name = CharField() 222 | status = CharField( 223 | choices=[ 224 | (0, "Pending"), 225 | (1, "Running"), 226 | (2, "Success"), 227 | (3, "Failure"), 228 | ] 229 | ) 230 | agent_job_id = CharField(null=True) 231 | data = TextField(null=True, default="{}") 232 | 233 | enqueue = DateTimeField(default=datetime.datetime.now) 234 | 235 | start = DateTimeField(null=True) 236 | end = DateTimeField(null=True) 237 | duration = TimeField(null=True) 238 | 239 | class Meta: 240 | database = agent_database 241 | 242 | 243 | class StepModel(Model): 244 | name = CharField() 245 | job = ForeignKeyField(JobModel, backref="steps", lazy_load=False) 246 | status = CharField(choices=[(1, "Running"), (2, "Success"), (3, "Failure")]) 247 | data = TextField(null=True, default="{}") 248 | 249 | start = DateTimeField() 250 | end = DateTimeField(null=True) 251 | duration = TimeField(null=True) 252 | 253 | class Meta: 254 | database = agent_database 255 | 256 | 257 | class PatchLogModel(Model): 258 | name = AutoField() 259 | patch = TextField() 260 | 261 | class Meta: 262 | database = agent_database 263 | -------------------------------------------------------------------------------- /agent/minio.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | 5 | from agent.job import job, step 6 | from agent.server import Server 7 | 8 | 9 | class Minio(Server): 10 | def __init__(self, directory=None): 11 | super().__init__(directory=directory) 12 | 13 | self.directory = directory or os.getcwd() 14 | self.policy_path = "/home/frappe/minio/tmp_policy.json" 15 | self.host = "localhost" 16 | self.job = None 17 | self.step = None 18 | 19 | @job("Create Minio User") 20 | def create_subscription(self, access_key, secret_key, policy_name, policy_json): 21 | self.create_user(access_key, secret_key) 22 | self.create_policy(policy_name, policy_json) 23 | self.add_policy(access_key, policy_name) 24 | 25 | @step("Create Minio User") 26 | def create_user(self, access_key, secret_key): 27 | # access_key = username on minio 28 | self.execute(f"mc admin user add {self.host} {access_key} {secret_key}") 29 | 30 | @step("Create Minio Policy") 31 | def create_policy(self, policy_name, policy_json): 32 | self.execute(f"echo '{policy_json}' > {self.policy_path}") 33 | self.execute(f"mc admin policy add {self.host} {policy_name} {self.policy_path}") 34 | 35 | @step("Add Minio Policy") 36 | def add_policy(self, access_key, policy_name): 37 | self.execute(f"mc admin policy set {self.host} {policy_name} user={access_key}") 38 | 39 | @job("Disable Minio User") 40 | def disable_user(self, username): 41 | self.disable(username) 42 | 43 | @step("Disable Minio User") 44 | def disable(self, username): 45 | self.execute(f"mc admin user disable {self.host} {username}") 46 | 47 | @job("Enable Minio User") 48 | def enable_user(self, username): 49 | self.enable(username) 50 | 51 | @step("Enable Minio User") 52 | def enable(self, username): 53 | self.execute(f"mc admin user enable {self.host} {username}") 54 | 55 | @job("Remove Minio User") 56 | def remove_user(self, username): 57 | self.remove(username) 58 | 59 | @step("Remove Minio User") 60 | def remove(self, username): 61 | self.execute(f"mc admin user remove {self.host} {username}") 62 | -------------------------------------------------------------------------------- /agent/monitor.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | import tempfile 5 | 6 | import requests 7 | 8 | from agent.server import Server 9 | 10 | 11 | class Monitor(Server): 12 | def __init__(self, directory=None): 13 | super().__init__(directory=directory) 14 | self.prometheus_directory = "/home/frappe/prometheus" 15 | self.alertmanager_directory = "/home/frappe/alertmanager" 16 | 17 | def update_rules(self, rules): 18 | rules_file = os.path.join(self.prometheus_directory, "rules", "agent.yml") 19 | self._render_template( 20 | "prometheus/rules.yml", 21 | {"rules": rules}, 22 | rules_file, 23 | { 24 | "variable_start_string": "###", 25 | "variable_end_string": "###", 26 | }, 27 | ) 28 | 29 | promtool = os.path.join(self.prometheus_directory, "promtool") 30 | self.execute(f"{promtool} check rules {rules_file}") 31 | 32 | self.execute("sudo systemctl reload prometheus") 33 | 34 | def update_routes(self, routes): 35 | config_file = os.path.join(self.alertmanager_directory, "alertmanager.yml") 36 | self._render_template( 37 | "alertmanager/routes.yml", 38 | {"routes": routes}, 39 | config_file, 40 | { 41 | "variable_start_string": "###", 42 | "variable_end_string": "###", 43 | }, 44 | ) 45 | amtool = os.path.join(self.alertmanager_directory, "amtool") 46 | self.execute(f"{amtool} check-config {config_file}") 47 | 48 | self.execute("sudo systemctl reload alertmanager") 49 | 50 | def discover_targets(self): 51 | targets = self.fetch_targets() 52 | for cluster in targets["clusters"]: 53 | self.generate_prometheus_cluster_config(cluster) 54 | 55 | self.generate_prometheus_tls_config(targets["tls"]) 56 | self.generate_prometheus_sites_config(targets["benches"]) 57 | self.generate_prometheus_domains_config(targets["domains"]) 58 | 59 | def fetch_targets(self): 60 | press_url = self.config.get("press_url") 61 | press_token = self.config.get("press_token") 62 | return requests.post( 63 | f"{press_url}/api/method/press.api.monitoring.targets", 64 | data={"token": press_token}, 65 | ).json()["message"] 66 | 67 | def generate_prometheus_sites_config(self, benches): 68 | prometheus_sites_config = os.path.join(self.prometheus_directory, "file_sd", "sites.yml") 69 | temp_sites_config = tempfile.mkstemp(prefix="agent-prometheus-sites-", suffix=".yml")[1] 70 | self._render_template( 71 | "prometheus/sites.yml", 72 | {"benches": benches}, 73 | temp_sites_config, 74 | {"block_start_string": "##", "block_end_string": "##"}, 75 | ) 76 | os.rename(temp_sites_config, prometheus_sites_config) 77 | 78 | def generate_prometheus_tls_config(self, servers): 79 | prometheus_tls_config = os.path.join(self.prometheus_directory, "file_sd", "tls.yml") 80 | temp_tls_config = tempfile.mkstemp(prefix="agent-prometheus-tls-", suffix=".yml")[1] 81 | self._render_template( 82 | "prometheus/tls.yml", 83 | {"servers": servers}, 84 | temp_tls_config, 85 | {"block_start_string": "##", "block_end_string": "##"}, 86 | ) 87 | os.rename(temp_tls_config, prometheus_tls_config) 88 | 89 | def generate_prometheus_domains_config(self, domains): 90 | prometheus_domains_config = os.path.join(self.prometheus_directory, "file_sd", "domains.yml") 91 | temp_domains_config = tempfile.mkstemp(prefix="agent-prometheus-domains-", suffix=".yml")[1] 92 | self._render_template( 93 | "prometheus/domains.yml", 94 | {"domains": domains}, 95 | temp_domains_config, 96 | {"block_start_string": "##", "block_end_string": "##"}, 97 | ) 98 | os.rename(temp_domains_config, prometheus_domains_config) 99 | 100 | def generate_prometheus_cluster_config(self, cluster): 101 | prometheus_cluster_config = os.path.join( 102 | self.prometheus_directory, 103 | "file_sd", 104 | f"cluster.{cluster['name']}.yml", 105 | ) 106 | 107 | temp_cluster_config = tempfile.mkstemp(prefix="agent-prometheus-cluster-", suffix=".yml")[1] 108 | self._render_template( 109 | "prometheus/servers.yml", 110 | {"cluster": cluster}, 111 | temp_cluster_config, 112 | {"block_start_string": "##", "block_end_string": "##"}, 113 | ) 114 | os.rename(temp_cluster_config, prometheus_cluster_config) 115 | -------------------------------------------------------------------------------- /agent/nginx_reload_manager.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import contextlib 4 | import glob 5 | import json 6 | import os 7 | import re 8 | import signal 9 | import subprocess 10 | import time 11 | import traceback 12 | from datetime import datetime 13 | from enum import Enum, auto 14 | 15 | import redis 16 | 17 | PENDING_QUEUE = "nginx||pending_queue" 18 | PROCESSING_QUEUE = "nginx||processing_queue" 19 | RELOAD_REQUEST_STATUS_FORMAT = "nginx_reload_status||{}" 20 | 21 | 22 | class ReloadStatus(Enum): 23 | Queued = "Queued" 24 | Success = "Success" 25 | Failure = "Failure" 26 | Skipped = "Skipped" 27 | 28 | 29 | class ManagerState(Enum): 30 | INIT = auto() 31 | FETCH_JOBS = auto() 32 | RELOAD_PENDING = auto() 33 | RELOAD_SUCCESS = auto() 34 | RELOAD_FAILURE = auto() 35 | AUTO_FIX_CONFIG = auto() 36 | WAIT = auto() 37 | 38 | 39 | class NginxReloadManager: 40 | redis_instance = None 41 | batch_size = 1000 42 | 43 | def __init__(self, directory=None, debug=False): 44 | self.directory = directory or os.getcwd() 45 | self.config_file = os.path.join(self.directory, "config.json") 46 | self.last_reload_at: datetime = None 47 | self.exit_requested = False 48 | self.debug = debug 49 | self.state = ManagerState.INIT 50 | 51 | def request_reload(self, request_id: str): 52 | self.redis.rpush(PENDING_QUEUE, request_id) 53 | self.redis.set(RELOAD_REQUEST_STATUS_FORMAT.format(request_id), ReloadStatus.Queued.value) 54 | 55 | def get_status(self, request_id: str, not_found_status: ReloadStatus) -> ReloadStatus | None: 56 | status = self.redis.get(RELOAD_REQUEST_STATUS_FORMAT.format(request_id)) 57 | if status is None: 58 | return not_found_status 59 | return ReloadStatus(status) 60 | 61 | def process_requests(self): 62 | # Handle Signals 63 | signal.signal(signal.SIGINT, self.exit_gracefully) 64 | signal.signal(signal.SIGTERM, self.exit_gracefully) 65 | 66 | self.last_reload_at = datetime.now() 67 | self.start_time = datetime.now() 68 | self.state: ReloadStatus = ManagerState.FETCH_JOBS 69 | self.job_ids = [] 70 | self.error = None 71 | 72 | while not self.exit_requested: 73 | try: 74 | self._process_state() 75 | except Exception: 76 | traceback.print_exc() 77 | self.state = ManagerState.WAIT 78 | 79 | def _process_state(self): # noqa: C901 80 | if self.state == ManagerState.FETCH_JOBS: 81 | self.start_time = datetime.now() 82 | self.job_ids = self._dequeue_jobs() 83 | if self.job_ids or self.is_mandatory_reload_required: 84 | self.state = ManagerState.RELOAD_PENDING 85 | return 86 | 87 | self.state = ManagerState.WAIT 88 | 89 | elif self.state == ManagerState.RELOAD_PENDING: 90 | if self._should_skip_nginx_reload(): 91 | self.state = ManagerState.WAIT 92 | return 93 | 94 | status = self._reload_nginx() 95 | self.log(f"Reload {status.name} | {len(self.job_ids)} requests", print_always=True) 96 | self.state = ( 97 | ManagerState.RELOAD_SUCCESS 98 | if status == ReloadStatus.Success 99 | else ManagerState.AUTO_FIX_CONFIG 100 | ) 101 | 102 | elif self.state == ManagerState.RELOAD_SUCCESS: 103 | self._update_status_and_cleanup(self.job_ids, ReloadStatus.Success) 104 | self.last_reload_at = datetime.now() 105 | self.state = ManagerState.WAIT 106 | 107 | elif self.state == ManagerState.RELOAD_FAILURE: 108 | self._update_status_and_cleanup(self.job_ids, ReloadStatus.Failure) 109 | self.state = ManagerState.WAIT 110 | 111 | elif self.state == ManagerState.AUTO_FIX_CONFIG: 112 | if isinstance(self.error, subprocess.CalledProcessError): 113 | error_msg = self.error.stderr or "" 114 | domain = self._find_conflicting_domain_from_error_message(error_msg) 115 | if domain and self._fix_conflicting_domain_in_config(domain): 116 | self.state = ManagerState.RELOAD_PENDING 117 | return 118 | 119 | self.state = ManagerState.RELOAD_FAILURE 120 | 121 | elif self.state == ManagerState.WAIT: 122 | elapsed_time = datetime.now() - self.start_time 123 | if (sleep_time := self.max_permissible_wait_time - elapsed_time.total_seconds()) > 0: 124 | self.log(f"Waiting for {sleep_time:.2f} seconds before next check") 125 | time.sleep(sleep_time) 126 | 127 | self.state = ManagerState.FETCH_JOBS 128 | 129 | # Private methods 130 | def _reload_nginx(self) -> ReloadStatus: 131 | from agent.proxy import Proxy 132 | 133 | try: 134 | proxy = Proxy() 135 | 136 | proxy._generate_proxy_config() 137 | subprocess.run( 138 | "sudo nginx -s reload", 139 | shell=True, 140 | check=True, 141 | capture_output=True, 142 | text=True, 143 | ) 144 | 145 | self.last_reload_at = datetime.now() 146 | return ReloadStatus.Success 147 | except Exception as e: 148 | self.error = e 149 | 150 | traceback.print_exc() 151 | self.log(f"Error while reloading nginx : {e!s}") 152 | return ReloadStatus.Failure 153 | 154 | def _should_skip_nginx_reload(self): 155 | """ 156 | ├─983331 nginx: worker process is shutting down 157 | ├─983332 nginx: worker process is shutting down 158 | ├─983333 nginx: worker process is shutting down 159 | ├─983334 nginx: worker process is shutting down 160 | ├─983361 nginx: worker process 161 | ├─983362 nginx: worker process 162 | ├─983363 nginx: worker process 163 | ├─983364 nginx: worker process 164 | └─983369 nginx: cache manager process 165 | """ 166 | try: 167 | status = subprocess.run( 168 | "sudo systemctl status nginx", capture_output=True, shell=True 169 | ).stdout.decode("utf-8") 170 | total_workers = status.count("nginx: worker process") 171 | dying_workers = status.count("nginx: worker process is shutting down") 172 | active_workers = max(1, total_workers - dying_workers) 173 | return dying_workers >= active_workers 174 | except Exception as e: 175 | self.log(f"Failed to check nginx status : {e!s}") 176 | return False 177 | 178 | def _find_conflicting_domain_from_error_message(self, error_message: str) -> str | None: 179 | match = re.search(r'conflicting parameter "(.*?)"', error_message) 180 | if not match: 181 | return None 182 | return match.group(1) 183 | 184 | def _fix_conflicting_domain_in_config(self, domain: str) -> bool: 185 | self.log(f"Domain {domain} is conflicting, attempting auto fix", print_always=True) 186 | 187 | # Find the paths 188 | paths = glob.glob(os.path.join(self.directory, f"nginx/upstreams/*/{domain}")) 189 | paths.sort(key=lambda x: os.path.getmtime(x)) 190 | 191 | if len(paths) < 2: 192 | self.log(f"Not able to auto fix the issue for domain : {domain!s}", print_always=True) 193 | return False 194 | 195 | # Keep newest file and delete the rest 196 | to_keep = paths[-1] 197 | self.log(f"Keeping {to_keep}", print_always=True) 198 | to_delete = paths[:-1] 199 | for path in to_delete: 200 | with contextlib.suppress(FileNotFoundError): 201 | # In case RQ worker removed the file 202 | # before we could delete it 203 | os.remove(path) 204 | self.log(f"Deleted {path}", print_always=True) 205 | 206 | return True 207 | 208 | def _dequeue_jobs(self) -> list[str]: 209 | # Fetch up to 10000 jobs, which might not be hit ever 210 | job_ids = self.redis.lrange(PENDING_QUEUE, 0, self.batch_size) 211 | if job_ids: 212 | # Trim pending queue 213 | self.redis.ltrim(PENDING_QUEUE, len(job_ids), -1) 214 | # Move jobs to processing queue 215 | self.redis.rpush(PROCESSING_QUEUE, *job_ids) 216 | 217 | # Fetch job IDs from processing queue 218 | # Important to fetch from processing queue to ensure than if worker died previously, 219 | # old jobs in processing queue are still processed 220 | jobs = self.redis.lrange(PROCESSING_QUEUE, 0, self.batch_size) 221 | self.log(f"Dequeued {len(job_ids)} jobs") 222 | return jobs 223 | 224 | def _update_status_and_cleanup(self, job_ids: list[str], status: ReloadStatus): 225 | for job_id in job_ids: 226 | self.redis.set(RELOAD_REQUEST_STATUS_FORMAT.format(job_id), status.value) 227 | self.redis.ltrim(PROCESSING_QUEUE, len(job_ids), -1) 228 | 229 | def exit_gracefully(self, signum, frame): 230 | if self.exit_requested: 231 | self.log("Stopping forefully") 232 | exit(0) 233 | 234 | self.exit_requested = True 235 | self.log("Requested to exit gracefully") 236 | 237 | def log(self, message, print_always=False): 238 | if not self.debug and not print_always: 239 | return 240 | print(f"[{datetime.now()}] {message!s}") 241 | 242 | # Properties 243 | @property 244 | def redis(self): 245 | if not self.redis_instance: 246 | self.redis_instance = redis.Redis( 247 | port=self.config.get("redis_port", 25025), 248 | decode_responses=True, 249 | ) 250 | return self.redis_instance 251 | 252 | @property 253 | def config(self) -> dict: 254 | if not hasattr(self, "_config"): 255 | with open(self.config_file, "r") as f: 256 | self._config = json.load(f) 257 | return self._config 258 | 259 | @property 260 | def max_permissible_wait_time(self) -> float: 261 | return 60 / self.max_reloads_per_minute 262 | 263 | @property 264 | def max_reloads_per_minute(self) -> int: 265 | return self.config.get("max_reloads_per_minute", 30) 266 | 267 | @property 268 | def max_interval_without_reload_minutes(self) -> int: 269 | """Maximum allowed minutes without a reload before forcing a voluntary reload.""" 270 | return self.config.get("max_interval_without_reload_minutes", 10) 271 | 272 | @property 273 | def is_mandatory_reload_required(self) -> bool: 274 | """Check if a mandatory reload is required.""" 275 | return ( 276 | self.last_reload_at 277 | and (datetime.now() - self.last_reload_at).total_seconds() 278 | >= self.max_interval_without_reload_minutes * 60 279 | ) 280 | 281 | 282 | if __name__ == "__main__": 283 | manager = NginxReloadManager() 284 | manager.process_requests() 285 | -------------------------------------------------------------------------------- /agent/pages/deactivated.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | This Site is Inactive 6 | 75 | 76 | 77 | 78 |
79 | 99 |
100 |
101 | 102 | 105 | 106 |
107 |

108 | This Site is Inactive. 109 |

110 |

111 | This site has been deactivated. If you are the owner of this site, you can activate it from your 112 | Frappe Cloud Dashboard. 113 |

114 |
115 |
116 | 117 | 118 | 123 | 124 | -------------------------------------------------------------------------------- /agent/pages/exceeded.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Daily Usage Limit Reached 7 | 76 | 77 | 78 | 79 |
80 | 100 |
101 |
102 | 103 | 106 | 107 |
108 |

109 | Daily Usage Limit Reached. 110 |

111 |

112 | Daily usage limit is reached for this site. If you are the owner of this site, you can upgrade your plan 113 | from your Frappe Cloud Dashboard. 114 |

115 |
116 |
117 | 118 | 119 | 124 | 125 | -------------------------------------------------------------------------------- /agent/pages/home.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Frappe Cloud 6 | 74 | 75 | 76 | 77 |
78 | 98 |
99 |
100 | 101 | 104 | 105 |
106 |

107 | Are you lost? 108 |

109 |

110 | Why don't you start over from Frappe Cloud. 111 |

112 |
113 |
114 | 115 | 116 | -------------------------------------------------------------------------------- /agent/pages/suspended.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | This Site is Suspended 6 | 74 | 75 | 76 | 77 |
78 | 98 |
99 |
100 | 101 | 104 | 105 |
106 |

107 | This Site is Suspended. 108 |

109 |

110 | This site has been suspended due to exceeding site limits or a payment failure. If you are the owner 111 | of this site, resolve issues related to your site's plan from the Frappe Cloud Dashboard. 112 |

113 |
114 |
115 | 116 | 117 | 122 | 123 | -------------------------------------------------------------------------------- /agent/pages/suspended_saas.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | This Site is Suspended 6 | 65 | 66 | 67 | 68 |
69 |
70 |
71 | 72 | 75 | 76 |
77 |

78 | This Site is Suspended. 79 |

80 |

81 | This site has been suspended due to exceeding site limits or a payment failure. If you are the owner 82 | of this site, resolve issues related to your site's plan from the Dashboard. 83 |

84 |
85 |
86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /agent/patch_handler.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import importlib 4 | import os 5 | 6 | 7 | class PatchHandler: 8 | def __init__(self, patch=None, path=None): 9 | self.patch = patch 10 | self.path = path 11 | self._executed_patches = set() 12 | 13 | @property 14 | def executed_patches(self): 15 | if not self._executed_patches: 16 | self._executed_patches = set(self.retrieve_patches()) 17 | return self._executed_patches 18 | 19 | def retrieve_patches(self): 20 | from agent.job import PatchLogModel 21 | 22 | patches = PatchLogModel.select() 23 | return [patch.patch for patch in patches] 24 | 25 | def execute(self): 26 | if self.patch not in self.executed_patches: 27 | print("Executing patch", self.patch) 28 | try: 29 | self.get_method()() 30 | except Exception as e: 31 | print("Failed to execute patch", self.patch) 32 | raise e 33 | else: 34 | self.log_patch() 35 | 36 | def get_method(self, attr="execute"): 37 | _patch = self.patch.split(maxsplit=1)[0] 38 | module = importlib.import_module(_patch) 39 | return getattr(module, attr) 40 | 41 | def log_patch(self): 42 | from agent.job import PatchLogModel 43 | 44 | patch_log = PatchLogModel() 45 | patch_log.patch = self.patch 46 | patch_log.save() 47 | 48 | 49 | def run_patches(): 50 | directory = os.getcwd() 51 | patches_dir = f"{directory}/repo/agent/patches.txt" 52 | 53 | if not _patch_log_exists(): 54 | print("Creating patch log") 55 | _create_patch_log() 56 | 57 | with open(patches_dir, "r") as f: 58 | patches = f.readlines() 59 | for patch in patches: 60 | patch = patch.strip() 61 | patch_path = f"{directory}/patches/{patch}" 62 | 63 | patch_handler = PatchHandler(patch=patch, path=patch_path) 64 | patch_handler.execute() 65 | 66 | 67 | def _patch_log_exists(): 68 | from agent.job import agent_database as database 69 | 70 | tables = database.get_tables() 71 | return "patchlogmodel" in tables 72 | 73 | 74 | def _create_patch_log(): 75 | from agent.job import PatchLogModel 76 | 77 | PatchLogModel.create_table() 78 | -------------------------------------------------------------------------------- /agent/patches.txt: -------------------------------------------------------------------------------- 1 | agent.patches.add_agent_id_field 2 | agent.patches.add_index_on_agent_job_id_field 3 | agent.patches.add_index_on_jobmodel_id_field 4 | agent.patches.create_binlog_index_folder -------------------------------------------------------------------------------- /agent/patches/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | -------------------------------------------------------------------------------- /agent/patches/add_agent_id_field.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | 4 | def execute(): 5 | """add a new fc_agent_job_id field to JobModel""" 6 | from peewee import CharField 7 | from playhouse.migrate import SqliteMigrator, migrate 8 | 9 | from agent.job import agent_database as database 10 | 11 | migrator = SqliteMigrator(database) 12 | try: 13 | migrate(migrator.add_column("JobModel", "agent_job_id", CharField(null=True))) 14 | except Exception as e: 15 | print(e) 16 | -------------------------------------------------------------------------------- /agent/patches/add_index_on_agent_job_id_field.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | 4 | def execute(): 5 | """add index on agent_job_id field of JobModel""" 6 | from agent.job import agent_database as database 7 | 8 | database.execute_sql("CREATE INDEX IF NOT EXISTS idx_jobmodel_agent_job_id ON jobmodel (agent_job_id)") 9 | -------------------------------------------------------------------------------- /agent/patches/add_index_on_jobmodel_id_field.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | 4 | def execute(): 5 | """add index on id field of JobModel""" 6 | from agent.job import agent_database as database 7 | 8 | database.execute_sql("CREATE INDEX IF NOT EXISTS idx_jobmodel_id ON jobmodel (id)") 9 | -------------------------------------------------------------------------------- /agent/patches/create_binlog_index_folder.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | 5 | 6 | def execute(): 7 | """create the folder to store binlog indexes""" 8 | path = os.path.join(os.getcwd(), "binlog-indexes") 9 | 10 | if not os.path.exists(path): 11 | os.mkdir(path) 12 | -------------------------------------------------------------------------------- /agent/proxy.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | import os 5 | import shutil 6 | from collections import defaultdict 7 | from contextlib import contextmanager 8 | from functools import wraps 9 | from hashlib import sha512 as sha 10 | from pathlib import Path 11 | 12 | import filelock 13 | 14 | from agent.job import job, step 15 | from agent.server import Server 16 | 17 | 18 | def with_proxy_config_lock(): 19 | def decorator(func): 20 | @wraps(func) 21 | def wrapper(self, *args, **kwargs): 22 | with self.proxy_config_modification_lock: 23 | return func(self, *args, **kwargs) 24 | 25 | return wrapper 26 | 27 | return decorator 28 | 29 | 30 | class Proxy(Server): 31 | def __init__(self, directory=None): 32 | super().__init__(directory) 33 | self.directory = directory or os.getcwd() 34 | self.config_file = os.path.join(self.directory, "config.json") 35 | self.name = self.config["name"] 36 | self.domain = self.config.get("domain") 37 | self.nginx_directory = self.config["nginx_directory"] 38 | self.upstreams_directory = os.path.join(self.nginx_directory, "upstreams") 39 | self.hosts_directory = os.path.join(self.nginx_directory, "hosts") 40 | self.error_pages_directory = os.path.join(self.directory, "repo", "agent", "pages") 41 | self._proxy_config_modification_lock = None 42 | self.job = None 43 | self.step = None 44 | 45 | def setup_proxy(self): 46 | self._create_default_host() 47 | self._generate_proxy_config() 48 | self.execute("sudo systemctl reload nginx") 49 | 50 | @job("Add Host to Proxy") 51 | def add_host_job(self, host, target, certificate): 52 | self.add_host(host, target, certificate) 53 | self.reload_nginx() 54 | 55 | @step("Add Host to Proxy") 56 | @with_proxy_config_lock() 57 | def add_host(self, host, target, certificate): 58 | host_directory = os.path.join(self.hosts_directory, host) 59 | os.makedirs(host_directory, exist_ok=True) 60 | 61 | map_file = os.path.join(host_directory, "map.json") 62 | with open(map_file, "w") as m: 63 | json.dump({host: target}, m, indent=4) 64 | 65 | for key, value in certificate.items(): 66 | with open(os.path.join(host_directory, key), "w") as f: 67 | f.write(value) 68 | 69 | @job("Add Wildcard Hosts to Proxy") 70 | def add_wildcard_hosts_job(self, wildcards): 71 | self.add_wildcard_hosts(wildcards) 72 | self.reload_nginx() 73 | 74 | @step("Add Wildcard Hosts to Proxy") 75 | @with_proxy_config_lock() 76 | def add_wildcard_hosts(self, wildcards): 77 | for wildcard in wildcards: 78 | host = f"*.{wildcard['domain']}" 79 | host_directory = os.path.join(self.hosts_directory, host) 80 | os.makedirs(host_directory, exist_ok=True) 81 | 82 | map_file = os.path.join(host_directory, "map.json") 83 | with open(map_file, "w") as m: 84 | json.dump({host: "$host"}, m, indent=4) 85 | 86 | for key, value in wildcard["certificate"].items(): 87 | with open(os.path.join(host_directory, key), "w") as f: 88 | f.write(value) 89 | if wildcard.get("code_server"): 90 | Path(os.path.join(host_directory, "codeserver")).touch() 91 | 92 | def add_site_domain_to_upstream(self, upstream, site): 93 | with self.proxy_config_modification_lock: 94 | self.remove_conflicting_site(site) 95 | self.add_site_to_upstream(upstream, site) 96 | self.reload_nginx() 97 | 98 | @job("Add Site to Upstream") 99 | def add_site_to_upstream_job(self, upstream, site): 100 | self.add_site_domain_to_upstream(upstream, site) 101 | 102 | @job("Add Domain to Upstream") 103 | def add_domain_to_upstream_job(self, upstream, domain): 104 | self.add_site_domain_to_upstream(upstream, domain) 105 | 106 | @step("Remove Conflicting Site") 107 | @with_proxy_config_lock() 108 | def remove_conflicting_site(self, site): 109 | # Go through all upstreams and remove the site file matching the site name 110 | for upstream in self.upstreams: 111 | conflict = os.path.join(self.upstreams_directory, upstream, site) 112 | if os.path.exists(conflict): 113 | os.remove(conflict) 114 | 115 | @step("Add Site File to Upstream Directory") 116 | @with_proxy_config_lock() 117 | def add_site_to_upstream(self, upstream, site): 118 | upstream_directory = os.path.join(self.upstreams_directory, upstream) 119 | os.makedirs(upstream_directory, exist_ok=True) 120 | site_file = os.path.join(upstream_directory, site) 121 | Path(site_file).touch() 122 | 123 | @job("Add Upstream to Proxy") 124 | def add_upstream_job(self, upstream): 125 | self.add_upstream(upstream) 126 | self.reload_nginx() 127 | 128 | @step("Add Upstream Directory") 129 | @with_proxy_config_lock() 130 | def add_upstream(self, upstream): 131 | upstream_directory = os.path.join(self.upstreams_directory, upstream) 132 | os.makedirs(upstream_directory, exist_ok=True) 133 | 134 | @job("Rename Upstream") 135 | def rename_upstream_job(self, old, new): 136 | self.rename_upstream(old, new) 137 | self.reload_nginx() 138 | 139 | @step("Rename Upstream Directory") 140 | @with_proxy_config_lock() 141 | def rename_upstream(self, old, new): 142 | old_upstream_directory = os.path.join(self.upstreams_directory, old) 143 | new_upstream_directory = os.path.join(self.upstreams_directory, new) 144 | shutil.move(old_upstream_directory, new_upstream_directory) 145 | 146 | @job("Remove Host from Proxy") 147 | def remove_host_job(self, host): 148 | self.remove_host(host) 149 | 150 | @step("Remove Host from Proxy") 151 | @with_proxy_config_lock() 152 | def remove_host(self, host): 153 | host_directory = os.path.join(self.hosts_directory, host) 154 | if os.path.exists(host_directory): 155 | shutil.rmtree(host_directory) 156 | 157 | @job("Remove Site from Upstream") 158 | def remove_site_from_upstream_job(self, upstream, site, extra_domains=None): 159 | if not extra_domains: 160 | extra_domains = [] 161 | 162 | with self.proxy_config_modification_lock: 163 | upstream_directory = os.path.join(self.upstreams_directory, upstream) 164 | 165 | site_file = os.path.join(upstream_directory, site) 166 | if os.path.exists(site_file): 167 | self.remove_site_from_upstream(site_file) 168 | 169 | for domain in extra_domains: 170 | site_file = os.path.join(upstream_directory, domain) 171 | if os.path.exists(site_file): 172 | self.remove_site_from_upstream(site_file) 173 | 174 | @step("Remove Site File from Upstream Directory") 175 | @with_proxy_config_lock() 176 | def remove_site_from_upstream(self, site_file): 177 | os.remove(site_file) 178 | 179 | @job("Rename Site on Upstream") 180 | def rename_site_on_upstream_job( 181 | self, 182 | upstream: str, 183 | hosts: list[str], 184 | site: str, 185 | new_name: str, 186 | ): 187 | with self.proxy_config_modification_lock: 188 | self.remove_conflicting_site(new_name) 189 | self.rename_site_on_upstream(upstream, site, new_name) 190 | site_host_dir = os.path.join(self.hosts_directory, site) 191 | if os.path.exists(site_host_dir): 192 | self.rename_host_dir(site, new_name) 193 | self.rename_site_in_host_dir(new_name, site, new_name) 194 | for host in hosts: 195 | self.rename_site_in_host_dir(host, site, new_name) 196 | self.reload_nginx() 197 | 198 | def replace_str_in_json(self, file: str, old: str, new: str): 199 | """Replace quoted strings in json file.""" 200 | with open(file) as f: 201 | text = f.read() 202 | text = text.replace('"' + old + '"', '"' + new + '"') 203 | with open(file, "w") as f: 204 | f.write(text) 205 | 206 | @step("Rename Host Directory") 207 | @with_proxy_config_lock() 208 | def rename_host_dir(self, old_name: str, new_name: str): 209 | """Rename site's host directory.""" 210 | old_host_dir = os.path.join(self.hosts_directory, old_name) 211 | new_host_dir = os.path.join(self.hosts_directory, new_name) 212 | os.rename(old_host_dir, new_host_dir) 213 | 214 | @step("Rename Site in Host Directory") 215 | @with_proxy_config_lock() 216 | def rename_site_in_host_dir(self, host: str, old_name: str, new_name: str): 217 | host_directory = os.path.join(self.hosts_directory, host) 218 | 219 | map_file = os.path.join(host_directory, "map.json") 220 | if os.path.exists(map_file): 221 | self.replace_str_in_json(map_file, old_name, new_name) 222 | 223 | redirect_file = os.path.join(host_directory, "redirect.json") 224 | if os.path.exists(redirect_file): 225 | self.replace_str_in_json(redirect_file, old_name, new_name) 226 | 227 | @step("Rename Site File in Upstream Directory") 228 | @with_proxy_config_lock() 229 | def rename_site_on_upstream(self, upstream: str, site: str, new_name: str): 230 | upstream_directory = os.path.join(self.upstreams_directory, upstream) 231 | old_site_file = os.path.join(upstream_directory, site) 232 | new_site_file = os.path.join(upstream_directory, new_name) 233 | if not os.path.exists(old_site_file) and os.path.exists(new_site_file): 234 | return 235 | os.rename(old_site_file, new_site_file) 236 | 237 | @job("Update Site Status") 238 | def update_site_status_job(self, upstream, site, status, extra_domains=None, skip_reload=False): 239 | self.update_site_status(upstream, site, status) 240 | if not extra_domains: 241 | extra_domains = [] 242 | for domain in extra_domains: 243 | self.update_site_status(upstream, domain, status) 244 | if not skip_reload: 245 | self.reload_nginx() 246 | 247 | @step("Update Site File") 248 | @with_proxy_config_lock() 249 | def update_site_status(self, upstream, site, status): 250 | upstream_directory = os.path.join(self.upstreams_directory, upstream) 251 | site_file = os.path.join(upstream_directory, site) 252 | with open(site_file, "w") as f: 253 | f.write(status) 254 | 255 | @job("Setup Redirects on Hosts") 256 | def setup_redirects_job(self, hosts, target): 257 | with self.proxy_config_modification_lock: 258 | if target in hosts: 259 | hosts.remove(target) 260 | self.remove_redirect(target) 261 | for host in hosts: 262 | self.setup_redirect(host, target) 263 | self.reload_nginx() 264 | 265 | @step("Setup Redirect on Host") 266 | @with_proxy_config_lock() 267 | def setup_redirect(self, host, target): 268 | host_directory = os.path.join(self.hosts_directory, host) 269 | os.makedirs(host_directory, exist_ok=True) 270 | redirect_file = os.path.join(host_directory, "redirect.json") 271 | if os.path.exists(redirect_file): 272 | with open(redirect_file) as r: 273 | redirects = json.load(r) 274 | else: 275 | redirects = {} 276 | redirects[host] = target 277 | with open(redirect_file, "w") as r: 278 | json.dump(redirects, r, indent=4) 279 | 280 | @job("Remove Redirects on Hosts") 281 | def remove_redirects_job(self, hosts): 282 | with self.proxy_config_modification_lock: 283 | for host in hosts: 284 | self.remove_redirect(host) 285 | 286 | @step("Remove Redirect on Host") 287 | @with_proxy_config_lock() 288 | def remove_redirect(self, host: str): 289 | host_directory = os.path.join(self.hosts_directory, host) 290 | redirect_file = os.path.join(host_directory, "redirect.json") 291 | if os.path.exists(redirect_file): 292 | os.remove(redirect_file) 293 | if host.endswith("." + self.domain): 294 | # default domain 295 | os.rmdir(host_directory) 296 | 297 | @step("Reload NGINX") 298 | def reload_nginx(self): 299 | return self._reload_nginx() 300 | 301 | @job("Reload NGINX Job") 302 | def reload_nginx_job(self): 303 | return self.reload_nginx() 304 | 305 | def _generate_proxy_config(self): 306 | proxy_config_file = os.path.join(self.nginx_directory, "proxy.conf") 307 | config = self.get_config() 308 | with self.proxy_config_modification_lock: 309 | data = { 310 | "hosts": self.hosts, 311 | "upstreams": self.upstreams, 312 | "domain": config["domain"], 313 | "wildcards": self.wildcards, 314 | "nginx_directory": config["nginx_directory"], 315 | "error_pages_directory": self.error_pages_directory, 316 | "tls_protocols": config.get("tls_protocols"), 317 | } 318 | 319 | self._render_template( 320 | "proxy/nginx.conf.jinja2", 321 | data, 322 | proxy_config_file, 323 | ) 324 | 325 | def _reload_nginx(self): 326 | from agent.nginx_reload_manager import NginxReloadManager 327 | 328 | if not self.job: 329 | raise Exception("NGINX Reload should be trigerred by a job") 330 | 331 | return NginxReloadManager().request_reload(request_id=self.job_record.model.agent_job_id) 332 | 333 | def _create_default_host(self): 334 | default_host = f"*.{self.config['domain']}" 335 | default_host_directory = os.path.join(self.hosts_directory, default_host) 336 | os.makedirs(default_host_directory, exist_ok=True) 337 | map_file = os.path.join(default_host_directory, "map.json") 338 | with open(map_file, "w") as mf: 339 | json.dump({"default": "$host"}, mf, indent=4) 340 | 341 | tls_directory = self.config["tls_directory"] 342 | for f in ["chain.pem", "fullchain.pem", "privkey.pem"]: 343 | source = os.path.join(tls_directory, f) 344 | destination = os.path.join(default_host_directory, f) 345 | if os.path.exists(destination): 346 | os.remove(destination) 347 | os.symlink(source, destination) 348 | 349 | @property 350 | def upstreams(self): 351 | upstreams = {} 352 | for upstream in os.listdir(self.upstreams_directory): # for each server ip 353 | upstream_directory = os.path.join(self.upstreams_directory, upstream) 354 | if not os.path.isdir(upstream_directory): 355 | continue 356 | hashed_upstream = sha(upstream.encode()).hexdigest()[:16] 357 | upstreams[upstream] = {"sites": [], "hash": hashed_upstream} 358 | for site in os.listdir(upstream_directory): 359 | with open(os.path.join(upstream_directory, site)) as f: 360 | status = f.read().strip() 361 | if status in ( 362 | "deactivated", 363 | "suspended", 364 | "suspended_saas", 365 | ): 366 | actual_upstream = status 367 | else: 368 | actual_upstream = hashed_upstream 369 | upstreams[upstream]["sites"].append({"name": site, "upstream": actual_upstream}) 370 | return upstreams 371 | 372 | @property 373 | def hosts(self) -> dict[str, dict[str, str]]: 374 | hosts = defaultdict(lambda: defaultdict(str)) 375 | for host in os.listdir(self.hosts_directory): 376 | host_directory = os.path.join(self.hosts_directory, host) 377 | 378 | map_file = os.path.join(host_directory, "map.json") 379 | if os.path.exists(map_file): 380 | with open(map_file) as m: 381 | hosts[host] = json.load(m) 382 | 383 | redirect_file = os.path.join(host_directory, "redirect.json") 384 | if os.path.exists(redirect_file): 385 | with open(redirect_file) as r: 386 | redirects = json.load(r) 387 | 388 | for _from, to in redirects.items(): 389 | if "*" in host: 390 | hosts[_from] = {_from: _from} 391 | hosts[_from]["redirect"] = to 392 | hosts[host]["codeserver"] = os.path.exists(os.path.join(host_directory, "codeserver")) 393 | 394 | return hosts 395 | 396 | @property 397 | def wildcards(self) -> list[str]: 398 | wildcards = [] 399 | for host in os.listdir(self.hosts_directory): 400 | if "*" in host: 401 | wildcards.append(host.strip("*.")) 402 | return wildcards 403 | 404 | @property 405 | @contextmanager 406 | def proxy_config_modification_lock(self): 407 | if self._proxy_config_modification_lock is None: 408 | lock_path = os.path.join(self.nginx_directory, "proxy_config.lock") 409 | self._proxy_config_modification_lock = filelock.FileLock( 410 | lock_path, 411 | ) 412 | 413 | with self._proxy_config_modification_lock: 414 | yield 415 | -------------------------------------------------------------------------------- /agent/proxysql.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from agent.job import job, step 4 | from agent.server import Server 5 | 6 | 7 | class ProxySQL(Server): 8 | def __init__(self): 9 | super().__init__() 10 | 11 | self.proxysql_admin_password = self.config.get("proxysql_admin_password") 12 | 13 | def proxysql_execute(self, command): 14 | command = ( 15 | "mysql -h 127.0.0.1 -P 6032 " 16 | f"-u frappe -p{self.proxysql_admin_password} " 17 | f"--disable-column-names -e '{command}'" 18 | ) 19 | return self.execute(command) 20 | 21 | @job("Add User to ProxySQL") 22 | def add_user_job( 23 | self, 24 | username: str, 25 | password: str, 26 | database: str, 27 | max_connections: int, 28 | backend: dict, 29 | ): 30 | self.add_backend(backend) 31 | self.add_user(username, password, database, max_connections, backend) 32 | 33 | @job("Add Backend to ProxySQL") 34 | def add_backend_job(self, backend): 35 | self.add_backend(backend) 36 | 37 | @step("Add Backend to ProxySQL") 38 | def add_backend(self, backend): 39 | backend_id = backend["id"] 40 | backend_ip = backend["ip"] 41 | if self.proxysql_execute(f"SELECT 1 from mysql_servers where hostgroup_id = {backend_id}")["output"]: 42 | return 43 | commands = [ 44 | (f'INSERT INTO mysql_servers (hostgroup_id, hostname) VALUES ({backend_id}, "{backend_ip}")'), 45 | "LOAD MYSQL SERVERS TO RUNTIME", 46 | "SAVE MYSQL SERVERS TO DISK", 47 | ] 48 | for command in commands: 49 | self.proxysql_execute(command) 50 | 51 | @step("Add User to ProxySQL") 52 | def add_user(self, username: str, password: str, database: str, max_connections: int, backend: dict): 53 | backend_id = backend["id"] 54 | commands = [ 55 | ( 56 | "INSERT INTO mysql_users ( " 57 | "username, password, default_hostgroup, default_schema, " 58 | "use_ssl, max_connections) " 59 | "VALUES ( " 60 | f'"{username}", "{password}", {backend_id}, "{database}", ' 61 | f"1, {max_connections})" 62 | ), 63 | "LOAD MYSQL USERS TO RUNTIME", 64 | "SAVE MYSQL USERS FROM RUNTIME", 65 | "SAVE MYSQL USERS TO DISK", 66 | ] 67 | for command in commands: 68 | self.proxysql_execute(command) 69 | 70 | @job("Remove User from ProxySQL") 71 | def remove_user_job(self, username): 72 | self.remove_user(username) 73 | 74 | @step("Remove User from ProxySQL") 75 | def remove_user(self, username): 76 | commands = [ 77 | f'DELETE FROM mysql_users WHERE username = "{username}"', 78 | "LOAD MYSQL USERS TO RUNTIME", 79 | "SAVE MYSQL USERS FROM RUNTIME", 80 | "SAVE MYSQL USERS TO DISK", 81 | ] 82 | for command in commands: 83 | self.proxysql_execute(command) 84 | -------------------------------------------------------------------------------- /agent/security.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import re 4 | 5 | from agent.base import Base 6 | 7 | 8 | class Security(Base): 9 | @property 10 | def logs_directory(self): 11 | return "/var/log/ssh_sessions" 12 | 13 | @property 14 | def ssh_session_logs(self): 15 | return self.logs 16 | 17 | def retrieve_ssh_session_log(self, filename): 18 | content = self.retrieve_log(filename) 19 | return self.escape_ansi(content) 20 | 21 | def escape_ansi(self, line): 22 | ansi_escape = re.compile(r"(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]") 23 | return ansi_escape.sub("", line) 24 | -------------------------------------------------------------------------------- /agent/ssh.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | import tempfile 5 | 6 | from agent.job import job, step 7 | from agent.server import Server 8 | 9 | 10 | class SSHProxy(Server): 11 | def __init__(self, directory=None): 12 | super().__init__(directory=directory) 13 | 14 | self.directory = directory or os.getcwd() 15 | self.config_file = os.path.join(self.directory, "config.json") 16 | self.name = self.config["name"] 17 | 18 | self.ssh_directory = os.path.join(self.directory, "ssh") 19 | 20 | self.job = None 21 | self.step = None 22 | 23 | def docker_execute(self, command): 24 | command = f"docker exec ssh {command}" 25 | return self.execute(command) 26 | 27 | @job("Add User to Proxy") 28 | def add_user_job(self, name, principal, ssh, certificate): 29 | self.add_user(name) 30 | self.add_certificate(name, certificate) 31 | self.add_principal(name, principal, ssh) 32 | 33 | @step("Add User to Proxy") 34 | def add_user(self, name): 35 | return self.docker_execute(f"useradd -m -p '*' {name}") 36 | 37 | @step("Add Certificate to User") 38 | def add_certificate(self, name, certificate): 39 | self.docker_execute(f"mkdir /home/{name}/.ssh") 40 | self.docker_execute(f"chown {name}:{name} /home/{name}/.ssh") 41 | for key, value in certificate.items(): 42 | source = tempfile.mkstemp()[1] 43 | with open(source, "w") as f: 44 | f.write(value) 45 | target = f"/home/{name}/.ssh/{key}" 46 | self.execute(f"docker cp {source} ssh:{target}") 47 | self.docker_execute(f"chown {name}:{name} {target}") 48 | os.remove(source) 49 | 50 | @step("Add Principal to User") 51 | def add_principal(self, name, principal, ssh): 52 | cd_command = "cd frappe-bench; exec bash --login" 53 | force_command = f"ssh frappe@{ssh['ip']} -p {ssh['port']} -t '{cd_command}'" 54 | principal_line = f'restrict,pty,command="{force_command}" {principal}' 55 | source = tempfile.mkstemp()[1] 56 | with open(source, "w") as f: 57 | f.write(principal_line) 58 | target = f"/etc/ssh/principals/{name}" 59 | self.execute(f"docker cp {source} ssh:{target}") 60 | self.docker_execute(f"chown {name}:{name} {target}") 61 | os.remove(source) 62 | 63 | @job("Remove User from Proxy") 64 | def remove_user_job(self, name): 65 | self.remove_user(name) 66 | self.remove_principal(name) 67 | 68 | @step("Remove User from Proxy") 69 | def remove_user(self, name): 70 | return self.docker_execute(f"userdel -f -r {name}") 71 | 72 | @step("Remove Principal from User") 73 | def remove_principal(self, name): 74 | command = f"rm /etc/ssh/principals/{name}" 75 | return self.docker_execute(command) 76 | -------------------------------------------------------------------------------- /agent/templates/agent/nginx.conf.jinja2: -------------------------------------------------------------------------------- 1 | ## Set a variable to help us decide if we need to add the 2 | ## 'Docker-Distribution-Api-Version' header. 3 | ## The registry always sets this header. 4 | ## In the case of nginx performing auth, the header is unset 5 | ## since nginx is auth-ing before proxying. 6 | map $upstream_http_docker_distribution_api_version $docker_distribution_api_version { 7 | '' 'registry/2.0'; 8 | } 9 | 10 | ## this is required to proxy Grafana Live WebSocket connections 11 | map $http_upgrade $connection_upgrade { 12 | default upgrade; 13 | '' close; 14 | } 15 | 16 | 17 | server { 18 | listen 443 ssl http2; 19 | server_name {{ name }}; 20 | 21 | ssl_certificate {{ tls_directory }}/fullchain.pem; 22 | ssl_certificate_key {{ tls_directory }}/privkey.pem; 23 | ssl_trusted_certificate {{ tls_directory }}/chain.pem; 24 | 25 | ssl_session_timeout 1d; 26 | ssl_session_cache shared:MozSSL:10m; # about 40000 sessions 27 | ssl_session_tickets off; 28 | 29 | ssl_protocols {{ tls_protocols or 'TLSv1.3' }}; 30 | ssl_prefer_server_ciphers off; 31 | 32 | ssl_stapling on; 33 | ssl_stapling_verify on; 34 | 35 | resolver 1.1.1.1 1.0.0.1 8.8.8.8 8.8.4.4 208.67.222.222 208.67.220.220 valid=60s; 36 | resolver_timeout 2s; 37 | 38 | # disable any limits to avoid HTTP 413 for large image uploads 39 | client_max_body_size 0; 40 | 41 | # required to avoid HTTP 411: see Issue #1486 (https://github.com/moby/moby/issues/1486) 42 | chunked_transfer_encoding on; 43 | 44 | # Allow press signup pages to check browser-proxy latency 45 | {% if press_url -%} 46 | more_set_headers "Access-Control-Allow-Origin: {{ press_url }}"; 47 | {%- endif %} 48 | 49 | location /agent/ { 50 | proxy_http_version 1.1; 51 | proxy_cache_bypass $http_upgrade; 52 | 53 | proxy_set_header Upgrade $http_upgrade; 54 | proxy_set_header Connection "upgrade"; 55 | proxy_set_header host $host; 56 | proxy_set_header X-Real-IP $remote_addr; 57 | proxy_set_header X-Forwarded-For $remote_addr; 58 | proxy_set_header X-Forwarded-Proto $scheme; 59 | proxy_set_header X-Forwarded-Host $host; 60 | proxy_set_header X-Forwarded-Port $server_port; 61 | 62 | location /agent/benches/metrics { 63 | return 301 /metrics/rq; 64 | } 65 | 66 | proxy_pass http://127.0.0.1:{{ web_port }}/; 67 | } 68 | 69 | {% if nginx_vts_module_enabled %} 70 | location /status { 71 | auth_basic "NGINX VTS"; 72 | auth_basic_user_file {{ nginx_directory }}/monitoring.htpasswd; 73 | 74 | vhost_traffic_status_display; 75 | vhost_traffic_status_display_format html; 76 | } 77 | {% endif %} 78 | 79 | location /metrics { 80 | auth_basic "Prometheus"; 81 | auth_basic_user_file {{ nginx_directory }}/monitoring.htpasswd; 82 | 83 | location /metrics/node { 84 | proxy_pass http://127.0.0.1:9100/metrics; 85 | } 86 | 87 | location /metrics/docker { 88 | proxy_pass http://127.0.0.1:9323/metrics; 89 | } 90 | 91 | location /metrics/cadvisor { 92 | proxy_pass http://127.0.0.1:9338/metrics; 93 | } 94 | 95 | {% if nginx_vts_module_enabled %} 96 | location /metrics/nginx { 97 | vhost_traffic_status_display; 98 | vhost_traffic_status_display_format prometheus; 99 | } 100 | {% endif %} 101 | 102 | location /metrics/mariadb { 103 | proxy_pass http://127.0.0.1:9104/metrics; 104 | } 105 | 106 | location /metrics/mariadb_proxy { 107 | proxy_pass http://127.0.0.1:9104/metrics; 108 | } 109 | 110 | location /metrics/gunicorn { 111 | proxy_pass http://127.0.0.1:9102/metrics; 112 | } 113 | 114 | location /metrics/registry { 115 | proxy_pass http://127.0.0.1:5001/metrics; 116 | } 117 | 118 | location /metrics/prometheus { 119 | proxy_pass http://127.0.0.1:9090/prometheus/metrics; 120 | } 121 | 122 | location /metrics/alertmanager { 123 | proxy_pass http://127.0.0.1:9093/alertmanager/metrics; 124 | } 125 | 126 | location /metrics/blackbox { 127 | proxy_pass http://127.0.0.1:9115/blackbox/metrics; 128 | } 129 | 130 | location /metrics/grafana { 131 | proxy_pass http://127.0.0.1:3000/grafana/metrics; 132 | } 133 | 134 | location /metrics/proxysql { 135 | proxy_pass http://127.0.0.1:6070/metrics; 136 | } 137 | 138 | location /metrics/elasticsearch { 139 | proxy_pass http://127.0.0.1:9114/metrics; 140 | } 141 | 142 | location /metrics/rq { 143 | proxy_pass http://127.0.0.1:{{ web_port }}/benches/metrics; 144 | } 145 | 146 | } 147 | 148 | {% if registry %} 149 | 150 | location /v2/ { 151 | # Do not allow connections from docker 1.5 and earlier 152 | # docker pre-1.6.0 did not properly set the user agent on ping, catch "Go *" user agents 153 | if ($http_user_agent ~ "^(docker\/1\.(3|4|5(?!\.[0-9]-dev))|Go ).*$" ) { 154 | return 404; 155 | } 156 | 157 | # To add basic authentication to v2 use auth_basic setting. 158 | auth_basic "Registry realm"; 159 | auth_basic_user_file /home/frappe/registry/registry.htpasswd; 160 | 161 | ## If $docker_distribution_api_version is empty, the header is not added. 162 | ## See the map directive above where this variable is defined. 163 | add_header 'Docker-Distribution-Api-Version' $docker_distribution_api_version always; 164 | 165 | add_header Access-Control-Allow-Origin '*'; 166 | add_header Access-Control-Allow-Credentials 'true'; 167 | add_header Access-Control-Allow-Headers 'Authorization, Accept'; 168 | add_header Access-Control-Allow-Methods 'HEAD, GET, OPTIONS, DELETE'; 169 | add_header Access-Control-Expose-Headers 'Docker-Content-Digest'; 170 | 171 | proxy_pass http://127.0.0.1:5000; 172 | proxy_set_header Host $http_host; # required for docker client's sake 173 | proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP 174 | proxy_set_header X-Forwarded-For $remote_addr; 175 | proxy_set_header X-Forwarded-Proto $scheme; 176 | proxy_read_timeout 900; 177 | } 178 | 179 | location / { 180 | # To add basic authentication to v2 use auth_basic setting. 181 | auth_basic "Registry realm"; 182 | auth_basic_user_file /home/frappe/registry/registry.htpasswd; 183 | 184 | proxy_pass http://127.0.0.1:6000; 185 | proxy_set_header Host $http_host; # required for docker client's sake 186 | proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP 187 | proxy_set_header X-Forwarded-For $remote_addr; 188 | proxy_set_header X-Forwarded-Proto $scheme; 189 | proxy_read_timeout 900; 190 | } 191 | 192 | {% elif monitor %} 193 | 194 | location /prometheus { 195 | auth_basic "Monitoring"; 196 | auth_basic_user_file /home/frappe/agent/nginx/grafana.htpasswd; 197 | proxy_pass http://127.0.0.1:9090/prometheus; 198 | proxy_read_timeout 1500; 199 | } 200 | 201 | location /alertmanager { 202 | auth_basic "Monitoring"; 203 | auth_basic_user_file /home/frappe/agent/nginx/grafana.htpasswd; 204 | proxy_pass http://127.0.0.1:9093/alertmanager; 205 | } 206 | 207 | location /blackbox { 208 | auth_basic "Monitoring"; 209 | auth_basic_user_file /home/frappe/agent/nginx/grafana.htpasswd; 210 | proxy_pass http://127.0.0.1:9115/blackbox; 211 | } 212 | 213 | location /grafana { 214 | auth_basic "Grafana UI"; 215 | auth_basic_user_file /home/frappe/agent/nginx/grafana-ui.htpasswd; 216 | 217 | proxy_pass http://127.0.0.1:3000/grafana; 218 | 219 | location /grafana/metrics { 220 | return 307 https://$host/metrics/grafana; 221 | } 222 | 223 | # Proxy Grafana Live WebSocket connections. 224 | location /grafana/api/live/ { 225 | rewrite ^/grafana/(.*) /$1 break; 226 | proxy_http_version 1.1; 227 | proxy_set_header Upgrade $http_upgrade; 228 | proxy_set_header Connection $connection_upgrade; 229 | proxy_set_header Host $http_host; 230 | proxy_pass http://127.0.0.1:3000/grafana; 231 | } 232 | } 233 | 234 | location / { 235 | return 307 https://$host/grafana; 236 | } 237 | 238 | {% elif log %} 239 | 240 | location /kibana/ { 241 | auth_basic "Kibana"; 242 | auth_basic_user_file /home/frappe/agent/nginx/kibana.htpasswd; 243 | 244 | proxy_set_header Upgrade $http_upgrade; 245 | proxy_set_header Connection "upgrade"; 246 | proxy_set_header host $host; 247 | proxy_set_header X-Real-IP $remote_addr; 248 | proxy_set_header X-Forwarded-For $remote_addr; 249 | proxy_set_header X-Forwarded-Proto $scheme; 250 | proxy_set_header X-Forwarded-Host $host; 251 | proxy_set_header X-Forwarded-Port $server_port; 252 | 253 | 254 | proxy_pass http://127.0.0.1:5601/; 255 | } 256 | 257 | location /elasticsearch/ { 258 | auth_basic "Elasticsearch"; 259 | auth_basic_user_file /home/frappe/agent/nginx/kibana.htpasswd; 260 | 261 | proxy_set_header Upgrade $http_upgrade; 262 | proxy_set_header Connection "upgrade"; 263 | proxy_set_header host $host; 264 | proxy_set_header X-Real-IP $remote_addr; 265 | proxy_set_header X-Forwarded-For $remote_addr; 266 | proxy_set_header X-Forwarded-Proto $scheme; 267 | proxy_set_header X-Forwarded-Host $host; 268 | proxy_set_header X-Forwarded-Port $server_port; 269 | 270 | proxy_pass http://127.0.0.1:9200/; 271 | } 272 | 273 | location / { 274 | return 307 https://$host/kibana; 275 | } 276 | 277 | {% elif analytics %} 278 | 279 | location / { 280 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 281 | proxy_pass http://127.0.0.1:8000/; 282 | } 283 | 284 | {% elif trace %} 285 | 286 | location / { 287 | proxy_buffer_size 32k; 288 | proxy_buffers 8 16k; 289 | proxy_set_header X-Forwarded-For $remote_addr; 290 | proxy_pass http://127.0.0.1:9000/; 291 | } 292 | 293 | {% else %} 294 | 295 | location / { 296 | root {{ pages_directory }}; 297 | try_files /home.html /dev/null; 298 | } 299 | 300 | {% endif %} 301 | } 302 | -------------------------------------------------------------------------------- /agent/templates/agent/redis.conf.jinja2: -------------------------------------------------------------------------------- 1 | dbfilename redis.rdb 2 | bind 127.0.0.1 3 | port {{ redis_port }} 4 | appendonly yes 5 | appendfilename "redis.aof" 6 | save 60 1 7 | -------------------------------------------------------------------------------- /agent/templates/agent/supervisor.conf.jinja2: -------------------------------------------------------------------------------- 1 | [program:web] 2 | command=bash -c "{{ directory }}/repo/wait-for-it.sh redis://127.0.0.1:{{ redis_port }} && {{ directory }}/env/bin/gunicorn --bind 127.0.0.1:{{ web_port }} --workers {{ gunicorn_workers }} agent.web:application" 3 | environment=PYTHONUNBUFFERED=1{% if sentry_dsn %}, SENTRY_DSN="{{ sentry_dsn }}"{% endif %} 4 | autostart=true 5 | autorestart=true 6 | stdout_logfile={{ directory }}/logs/web.log 7 | stderr_logfile={{ directory }}/logs/web.error.log 8 | user={{ user }} 9 | directory={{ directory }} 10 | 11 | [program:redis] 12 | command=redis-server redis.conf 13 | autostart=true 14 | autorestart=true 15 | stdout_logfile={{ directory }}/logs/redis.log 16 | stderr_logfile={{ directory }}/logs/redis.error.log 17 | user={{ user }} 18 | directory={{ directory }} 19 | 20 | [program:worker] 21 | command=bash -c "{{ directory }}/repo/wait-for-it.sh redis://127.0.0.1:{{ redis_port }} && exec {{ directory }}/env/bin/rq worker {% if sentry_dsn %}--sentry-dsn '{{ sentry_dsn }}'{% endif %} --url redis://127.0.0.1:{{ redis_port }} high default low" 22 | environment=PYTHONUNBUFFERED=1 23 | autostart=true 24 | autorestart=true 25 | stopwaitsecs=1500 26 | killasgroup=true 27 | numprocs={{ workers }} 28 | process_name=%(program_name)s-%(process_num)d 29 | stdout_logfile={{ directory }}/logs/worker.log 30 | stderr_logfile={{ directory }}/logs/worker.error.log 31 | user={{ user }} 32 | directory={{ directory }} 33 | 34 | {% if is_proxy_server %} 35 | [program:nginx_reload_manager] 36 | command=bash -c "{{ directory }}/repo/wait-for-it.sh redis://127.0.0.1:{{ redis_port }} && exec {{directory}}/env/bin/python {{ directory }}/repo/agent/nginx_reload_manager.py" 37 | environment=PYTHONUNBUFFERED=1 38 | autostart=true 39 | autorestart=true 40 | stopwaitsecs=20 41 | stdout_logfile={{ directory }}/logs/nginx_reload_manager.log 42 | stderr_logfile={{ directory }}/logs/nginx_reload_manager.error.log 43 | user={{ user }} 44 | directory={{ directory }} 45 | {% endif %} 46 | 47 | [group:agent] 48 | {% if is_proxy_server %}programs=web, redis, worker, nginx_reload_manager{% else %}programs=web, redis, worker{% endif %} 49 | -------------------------------------------------------------------------------- /agent/templates/alertmanager/routes.yml: -------------------------------------------------------------------------------- 1 | ### routes ### 2 | -------------------------------------------------------------------------------- /agent/templates/bench/docker-compose.yml.jinja2: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | web: 4 | image: {{ docker_image }} 5 | command: 6 | [ 7 | "/home/frappe/frappe-bench/env/bin/gunicorn", 8 | "--bind", 9 | "0.0.0.0:8000", 10 | "--timeout", 11 | "{{ http_timeout }}", 12 | "--workers", 13 | "{{ gunicorn_workers }}", 14 | "--worker-tmp-dir", 15 | "/dev/shm", 16 | "--preload", 17 | "frappe.app:application", 18 | ] 19 | working_dir: /home/frappe/frappe-bench/sites 20 | ports: 21 | - "{{ web_port }}:8000" 22 | volumes: 23 | - logs:/home/frappe/frappe-bench/logs:rw 24 | - sites:/home/frappe/frappe-bench/sites:rw 25 | depends_on: 26 | - redis-cache 27 | - redis-queue 28 | - redis-socketio 29 | socketio: 30 | image: {{ docker_image }} 31 | command: ["node", "/home/frappe/frappe-bench/apps/frappe/socketio.js"] 32 | working_dir: /home/frappe/frappe-bench 33 | ports: 34 | - "{{ socketio_port }}:9000" 35 | volumes: 36 | - logs:/home/frappe/frappe-bench/logs:rw 37 | - sites:/home/frappe/frappe-bench/sites:ro 38 | depends_on: 39 | - redis-socketio 40 | - web 41 | scheduler: 42 | image: {{ docker_image }} 43 | command: ["bench", "schedule"] 44 | working_dir: /home/frappe/frappe-bench 45 | volumes: 46 | - sites:/home/frappe/frappe-bench/sites:rw 47 | - logs:/home/frappe/frappe-bench/logs:rw 48 | depends_on: 49 | - redis-cache 50 | - redis-queue 51 | - redis-socketio 52 | worker_default: 53 | image: {{ docker_image }} 54 | command: ["bench", "worker", "--queue", "default"] 55 | working_dir: /home/frappe/frappe-bench 56 | volumes: 57 | - sites:/home/frappe/frappe-bench/sites:rw 58 | - logs:/home/frappe/frappe-bench/logs:rw 59 | depends_on: 60 | - redis-cache 61 | - redis-queue 62 | - redis-socketio 63 | deploy: 64 | replicas: {{ background_workers }} 65 | worker_short: 66 | image: {{ docker_image }} 67 | command: ["bench", "worker", "--queue", "short"] 68 | working_dir: /home/frappe/frappe-bench 69 | volumes: 70 | - sites:/home/frappe/frappe-bench/sites:rw 71 | - logs:/home/frappe/frappe-bench/logs:rw 72 | depends_on: 73 | - redis-cache 74 | - redis-queue 75 | - redis-socketio 76 | deploy: 77 | replicas: {{ background_workers }} 78 | worker_long: 79 | image: {{ docker_image }} 80 | command: ["bench", "worker", "--queue", "long"] 81 | working_dir: /home/frappe/frappe-bench 82 | volumes: 83 | - sites:/home/frappe/frappe-bench/sites:rw 84 | - logs:/home/frappe/frappe-bench/logs:rw 85 | depends_on: 86 | - redis-cache 87 | - redis-queue 88 | - redis-socketio 89 | deploy: 90 | replicas: {{ background_workers }} 91 | redis-cache: 92 | image: redis:6.0 93 | redis-queue: 94 | image: redis:6.0 95 | redis-socketio: 96 | image: redis:6.0 97 | 98 | # Any container should be allowed to manually attach to default overlay network 99 | networks: 100 | default: 101 | driver: overlay 102 | attachable: true 103 | 104 | volumes: 105 | logs: 106 | driver: local 107 | driver_opts: 108 | type: none 109 | o: bind 110 | device: {{ directory }}/logs 111 | sites: 112 | driver: local 113 | driver_opts: 114 | type: none 115 | o: bind 116 | device: {{ directory }}/sites 117 | -------------------------------------------------------------------------------- /agent/templates/bench/nginx.conf.jinja2: -------------------------------------------------------------------------------- 1 | 2 | upstream {{ bench_name }}-web-server { 3 | server 127.0.0.1:{{ web_port }} fail_timeout=0; 4 | } 5 | 6 | upstream {{ bench_name}}-socketio-server { 7 | server 127.0.0.1:{{ socketio_port }} fail_timeout=0; 8 | } 9 | 10 | map $host $site_name_{{ bench_name_slug }} { 11 | 12 | {% for (from, to) in domains.items() -%} 13 | {{ from }} {{ to }}; 14 | {% endfor %} 15 | 16 | default $host; 17 | } 18 | 19 | # server blocks 20 | {% if sites -%} 21 | 22 | {% if standalone %} 23 | {% for site in sites + ( domains.keys() | list ) %} 24 | server { 25 | listen 443 http2 ssl; 26 | server_name {{ site.name or site}}; 27 | 28 | ssl_certificate {{ nginx_directory }}/hosts/{{ site.host or site }}/fullchain.pem; 29 | ssl_certificate_key {{ nginx_directory }}/hosts/{{ site.host or site }}/privkey.pem; 30 | ssl_trusted_certificate {{ nginx_directory }}/hosts/{{ site.host or site }}/chain.pem; 31 | 32 | ssl_session_timeout 1d; 33 | ssl_session_cache shared:MozSSL:10m; # about 40000 sessions 34 | ssl_session_tickets off; 35 | 36 | ssl_protocols {{ tls_protocols or 'TLSv1.2 TLSv1.3' }}; 37 | ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; 38 | ssl_prefer_server_ciphers off; 39 | 40 | ssl_stapling on; 41 | ssl_stapling_verify on; 42 | 43 | resolver 1.1.1.1 1.0.0.1 8.8.8.8 8.8.4.4 208.67.222.222 208.67.220.220 valid=60s; 44 | resolver_timeout 2s; 45 | 46 | client_max_body_size 250M; 47 | client_body_buffer_size 16K; 48 | client_header_buffer_size 1k; 49 | 50 | more_set_headers "X-Frame-Options: SAMEORIGIN"; 51 | more_set_headers "X-XSS-Protection: 1; mode=block"; 52 | more_set_headers "X-Content-Type-Options: nosniff"; 53 | more_set_headers "Referrer-Policy: no-referrer-when-downgrade"; 54 | more_set_headers "Strict-Transport-Security: max-age=31536000; includeSubDomains; preload"; 55 | 56 | root {{ sites_directory }}; 57 | 58 | location /assets { 59 | try_files $uri =404; 60 | add_header Cache-Control "max-age=31536000, immutable"; 61 | } 62 | 63 | location ~ ^/protected/(.*) { 64 | internal; 65 | try_files /$site_name_{{ bench_name_slug }}/$1 =404; 66 | } 67 | 68 | location /socket.io { 69 | proxy_http_version 1.1; 70 | proxy_set_header Upgrade $http_upgrade; 71 | proxy_set_header Connection "upgrade"; 72 | proxy_set_header X-Frappe-Site-Name $site_name_{{ bench_name_slug }}; 73 | proxy_set_header Origin https://$http_host; 74 | proxy_set_header Host $host; 75 | 76 | proxy_pass http://{{ bench_name }}-socketio-server; 77 | } 78 | 79 | location / { 80 | 81 | rewrite ^(.+)/$ https://$host$1 permanent; 82 | rewrite ^(.+)/index\.html$ https://$host$1 permanent; 83 | rewrite ^(.+)\.html$ https://$host$1 permanent; 84 | 85 | location ~* ^/files/.*.(htm|html|svg|xml) { 86 | add_header Content-disposition "attachment"; 87 | add_header Cache-Control "max-age=3600,stale-while-revalidate=86400"; 88 | try_files /$site_name_{{ bench_name_slug }}/public/$uri @webserver; 89 | } 90 | 91 | location ~* ^/files/.*.(png|jpe?g|gif|css|js|mp3|wav|ogg|flac|avi|mov|mp4|m4v|mkv|webm) { 92 | add_header Cache-Control "max-age=3600,stale-while-revalidate=86400"; 93 | try_files /$site_name_{{ bench_name_slug }}/public/$uri @webserver; 94 | } 95 | 96 | try_files /$site_name_{{ bench_name_slug }}/public/$uri @webserver; 97 | } 98 | 99 | location @webserver { 100 | proxy_set_header X-Forwarded-For $remote_addr; 101 | proxy_set_header X-Forwarded-Proto https; 102 | proxy_set_header X-Frappe-Site-Name $site_name_{{ bench_name_slug }}; 103 | proxy_set_header Host $host; 104 | proxy_set_header X-Use-X-Accel-Redirect True; 105 | proxy_read_timeout {{ http_timeout }}; 106 | proxy_redirect off; 107 | 108 | proxy_pass http://{{ bench_name }}-web-server; 109 | } 110 | 111 | proxy_intercept_errors on; 112 | error_page 429 /exceeded.html; 113 | location /exceeded.html { 114 | root {{ error_pages_directory }}; 115 | internal; 116 | } 117 | 118 | # optimizations 119 | sendfile on; 120 | keepalive_timeout 15; 121 | 122 | # enable gzip compresion 123 | # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge 124 | gzip on; 125 | gzip_http_version 1.1; 126 | gzip_comp_level 5; 127 | gzip_min_length 256; 128 | gzip_proxied any; 129 | gzip_vary on; 130 | gzip_types 131 | application/atom+xml 132 | application/javascript 133 | application/json 134 | application/rss+xml 135 | application/vnd.ms-fontobject 136 | application/x-font-ttf 137 | application/font-woff 138 | application/x-web-app-manifest+json 139 | application/xhtml+xml 140 | application/xml 141 | font/opentype 142 | image/svg+xml 143 | image/x-icon 144 | text/css 145 | text/plain 146 | text/x-component 147 | ; 148 | # text/html is always compressed by HttpGzipModule 149 | } 150 | 151 | {% endfor %} 152 | 153 | server { 154 | listen 80; 155 | server_name 156 | {% for site in sites -%} 157 | {{ site.name }} 158 | {% endfor -%} 159 | {% for domain in domains -%} 160 | {{ domain }} 161 | {% endfor -%} 162 | ; 163 | 164 | location ^~ /.well-known/acme-challenge/ { 165 | return 301 http://ssl.{{ domain }}$request_uri; 166 | } 167 | location / { 168 | return 301 https://$host$request_uri; 169 | } 170 | } 171 | {% else %} 172 | server { 173 | listen 80; 174 | 175 | server_name 176 | {% for site in sites -%} 177 | {{ site.name }} 178 | {% endfor -%} 179 | {% for domain in domains -%} 180 | {{ domain }} 181 | {% endfor -%} 182 | ; 183 | 184 | root {{ sites_directory }}; 185 | 186 | add_header X-Frame-Options "SAMEORIGIN"; 187 | add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; 188 | add_header X-Content-Type-Options nosniff; 189 | add_header X-XSS-Protection "1; mode=block"; 190 | 191 | location /assets { 192 | try_files $uri =404; 193 | add_header Cache-Control "max-age=31536000, immutable"; 194 | } 195 | 196 | location ~ ^/protected/(.*) { 197 | internal; 198 | try_files /$site_name_{{ bench_name_slug }}/$1 =404; 199 | } 200 | 201 | location /socket.io { 202 | proxy_http_version 1.1; 203 | proxy_set_header Upgrade $http_upgrade; 204 | proxy_set_header Connection "upgrade"; 205 | proxy_set_header X-Frappe-Site-Name $site_name_{{ bench_name_slug }}; 206 | proxy_set_header Origin https://$http_host; 207 | proxy_set_header Host $host; 208 | 209 | proxy_pass http://{{ bench_name }}-socketio-server; 210 | } 211 | 212 | location / { 213 | 214 | rewrite ^(.+)/$ https://$host$1 permanent; 215 | rewrite ^(.+)/index\.html$ https://$host$1 permanent; 216 | rewrite ^(.+)\.html$ https://$host$1 permanent; 217 | 218 | location ~* ^/files/.*.(htm|html|svg|xml) { 219 | add_header Content-disposition "attachment"; 220 | add_header Cache-Control "max-age=3600,stale-while-revalidate=86400"; 221 | try_files /$site_name_{{ bench_name_slug }}/public/$uri @webserver; 222 | } 223 | 224 | location ~* ^/files/.*.(png|jpe?g|gif|css|js|mp3|wav|ogg|flac|avi|mov|mp4|m4v|mkv|webm) { 225 | add_header Cache-Control "max-age=3600,stale-while-revalidate=86400"; 226 | try_files /$site_name_{{ bench_name_slug }}/public/$uri @webserver; 227 | } 228 | 229 | try_files /$site_name_{{ bench_name_slug }}/public/$uri @webserver; 230 | } 231 | 232 | location @webserver { 233 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 234 | proxy_set_header X-Forwarded-Proto https; 235 | proxy_set_header X-Frappe-Site-Name $site_name_{{ bench_name_slug }}; 236 | proxy_set_header Host $host; 237 | proxy_set_header X-Use-X-Accel-Redirect True; 238 | proxy_read_timeout {{ http_timeout }}; 239 | proxy_redirect off; 240 | 241 | proxy_pass http://{{ bench_name }}-web-server; 242 | } 243 | 244 | # optimizations 245 | sendfile on; 246 | keepalive_timeout 15; 247 | client_max_body_size 250M; 248 | client_body_buffer_size 16K; 249 | client_header_buffer_size 1k; 250 | 251 | # enable gzip compresion 252 | # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge 253 | gzip on; 254 | gzip_http_version 1.1; 255 | gzip_comp_level 5; 256 | gzip_min_length 256; 257 | gzip_proxied any; 258 | gzip_vary on; 259 | gzip_types 260 | application/atom+xml 261 | application/javascript 262 | application/json 263 | application/rss+xml 264 | application/vnd.ms-fontobject 265 | application/x-font-ttf 266 | application/font-woff 267 | application/x-web-app-manifest+json 268 | application/xhtml+xml 269 | application/xml 270 | font/opentype 271 | image/svg+xml 272 | image/x-icon 273 | text/css 274 | text/plain 275 | text/x-component 276 | ; 277 | # text/html is always compressed by HttpGzipModule 278 | } 279 | 280 | {% endif %} 281 | {%- endif %} 282 | 283 | {% if code_server -%} 284 | server { 285 | listen 80; 286 | server_name {{ code_server.name }}; 287 | 288 | location / { 289 | proxy_set_header Host $host; 290 | proxy_set_header Upgrade $http_upgrade; 291 | proxy_set_header Connection upgrade; 292 | proxy_set_header Accept-Encoding gzip; 293 | proxy_pass http://127.0.0.1:{{ code_server.port }}/; 294 | } 295 | } 296 | {% endif %} 297 | -------------------------------------------------------------------------------- /agent/templates/bench/supervisor.conf: -------------------------------------------------------------------------------- 1 | {% if environment_variables %} 2 | 3 | [supervisord] 4 | environment={% for key, value in environment_variables.items() %}{{ key }}="{{ value }}",{% endfor %} 5 | {% endif %} 6 | 7 | [program:frappe-bench-frappe-web] 8 | command=/home/frappe/frappe-bench/env/bin/gunicorn --bind 0.0.0.0:8000 --workers {{ gunicorn_workers }} --timeout {{ http_timeout }} --graceful-timeout 30 --worker-tmp-dir /dev/shm frappe.app:application --preload --statsd-host={{ statsd_host }} --statsd-prefix={{ name }} {% if gunicorn_threads_per_worker > 0 %} --worker-class=gthread --threads={{ gunicorn_threads_per_worker }}{% endif %} {% if gunicorn_workers > 1 %} --max-requests 5000 --max-requests-jitter 1000 {% endif %} 9 | environment=FORWARDED_ALLOW_IPS="*" 10 | priority=4 11 | autostart=true 12 | autorestart=true 13 | stopwaitsecs=40 14 | killasgroup=true 15 | stdout_logfile=/home/frappe/frappe-bench/logs/web.log 16 | stderr_logfile=/home/frappe/frappe-bench/logs/web.error.log 17 | user=frappe 18 | directory=/home/frappe/frappe-bench/sites 19 | 20 | [program:frappe-bench-frappe-schedule] 21 | command=bench schedule 22 | priority=9 23 | autostart=true 24 | startsecs=0 25 | stdout_logfile=/home/frappe/frappe-bench/logs/schedule.log 26 | stderr_logfile=/home/frappe/frappe-bench/logs/schedule.error.log 27 | user=frappe 28 | directory=/home/frappe/frappe-bench 29 | 30 | {% if use_rq_workerpool and merge_all_rq_queues %} 31 | 32 | [program:frappe-bench-frappe-worker] 33 | command=bench worker-pool --queue short,default,long --num-workers={{ background_workers }} 34 | priority=4 35 | autostart=true 36 | autorestart=true 37 | stdout_logfile=/home/frappe/frappe-bench/logs/worker.log 38 | stderr_logfile=/home/frappe/frappe-bench/logs/worker.error.log 39 | user=frappe 40 | stopwaitsecs=1560 41 | directory=/home/frappe/frappe-bench 42 | killasgroup=true 43 | process_name=%(program_name)s 44 | 45 | {% elif use_rq_workerpool %} 46 | 47 | [program:frappe-bench-frappe-short-worker] 48 | command=bench worker-pool --queue short,default --num-workers={{ background_workers }} 49 | priority=4 50 | autostart=true 51 | autorestart=true 52 | stdout_logfile=/home/frappe/frappe-bench/logs/worker.log 53 | stderr_logfile=/home/frappe/frappe-bench/logs/worker.error.log 54 | user=frappe 55 | stopwaitsecs=360 56 | directory=/home/frappe/frappe-bench 57 | killasgroup=true 58 | process_name=%(program_name)s 59 | 60 | [program:frappe-bench-frappe-long-worker] 61 | command=bench worker-pool --queue long,default,short --num-workers={{ background_workers }} 62 | priority=4 63 | autostart=true 64 | autorestart=true 65 | stdout_logfile=/home/frappe/frappe-bench/logs/worker.log 66 | stderr_logfile=/home/frappe/frappe-bench/logs/worker.error.log 67 | user=frappe 68 | stopwaitsecs=1560 69 | directory=/home/frappe/frappe-bench 70 | killasgroup=true 71 | process_name=%(program_name)s 72 | 73 | {% elif merge_all_rq_queues %} 74 | 75 | [program:frappe-bench-frappe-worker] 76 | command=bench worker --queue short,default,long 77 | priority=4 78 | autostart=true 79 | autorestart=true 80 | stdout_logfile=/home/frappe/frappe-bench/logs/worker.log 81 | stderr_logfile=/home/frappe/frappe-bench/logs/worker.error.log 82 | user=frappe 83 | stopwaitsecs=1560 84 | directory=/home/frappe/frappe-bench 85 | killasgroup=true 86 | numprocs={{ background_workers }} 87 | process_name=%(program_name)s-%(process_num)d 88 | 89 | {% elif merge_default_and_short_rq_queues %} 90 | 91 | [program:frappe-bench-frappe-short-worker] 92 | command=bench worker --queue short,default 93 | priority=4 94 | autostart=true 95 | autorestart=true 96 | stdout_logfile=/home/frappe/frappe-bench/logs/worker.log 97 | stderr_logfile=/home/frappe/frappe-bench/logs/worker.error.log 98 | user=frappe 99 | stopwaitsecs=360 100 | directory=/home/frappe/frappe-bench 101 | killasgroup=true 102 | numprocs={{ background_workers }} 103 | process_name=%(program_name)s-%(process_num)d 104 | 105 | [program:frappe-bench-frappe-long-worker] 106 | command=bench worker --queue long,default,short 107 | priority=4 108 | autostart=true 109 | autorestart=true 110 | stdout_logfile=/home/frappe/frappe-bench/logs/worker.log 111 | stderr_logfile=/home/frappe/frappe-bench/logs/worker.error.log 112 | user=frappe 113 | stopwaitsecs=1560 114 | directory=/home/frappe/frappe-bench 115 | killasgroup=true 116 | numprocs={{ background_workers }} 117 | process_name=%(program_name)s-%(process_num)d 118 | 119 | {% else %} 120 | 121 | [program:frappe-bench-frappe-default-worker] 122 | command=bench worker --queue default 123 | priority=4 124 | autostart=true 125 | autorestart=true 126 | stdout_logfile=/home/frappe/frappe-bench/logs/worker.log 127 | stderr_logfile=/home/frappe/frappe-bench/logs/worker.error.log 128 | user=frappe 129 | stopwaitsecs=1560 130 | directory=/home/frappe/frappe-bench 131 | killasgroup=true 132 | numprocs={{ background_workers }} 133 | process_name=%(program_name)s-%(process_num)d 134 | 135 | [program:frappe-bench-frappe-short-worker] 136 | command=bench worker --queue short 137 | priority=4 138 | autostart=true 139 | autorestart=true 140 | stdout_logfile=/home/frappe/frappe-bench/logs/worker.log 141 | stderr_logfile=/home/frappe/frappe-bench/logs/worker.error.log 142 | user=frappe 143 | stopwaitsecs=360 144 | directory=/home/frappe/frappe-bench 145 | killasgroup=true 146 | numprocs={{ background_workers }} 147 | process_name=%(program_name)s-%(process_num)d 148 | 149 | [program:frappe-bench-frappe-long-worker] 150 | command=bench worker --queue long 151 | priority=4 152 | autostart=true 153 | autorestart=true 154 | stdout_logfile=/home/frappe/frappe-bench/logs/worker.log 155 | stderr_logfile=/home/frappe/frappe-bench/logs/worker.error.log 156 | user=frappe 157 | stopwaitsecs=1560 158 | directory=/home/frappe/frappe-bench 159 | killasgroup=true 160 | numprocs={{ background_workers }} 161 | process_name=%(program_name)s-%(process_num)d 162 | 163 | {% endif %} 164 | 165 | [program:frappe-bench-redis-cache] 166 | command=redis-server /home/frappe/frappe-bench/config/redis-cache.conf 167 | priority=1 168 | autostart=true 169 | autorestart=true 170 | stdout_logfile=/home/frappe/frappe-bench/logs/redis-cache.log 171 | stderr_logfile=/home/frappe/frappe-bench/logs/redis-cache.error.log 172 | user=frappe 173 | directory=/home/frappe/frappe-bench/sites 174 | 175 | [program:frappe-bench-redis-queue] 176 | command=redis-server /home/frappe/frappe-bench/config/redis-queue.conf 177 | priority=1 178 | autostart=true 179 | autorestart=true 180 | stdout_logfile=/home/frappe/frappe-bench/logs/redis-queue.log 181 | stderr_logfile=/home/frappe/frappe-bench/logs/redis-queue.error.log 182 | user=frappe 183 | directory=/home/frappe/frappe-bench/sites 184 | 185 | [program:frappe-bench-node-socketio] 186 | command=node /home/frappe/frappe-bench/apps/frappe/socketio.js 187 | priority=4 188 | autostart=true 189 | autorestart=true 190 | stdout_logfile=/home/frappe/frappe-bench/logs/node-socketio.log 191 | stderr_logfile=/home/frappe/frappe-bench/logs/node-socketio.error.log 192 | user=frappe 193 | directory=/home/frappe/frappe-bench 194 | 195 | [group:frappe-bench-web] 196 | programs=frappe-bench-frappe-web,frappe-bench-node-socketio 197 | 198 | {% if merge_all_rq_queues %} 199 | 200 | [group:frappe-bench-workers] 201 | programs=frappe-bench-frappe-schedule,frappe-bench-frappe-worker 202 | 203 | {% elif merge_default_and_short_rq_queues or use_rq_workerpool %} 204 | 205 | [group:frappe-bench-workers] 206 | programs=frappe-bench-frappe-schedule,frappe-bench-frappe-short-worker,frappe-bench-frappe-long-worker 207 | 208 | {% else %} 209 | 210 | [group:frappe-bench-workers] 211 | programs=frappe-bench-frappe-schedule,frappe-bench-frappe-default-worker,frappe-bench-frappe-short-worker,frappe-bench-frappe-long-worker 212 | 213 | {% endif %} 214 | 215 | [group:frappe-bench-redis] 216 | programs=frappe-bench-redis-cache,frappe-bench-redis-queue 217 | 218 | {% if is_code_server_enabled %} 219 | [program:code-server] 220 | command=code-server --bind-addr 0.0.0.0:8088 . 221 | autostart=false 222 | autorestart=true 223 | stdout_logfile=/home/frappe/frappe-bench/logs/code-server.log 224 | stderr_logfile=/home/frappe/frappe-bench/logs/code-server.error.log 225 | user=frappe 226 | directory=/home/frappe/frappe-bench/apps 227 | {% endif %} 228 | 229 | {% if is_ssh_enabled %} 230 | [program:sshd] 231 | command=/usr/sbin/sshd -f /home/frappe/frappe-bench/config/ssh/sshd_config -D -e 232 | autostart=true 233 | autorestart=true 234 | stdout_logfile=/home/frappe/frappe-bench/logs/ssh.log 235 | stderr_logfile=/home/frappe/frappe-bench/logs/ssh.error.log 236 | user=frappe 237 | directory=/home/frappe 238 | {% endif %} 239 | -------------------------------------------------------------------------------- /agent/templates/nginx/nginx.conf.jinja2: -------------------------------------------------------------------------------- 1 | user www-data; 2 | worker_processes auto; 3 | worker_rlimit_nofile 65535; 4 | worker_shutdown_timeout 30s; 5 | 6 | pid /run/nginx.pid; 7 | 8 | load_module modules/ngx_http_headers_more_filter_module.so; 9 | {% if nginx_vts_module_enabled %} 10 | load_module modules/ngx_http_vhost_traffic_status_module.so; 11 | {% endif %} 12 | 13 | 14 | events { 15 | worker_connections 65535; 16 | multi_accept on; 17 | } 18 | 19 | http { 20 | include /etc/nginx/mime.types; 21 | default_type application/octet-stream; 22 | 23 | {% if ip_whitelist -%} 24 | {%- for ip in ip_whitelist -%} 25 | allow {{ ip }}; 26 | {% endfor -%} 27 | 28 | deny all; 29 | {%- endif %} 30 | 31 | {% if nginx_vts_module_enabled %} 32 | vhost_traffic_status_zone; 33 | vhost_traffic_status_dump /var/log/nginx/vts.db; 34 | vhost_traffic_status_filter_by_host on; 35 | vhost_traffic_status_zone shared:vhost_traffic_status:256m; 36 | {% endif %} 37 | 38 | log_format main '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" "$host" $request_time'; 39 | 40 | access_log /var/log/nginx/access.log main; 41 | error_log /var/log/nginx/error.log warn; 42 | 43 | sendfile on; 44 | tcp_nopush on; 45 | tcp_nodelay on; 46 | server_tokens off; 47 | 48 | more_set_headers 'Server: Frappe Cloud'; 49 | 50 | keepalive_timeout 10; 51 | keepalive_requests 10; 52 | 53 | {% if proxy_ip %} 54 | real_ip_header X-Real-IP; 55 | set_real_ip_from {{ proxy_ip }}; 56 | {% endif %} 57 | gzip on; 58 | gzip_vary on; 59 | gzip_proxied any; 60 | gzip_comp_level 6; 61 | gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml; 62 | 63 | server_names_hash_max_size 4096; 64 | variables_hash_bucket_size 128; 65 | 66 | open_file_cache max=65000 inactive=1m; 67 | open_file_cache_valid 5s; 68 | open_file_cache_min_uses 1; 69 | open_file_cache_errors on; 70 | 71 | ssl_session_timeout 1d; 72 | ssl_session_cache shared:MozSSL:10m; # about 40000 sessions 73 | ssl_session_tickets off; 74 | 75 | # intermediate configuration 76 | ssl_protocols {{ tls_protocols or 'TLSv1.2 TLSv1.3' }}; 77 | ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; 78 | ssl_prefer_server_ciphers off; 79 | 80 | large_client_header_buffers 4 32k; 81 | 82 | proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=web-cache:8m max_size=1000m inactive=600m; 83 | 84 | include /etc/nginx/conf.d/*.conf; 85 | include /home/frappe/benches/*/nginx.conf; 86 | } 87 | -------------------------------------------------------------------------------- /agent/templates/prometheus/domains.yml: -------------------------------------------------------------------------------- 1 | - targets: 2 | 3 | ##- for domain in domains ## 4 | - targets: 5 | - https://{{ domain.name }}/api/method/ping 6 | labels: 7 | site: "{{ domain.site }}" 8 | ##- endfor ## 9 | -------------------------------------------------------------------------------- /agent/templates/prometheus/rules.yml: -------------------------------------------------------------------------------- 1 | ### rules ### 2 | -------------------------------------------------------------------------------- /agent/templates/prometheus/servers.yml: -------------------------------------------------------------------------------- 1 | - targets: 2 | 3 | ## for job, servers in cluster.jobs.items() ## 4 | - targets: 5 | ##- for server in servers ## 6 | - "{{ server }}" 7 | ##- endfor ## 8 | labels: 9 | cluster: "{{ cluster.name }}" 10 | job: "{{ job }}" 11 | __metrics_path__: /metrics/{{ job }} 12 | ## endfor ## 13 | -------------------------------------------------------------------------------- /agent/templates/prometheus/sites.yml: -------------------------------------------------------------------------------- 1 | - targets: 2 | 3 | ## for bench in benches ## 4 | - targets: 5 | ##- for site in bench.sites ## 6 | - https://{{ site }}/api/method/ping 7 | ##- endfor ## 8 | labels: 9 | cluster: "{{ bench.cluster }}" 10 | server: "{{ bench.server }}" 11 | group: "{{ bench.group }}" 12 | bench: "{{ bench.name }}" 13 | ## endfor ## 14 | -------------------------------------------------------------------------------- /agent/templates/prometheus/tls.yml: -------------------------------------------------------------------------------- 1 | - targets: 2 | 3 | ## for server in servers ## 4 | - targets: 5 | - https://{{ server.name }} 6 | ## endfor ## 7 | -------------------------------------------------------------------------------- /agent/templates/proxy/nginx.conf.jinja2: -------------------------------------------------------------------------------- 1 | map $host $subdomain_map { 2 | ~^(?[^.]*).*$ $subdomain.{{ domain }}; 3 | } 4 | 5 | {% for name, upstream in upstreams.items() %} 6 | upstream {{ upstream["hash"] }} { 7 | server {{ name }}:80; 8 | keepalive 100; 9 | } 10 | {% endfor %} 11 | 12 | upstream site_not_found { 13 | server 127.0.0.1:10090; 14 | keepalive 100; 15 | } 16 | 17 | upstream deactivated { 18 | server 127.0.0.1:10091; 19 | keepalive 100; 20 | } 21 | 22 | upstream suspended { 23 | server 127.0.0.1:10092; 24 | keepalive 100; 25 | } 26 | 27 | upstream suspended_saas { 28 | server 127.0.0.1:10093; 29 | keepalive 100; 30 | } 31 | 32 | map $actual_host $upstream_server_hash { 33 | {% for upstream in upstreams.values() -%} 34 | {% for site in upstream["sites"] %} 35 | {{ site.name }} http://{{ site.upstream }}; 36 | {%- endfor %} 37 | {% endfor %} 38 | default http://site_not_found; 39 | } 40 | 41 | map $host $actual_host { 42 | {% for host in hosts.values() -%} 43 | {% for from, to in host.items() if from != 'redirect'%} 44 | {% if from != 'codeserver' %} 45 | {{ from }} {{ to }}; 46 | {% endif %} 47 | {%- endfor %} 48 | {%- endfor %} 49 | } 50 | 51 | map $upstream_http_access_control_allow_origin $access_control_allow_origin { 52 | default $upstream_http_access_control_allow_origin; 53 | "" "https://$host"; 54 | } 55 | 56 | map $upstream_http_access_control_allow_headers $access_control_allow_headers { 57 | default $upstream_http_access_control_allow_headers; 58 | "" "'Origin, Content-Type, Accept"; 59 | } 60 | 61 | map $upstream_http_access_control_allow_credentials $access_control_allow_credentials { 62 | default $upstream_http_access_control_allow_credentials; 63 | "" "true"; 64 | } 65 | 66 | map $upstream_http_access_control_allow_methods $access_control_allow_methods { 67 | default $upstream_http_access_control_allow_methods; 68 | "" "'GET, POST, OPTIONS"; 69 | } 70 | 71 | proxy_cache_path /tmp/cache keys_zone=proxy_cache_zone:10m loader_threshold=300 loader_files=200 max_size=200m; 72 | 73 | {% for host, host_options in hosts.items() %} 74 | 75 | {% set ns = namespace(host=host) %} 76 | 77 | server { 78 | listen 443 http2 ssl{% if ns.host.strip("*.") == domain %} default_server{% endif %}; 79 | server_name {{ ns.host }}; 80 | 81 | 82 | {%- for wildcard_domain in wildcards -%} 83 | {%- if ns.host.endswith("." + wildcard_domain) -%} 84 | {% set ns.host = "*." + wildcard_domain %} 85 | {# use same certificate as root domain #} 86 | {%- endif %} 87 | {%- endfor %} 88 | ssl_certificate {{ nginx_directory }}/hosts/{{ ns.host }}/fullchain.pem; 89 | ssl_certificate_key {{ nginx_directory }}/hosts/{{ ns.host }}/privkey.pem; 90 | ssl_trusted_certificate {{ nginx_directory }}/hosts/{{ ns.host }}/chain.pem; 91 | 92 | ssl_session_timeout 1d; 93 | ssl_session_cache shared:MozSSL:10m; # about 40000 sessions 94 | ssl_session_tickets off; 95 | 96 | ssl_protocols {{ tls_protocols or 'TLSv1.2 TLSv1.3' }}; 97 | ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; 98 | ssl_prefer_server_ciphers off; 99 | 100 | ssl_stapling on; 101 | ssl_stapling_verify on; 102 | 103 | resolver 8.8.8.8 8.8.4.4 1.1.1.1 1.0.0.1 208.67.222.222 208.67.220.220 valid=600s; 104 | resolver_timeout 30s; 105 | 106 | client_max_body_size 250M; 107 | client_body_buffer_size 16K; 108 | client_header_buffer_size 1k; 109 | 110 | more_set_headers "X-Frame-Options: SAMEORIGIN"; 111 | more_set_headers "X-XSS-Protection: 1; mode=block"; 112 | more_set_headers "X-Content-Type-Options: nosniff"; 113 | more_set_headers "Referrer-Policy: no-referrer-when-downgrade"; 114 | more_set_headers "Strict-Transport-Security: max-age=31536000; includeSubDomains; preload"; 115 | 116 | {% if host_options.redirect %} 117 | 118 | location / { 119 | return 301 https://{{ host_options.redirect }}$request_uri; 120 | } 121 | 122 | {% else %} 123 | location /assets/ { 124 | proxy_http_version 1.1; 125 | 126 | proxy_cache proxy_cache_zone; 127 | proxy_cache_key $scheme$host$request_uri; 128 | proxy_cache_valid 200 302 2m; 129 | proxy_cache_valid 404 1m; 130 | proxy_cache_bypass $http_secret_cache_purge; 131 | 132 | proxy_set_header Host $host; 133 | proxy_set_header X-Real-IP $remote_addr; 134 | proxy_set_header X-Forwarded-For $remote_addr; 135 | proxy_set_header X-Forwarded-Proto $scheme; 136 | proxy_set_header Connection ""; 137 | 138 | more_set_headers "X-Proxy-Upstream: $upstream_server_hash"; 139 | more_set_headers "X-Proxy-Cache: $upstream_cache_status"; 140 | 141 | proxy_pass $upstream_server_hash; 142 | } 143 | 144 | location /socket.io { 145 | proxy_http_version 1.1; 146 | 147 | proxy_set_header Upgrade $http_upgrade; 148 | proxy_set_header Host $host; 149 | proxy_set_header X-Real-IP $remote_addr; 150 | proxy_set_header X-Forwarded-For $remote_addr; 151 | proxy_set_header X-Forwarded-Proto $scheme; 152 | proxy_set_header Connection "upgrade"; 153 | 154 | more_set_headers "X-Proxy-Upstream: $upstream_server_hash"; 155 | 156 | proxy_pass $upstream_server_hash; 157 | } 158 | 159 | location / { 160 | proxy_http_version 1.1; 161 | 162 | proxy_read_timeout 600; 163 | proxy_buffering off; 164 | proxy_buffer_size 128k; 165 | proxy_buffers 100 128k; 166 | 167 | proxy_set_header Host $host; 168 | proxy_set_header X-Real-IP $remote_addr; 169 | proxy_set_header X-Forwarded-For $remote_addr; 170 | proxy_set_header X-Forwarded-Proto $scheme; 171 | {% if host_options.codeserver %} 172 | proxy_set_header Connection "upgrade"; 173 | proxy_set_header Upgrade $http_upgrade; 174 | proxy_set_header Connection upgrade; 175 | proxy_set_header Accept-Encoding gzip; 176 | {% else %} 177 | proxy_set_header Connection ""; 178 | {% endif %} 179 | 180 | more_set_headers "X-Proxy-Upstream: $upstream_server_hash"; 181 | 182 | more_set_headers "Access-Control-Allow-Origin: $access_control_allow_origin"; 183 | more_set_headers "Access-Control-Allow-Headers: $access_control_allow_headers"; 184 | more_set_headers "Access-Control-Allow-Credentials: $access_control_allow_credentials"; 185 | more_set_headers "Access-Control-Allow-Methods: $access_control_allow_methods"; 186 | 187 | proxy_pass $upstream_server_hash; 188 | } 189 | 190 | proxy_intercept_errors on; 191 | error_page 429 /exceeded.html; 192 | location /exceeded.html { 193 | root {{ error_pages_directory }}; 194 | internal; 195 | } 196 | {% endif %} 197 | } 198 | 199 | {% endfor %} 200 | 201 | server { 202 | listen 80; 203 | server_name _; 204 | 205 | location ^~ /.well-known/acme-challenge/ { 206 | return 301 http://ssl.{{ domain }}$request_uri; 207 | } 208 | location / { 209 | return 301 https://$host$request_uri; 210 | } 211 | } 212 | 213 | 214 | server { 215 | listen 127.0.0.1:10090; 216 | location / { 217 | return 307 https://{{ domain }}/dashboard/#/sites/new?domain=$host; 218 | } 219 | } 220 | 221 | server { 222 | listen 127.0.0.1:10091; 223 | 224 | root {{ error_pages_directory }}; 225 | error_page 503 /deactivated.html; 226 | 227 | location / { 228 | return 503; 229 | } 230 | 231 | location = /deactivated.html { 232 | internal; 233 | } 234 | } 235 | 236 | 237 | server { 238 | listen 127.0.0.1:10092; 239 | 240 | root {{ error_pages_directory }}; 241 | error_page 402 /suspended.html; 242 | 243 | location / { 244 | return 402; 245 | } 246 | 247 | location = /suspended.html { 248 | internal; 249 | } 250 | } 251 | 252 | 253 | server { 254 | listen 127.0.0.1:10093; 255 | 256 | root {{ error_pages_directory }}; 257 | error_page 402 /suspended_saas.html; 258 | 259 | location / { 260 | return 402; 261 | } 262 | 263 | location = /suspended_saas.html { 264 | internal; 265 | } 266 | 267 | } 268 | -------------------------------------------------------------------------------- /agent/tests/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | -------------------------------------------------------------------------------- /agent/tests/test_proxy.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | import os 5 | import shutil 6 | import unittest 7 | from unittest.mock import patch 8 | 9 | from agent.proxy import Proxy 10 | 11 | 12 | class TestProxy(unittest.TestCase): 13 | """Tests for class methods of Proxy.""" 14 | 15 | def _create_needed_files(self): 16 | """Create host dirs for 2 domains test json files.""" 17 | os.makedirs(os.path.join(self.hosts_directory, self.domain_1), exist_ok=True) 18 | os.makedirs(os.path.join(self.hosts_directory, self.domain_2), exist_ok=True) 19 | os.makedirs(self.upstreams_directory, exist_ok=True) 20 | 21 | map_1 = os.path.join(self.hosts_directory, self.domain_1, "map.json") 22 | map_2 = os.path.join(self.hosts_directory, self.domain_2, "map.json") 23 | 24 | with open(map_1, "w") as m: 25 | json.dump({self.domain_1: self.default_domain}, m) 26 | with open(map_2, "w") as m: 27 | json.dump({self.domain_2: self.default_domain}, m) 28 | 29 | def setUp(self): 30 | self.test_dir = "test_dir" 31 | if os.path.exists(self.test_dir): 32 | raise FileExistsError( 33 | f""" 34 | Directory {self.test_dir} exists. This directory will be used 35 | for running tests and will be deleted 36 | """ 37 | ) 38 | 39 | self.default_domain = "xxx.frappe.cloud" 40 | self.domain_1 = "balu.codes" 41 | self.domain_2 = "www.balu.codes" 42 | self.tld = "frappe.cloud" 43 | 44 | self.hosts_directory = os.path.join(self.test_dir, "nginx/hosts") 45 | self.upstreams_directory = os.path.join(self.test_dir, "nginx/upstreams") 46 | self._create_needed_files() 47 | 48 | def tearDown(self): 49 | shutil.rmtree(self.test_dir) 50 | 51 | def _get_fake_proxy(self): 52 | """Get Proxy object with only config and hosts_directory attrs.""" 53 | with patch.object(Proxy, "__init__", new=lambda x: None): 54 | proxy = Proxy() 55 | proxy.hosts_directory = self.hosts_directory 56 | proxy.nginx_directory = os.path.join(self.test_dir, "nginx") 57 | proxy._proxy_config_modification_lock = None 58 | return proxy 59 | 60 | def test_hosts_redirects_default_domain(self): 61 | """ 62 | Ensure hosts property redirects default domain when redirect.json is 63 | present. 64 | """ 65 | proxy = self._get_fake_proxy() 66 | os.makedirs(os.path.join(self.hosts_directory, self.default_domain)) 67 | redirect_file = os.path.join(self.hosts_directory, self.default_domain, "redirect.json") 68 | with open(redirect_file, "w") as r: 69 | json.dump({self.default_domain: self.domain_1}, r) 70 | 71 | self.assertLessEqual( 72 | { 73 | self.default_domain: { 74 | "redirect": self.domain_1, 75 | "codeserver": False, 76 | } 77 | }.items(), 78 | proxy.hosts.items(), 79 | ) 80 | 81 | def _test_add_host(self, proxy, host): 82 | # TODO: test contents of map.json and certificate dirs 83 | with patch.object(Proxy, "add_host", new=Proxy.add_host.__wrapped__): 84 | # get undecorated method with __wrapped__ 85 | proxy.add_host(host, "www.test.com", {}) 86 | 87 | self.assertTrue(os.path.exists(os.path.join(proxy.hosts_directory, host, "map.json"))) 88 | 89 | def test_add_hosts_works_without_hosts_dir(self): 90 | """Ensure add_host works when hosts directory doesn't exist.""" 91 | proxy = self._get_fake_proxy() 92 | shutil.rmtree(proxy.hosts_directory) 93 | self._test_add_host(proxy, "test.com") 94 | 95 | def test_add_hosts_works_with_hosts_dir(self): 96 | """Ensure add_host works when hosts directory exists.""" 97 | proxy = self._get_fake_proxy() 98 | self._test_add_host(proxy, "test.com") 99 | 100 | def test_add_hosts_works_with_host_dir(self): 101 | """Ensure add_host works when host directory of host exists.""" 102 | proxy = self._get_fake_proxy() 103 | host = "test.com" 104 | host_directory = os.path.join(proxy.hosts_directory, host) 105 | os.mkdir(host_directory) 106 | self._test_add_host(proxy, host) 107 | 108 | def _test_add_upstream(self, proxy, upstream): 109 | upstream_dir = os.path.join(proxy.upstreams_directory, upstream) 110 | with patch.object(Proxy, "add_upstream", new=Proxy.add_upstream.__wrapped__): 111 | # get undecorated method with __wrapped__ 112 | proxy.add_upstream(upstream) 113 | self.assertTrue(os.path.exists(upstream_dir)) 114 | 115 | def test_add_upstream_works_with_upstreams_dir(self): 116 | """Ensure add_upstream works when upstreams directory exists.""" 117 | proxy = self._get_fake_proxy() 118 | proxy.upstreams_directory = self.upstreams_directory 119 | self._test_add_upstream(proxy, "0.0.0.0") 120 | 121 | def test_add_upstream_works_without_upstreams_dir(self): 122 | """Ensure add_upstream works when upstreams directory doesn't exist.""" 123 | proxy = self._get_fake_proxy() 124 | proxy.upstreams_directory = self.upstreams_directory 125 | os.rmdir(proxy.upstreams_directory) 126 | self._test_add_upstream(proxy, "0.0.0.0") 127 | 128 | def test_remove_redirect_for_default_domain_deletes_host_dir(self): 129 | """Ensure removing redirect of default domain deletes the host dir.""" 130 | proxy = self._get_fake_proxy() 131 | proxy.domain = self.tld 132 | host_dir = os.path.join(self.hosts_directory, self.default_domain) 133 | os.makedirs(host_dir) 134 | redir_file = os.path.join(host_dir, "redirect.json") 135 | with open(redir_file, "w") as r: 136 | json.dump({self.default_domain: self.domain_1}, r) 137 | 138 | with patch.object(Proxy, "remove_redirect", new=Proxy.remove_redirect.__wrapped__): 139 | proxy.remove_redirect(self.default_domain) 140 | self.assertFalse(os.path.exists(redir_file)) 141 | self.assertFalse(os.path.exists(host_dir)) 142 | 143 | def test_setup_redirect_creates_redirect_json_for_given_hosts(self): 144 | """Ensure setup redirect creates redirect.json files""" 145 | proxy = self._get_fake_proxy() 146 | proxy.domain = self.tld 147 | host = self.domain_2 148 | target = self.domain_1 149 | with patch.object(Proxy, "setup_redirect", new=Proxy.setup_redirect.__wrapped__): 150 | proxy.setup_redirect(host, target) 151 | host_dir = os.path.join(proxy.hosts_directory, host) 152 | redir_file = os.path.join(host_dir, "redirect.json") 153 | self.assertTrue(os.path.exists(redir_file)) 154 | 155 | def test_remove_redirect_deletes_redirect_json_for_given_hosts(self): 156 | """Ensure remove redirect deletes redirect.json files""" 157 | proxy = self._get_fake_proxy() 158 | proxy.domain = self.tld 159 | host = self.domain_2 160 | target = self.domain_1 161 | with patch.object(Proxy, "setup_redirect", new=Proxy.setup_redirect.__wrapped__): 162 | proxy.setup_redirect(host, target) 163 | # assume that setup redirects works properly based on previous test 164 | with patch.object(Proxy, "remove_redirect", new=Proxy.remove_redirect.__wrapped__): 165 | proxy.remove_redirect(host) 166 | host_dir = os.path.join(proxy.hosts_directory, host) 167 | redir_file = os.path.join(host_dir, "redirect.json") 168 | self.assertFalse(os.path.exists(redir_file)) 169 | 170 | def test_rename_on_site_host_renames_host_directory(self): 171 | """Ensure rename site renames host directory.""" 172 | proxy = self._get_fake_proxy() 173 | old_host_dir = os.path.join(proxy.hosts_directory, self.default_domain) 174 | os.makedirs(old_host_dir) 175 | with patch.object( 176 | Proxy, 177 | "rename_host_dir", 178 | new=Proxy.rename_host_dir.__wrapped__, 179 | ): 180 | proxy.rename_host_dir(self.default_domain, "yyy.frappe.cloud") 181 | new_host_dir = os.path.join(proxy.hosts_directory, "yyy.frappe.cloud") 182 | self.assertFalse(os.path.exists(old_host_dir)) 183 | self.assertTrue(os.path.exists(new_host_dir)) 184 | 185 | def test_rename_on_site_host_renames_redirect_json(self): 186 | """Ensure rename site updates redirect.json if exists.""" 187 | proxy = self._get_fake_proxy() 188 | old_host_dir = os.path.join(proxy.hosts_directory, self.default_domain) 189 | os.makedirs(old_host_dir) 190 | redirect_file = os.path.join(old_host_dir, "redirect.json") 191 | with open(redirect_file, "w") as r: 192 | json.dump({self.default_domain: self.domain_1}, r) 193 | with patch.object( 194 | Proxy, 195 | "rename_host_dir", 196 | new=Proxy.rename_host_dir.__wrapped__, 197 | ): 198 | proxy.rename_host_dir(self.default_domain, "yyy.frappe.cloud") 199 | with patch.object( 200 | Proxy, 201 | "rename_site_in_host_dir", 202 | new=Proxy.rename_site_in_host_dir.__wrapped__, 203 | ): 204 | proxy.rename_site_in_host_dir("yyy.frappe.cloud", self.default_domain, "yyy.frappe.cloud") 205 | new_host_dir = os.path.join(proxy.hosts_directory, "yyy.frappe.cloud") 206 | redirect_file = os.path.join(new_host_dir, "redirect.json") 207 | with open(redirect_file) as r: 208 | self.assertDictEqual(json.load(r), {"yyy.frappe.cloud": self.domain_1}) 209 | 210 | def test_rename_updates_map_json_of_custom(self): 211 | """Ensure custom domains have map.json updated on site rename.""" 212 | proxy = self._get_fake_proxy() 213 | with patch.object( 214 | Proxy, 215 | "rename_site_in_host_dir", 216 | new=Proxy.rename_site_in_host_dir.__wrapped__, 217 | ): 218 | proxy.rename_site_in_host_dir(self.domain_1, self.default_domain, "yyy.frappe.cloud") 219 | host_directory = os.path.join(proxy.hosts_directory, self.domain_1) 220 | map_file = os.path.join(host_directory, "map.json") 221 | with open(map_file) as m: 222 | self.assertDictEqual(json.load(m), {self.domain_1: "yyy.frappe.cloud"}) 223 | 224 | def test_rename_updates_redirect_json_of_custom(self): 225 | """Ensure redirect.json updated for domains redirected to default.""" 226 | proxy = self._get_fake_proxy() 227 | host_directory = os.path.join(proxy.hosts_directory, self.domain_1) 228 | redirect_file = os.path.join(host_directory, "redirect.json") 229 | with open(redirect_file, "w") as r: 230 | json.dump({self.domain_1: self.default_domain}, r) 231 | with patch.object( 232 | Proxy, 233 | "rename_site_in_host_dir", 234 | new=Proxy.rename_site_in_host_dir.__wrapped__, 235 | ): 236 | proxy.rename_site_in_host_dir(self.domain_1, self.default_domain, "yyy.frappe.cloud") 237 | redirect_file = os.path.join(host_directory, "redirect.json") 238 | with open(redirect_file) as r: 239 | self.assertDictEqual(json.load(r), {self.domain_1: "yyy.frappe.cloud"}) 240 | 241 | def test_rename_does_not_update_redirect_json_of_custom(self): 242 | """Test redirects not updated for domains not redirected to default.""" 243 | proxy = self._get_fake_proxy() 244 | host_directory = os.path.join(proxy.hosts_directory, self.domain_1) 245 | redirect_file = os.path.join(host_directory, "redirect.json") 246 | original_dict = {self.domain_1: self.domain_2} 247 | with open(redirect_file, "w") as r: 248 | json.dump(original_dict, r) 249 | with patch.object( 250 | Proxy, 251 | "rename_site_in_host_dir", 252 | new=Proxy.rename_site_in_host_dir.__wrapped__, 253 | ): 254 | proxy.rename_site_in_host_dir(self.domain_1, self.default_domain, "yyy.frappe.cloud") 255 | with open(redirect_file) as r: 256 | self.assertDictEqual(json.load(r), original_dict) 257 | 258 | def test_rename_does_not_update_partial_strings(self): 259 | """Test rename doesn't update part of other custom domains.""" 260 | proxy = self._get_fake_proxy() 261 | custom_domain = self.default_domain + ".balu.codes" 262 | self.domain_1 = custom_domain 263 | self._create_needed_files() 264 | with patch.object( 265 | Proxy, 266 | "rename_site_in_host_dir", 267 | new=Proxy.rename_site_in_host_dir.__wrapped__, 268 | ): 269 | proxy.rename_site_in_host_dir(self.domain_1, self.default_domain, "yyy.frappe.cloud") 270 | host_dir = os.path.join(self.hosts_directory, self.domain_1) 271 | map_file = os.path.join(host_dir, "map.json") 272 | with open(map_file) as m: 273 | self.assertDictEqual(json.load(m), {self.domain_1: "yyy.frappe.cloud"}) 274 | -------------------------------------------------------------------------------- /agent/tests/test_site.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | import os 5 | import shutil 6 | import unittest 7 | from unittest.mock import patch 8 | 9 | from agent.base import AgentException 10 | from agent.bench import Bench 11 | from agent.server import Server 12 | from agent.site import Site 13 | 14 | 15 | class TestSite(unittest.TestCase): 16 | """Tests for class methods of Site.""" 17 | 18 | def _create_needed_paths(self): 19 | os.makedirs(self.sites_directory) 20 | os.makedirs(self.apps_directory) 21 | os.makedirs(self.assets_dir) 22 | with open(self.common_site_config, "w") as c: 23 | json.dump({}, c) 24 | with open(self.bench_config, "w") as c: 25 | json.dump({"docker_image": "fake_img_url"}, c) 26 | with open(self.apps_txt, "w") as a: 27 | a.write("frappe\n") 28 | a.write("erpnext\n") 29 | 30 | def _make_site_config(self, site_name: str, content: str | None = None): 31 | if not content: 32 | content = json.dumps({"db_name": "fake_db_name", "db_password": "fake_db_password"}) 33 | site_config = os.path.join(self.sites_directory, site_name, "site_config.json") 34 | with open(site_config, "w") as s: 35 | s.write(content) 36 | 37 | def setUp(self): 38 | self.test_dir = "test_dir" 39 | if os.path.exists(self.test_dir): 40 | raise FileExistsError( 41 | f""" 42 | Directory {self.test_dir} exists. This directory will be used 43 | for running tests and will be deleted. 44 | """ 45 | ) 46 | 47 | self.bench_name = "test-bench" 48 | self.benches_directory = os.path.join(self.test_dir, "benches") 49 | self.bench_dir = os.path.join(self.benches_directory, self.bench_name) 50 | 51 | self.sites_directory = os.path.join(self.bench_dir, "sites") 52 | self.apps_directory = os.path.join(self.bench_dir, "apps") 53 | self.common_site_config = os.path.join(self.sites_directory, "common_site_config.json") 54 | self.bench_config = os.path.join(self.bench_dir, "config.json") 55 | self.apps_txt = os.path.join(self.sites_directory, "apps.txt") 56 | self.assets_dir = os.path.join(self.sites_directory, "assets") 57 | 58 | self._create_needed_paths() 59 | 60 | def tearDown(self): 61 | shutil.rmtree(self.test_dir) 62 | 63 | def _create_test_site(self, site_name: str): 64 | site_dir = os.path.join(self.sites_directory, site_name) 65 | os.makedirs(site_dir) 66 | site_config = os.path.join(site_dir, "site_config.json") 67 | with open(site_config, "w") as s: 68 | json.dump({}, s) 69 | 70 | def _get_test_bench(self) -> Bench: 71 | with patch.object(Server, "__init__", new=lambda x: None): 72 | server = Server() 73 | server.benches_directory = self.benches_directory 74 | return Bench(self.bench_name, server) 75 | 76 | def test_rename_site_works_for_existing_site(self): 77 | """Ensure rename_site renames site.""" 78 | bench = self._get_test_bench() 79 | old_name = "old-site-name" 80 | self._create_test_site(old_name) 81 | with patch.object(Site, "config"): 82 | site = Site(old_name, bench) 83 | new_name = "new-site-name" 84 | 85 | with patch.object(Site, "rename", new=Site.rename.__wrapped__): 86 | site.rename(new_name) 87 | 88 | self.assertTrue(os.path.exists(os.path.join(self.sites_directory, new_name))) 89 | self.assertFalse(os.path.exists(os.path.join(self.sites_directory, old_name))) 90 | 91 | @unittest.skip("fails with 'Server' has no attr 'job'") 92 | def test_valid_sites_property_of_bench_throws_if_site_config_is_corrupt( 93 | self, 94 | ): 95 | bench = self._get_test_bench() 96 | site_name = "corrupt-site.frappe.cloud" 97 | self._create_test_site(site_name) 98 | self._make_site_config( 99 | site_name, 100 | content=""" 101 | { 102 | "db_name": "fake_db_name", 103 | "db_password": "fake_db_password" 104 | "some_key": "some_value" 105 | } 106 | """, # missing comma above 107 | ) 108 | with self.assertRaises(AgentException): 109 | bench._sites(validate_configs=True) 110 | 111 | def test_valid_sites_property_of_bench_doesnt_throw_for_assets_apps_txt( 112 | self, 113 | ): 114 | bench = self._get_test_bench() 115 | site_name = "corrupt-site.frappe.cloud" 116 | self._create_test_site(site_name) 117 | self._make_site_config(site_name) 118 | try: 119 | bench._sites() 120 | except AgentException: 121 | self.fail("sites property of bench threw error for assets and apps.txt") 122 | self.assertEqual(len(bench.sites), 1) 123 | try: 124 | bench.valid_sites[site_name] 125 | except KeyError: 126 | self.fail("Site not found in bench.sites") 127 | -------------------------------------------------------------------------------- /agent/usage.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | import os 5 | import sys 6 | import traceback 7 | from datetime import datetime 8 | 9 | from agent.server import Server 10 | 11 | 12 | def cstr(text, encoding="utf-8"): 13 | """Similar to frappe.utils.cstr""" 14 | if isinstance(text, str): 15 | return text 16 | if text is None: 17 | return "" 18 | if isinstance(text, bytes): 19 | return str(text, encoding) 20 | return str(text) 21 | 22 | 23 | def get_traceback(): 24 | """Returns the traceback of the Exception""" 25 | exc_type, exc_value, exc_tb = sys.exc_info() 26 | trace_list = traceback.format_exception(exc_type, exc_value, exc_tb) 27 | return "".join(cstr(t) for t in trace_list) 28 | 29 | 30 | if __name__ == "__main__": 31 | info = [] 32 | server = Server() 33 | time = datetime.utcnow().isoformat() 34 | target_file = os.path.join( 35 | server.directory, 36 | "logs", 37 | f"{server.name}-usage-{time}.json.log", 38 | ) 39 | 40 | for bench in server.benches.values(): 41 | for site in bench.sites.values(): 42 | try: 43 | info.append( 44 | { 45 | "site": site.name, 46 | "timestamp": str(datetime.utcnow()), 47 | "timezone": site.timezone, 48 | **site.get_usage(), 49 | } 50 | ) 51 | except Exception: 52 | error_log = f"ERROR [{site.name}:{time}]: {get_traceback()}" 53 | print(error_log, file=sys.stderr) 54 | 55 | with open(target_file, "w") as f: 56 | json.dump(info, f, indent=1) 57 | -------------------------------------------------------------------------------- /agent/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import hashlib 4 | import os 5 | import re 6 | import subprocess 7 | from collections import defaultdict 8 | from datetime import datetime, timedelta 9 | from math import ceil 10 | from typing import TYPE_CHECKING 11 | from urllib.parse import urlparse 12 | 13 | import requests 14 | 15 | if TYPE_CHECKING: 16 | from typing import Literal, TypedDict 17 | 18 | ExecutionStatus = Literal[ 19 | "Pending", 20 | "Running", 21 | "Success", 22 | "Failure", 23 | ] 24 | 25 | class ExecutionResult(TypedDict): 26 | command: str 27 | directory: str 28 | start: datetime 29 | status: ExecutionStatus 30 | end: datetime | None 31 | duration: timedelta | None 32 | output: str | None 33 | returncode: int | None 34 | traceback: str | None 35 | 36 | 37 | def download_file(url, prefix): 38 | """Download file locally under path prefix and return local path""" 39 | filename = urlparse(url).path.split("/")[-1] 40 | local_filename = os.path.join(prefix, filename) 41 | 42 | with requests.get(url, stream=True) as r: 43 | total_size = int(r.headers.get("content-length", 0)) 44 | chunk_size = 1024 * 1024 if total_size > (100 * 1024 * 1024) else 8192 45 | r.raise_for_status() 46 | with open(local_filename, "wb") as f: 47 | for chunk in r.iter_content(chunk_size=chunk_size): 48 | f.write(chunk) 49 | 50 | return local_filename 51 | 52 | 53 | def get_size(folder, ignore_dirs=None): 54 | """Returns the size of the folder in bytes. Ignores symlinks""" 55 | total_size = os.path.getsize(folder) 56 | 57 | if ignore_dirs is None: 58 | ignore_dirs = [] 59 | 60 | for item in os.listdir(folder): 61 | itempath = os.path.join(folder, item) 62 | 63 | if item in ignore_dirs: 64 | continue 65 | 66 | if not os.path.islink(itempath): 67 | if os.path.isfile(itempath): 68 | total_size += os.path.getsize(itempath) 69 | elif os.path.isdir(itempath): 70 | total_size += get_size(itempath) 71 | 72 | return total_size 73 | 74 | 75 | def cint(x): 76 | """Convert to integer""" 77 | if x is None: 78 | return 0 79 | try: 80 | num = int(float(x)) 81 | except Exception: 82 | num = 0 83 | return num 84 | 85 | 86 | def b2mb(x): 87 | """Return B value in MiB""" 88 | return ceil(cint(x) / (1024**2)) 89 | 90 | 91 | def get_timestamp(): 92 | try: 93 | from datetime import UTC, datetime 94 | 95 | return str(datetime.now(UTC)) 96 | except Exception: 97 | from datetime import datetime 98 | 99 | return str(datetime.utcnow()) 100 | 101 | 102 | def get_execution_result( 103 | command: str = "", 104 | directory: str = "", 105 | start: datetime | None = None, 106 | status: ExecutionStatus | None = None, 107 | ) -> ExecutionResult: 108 | """returns an ExecutionResult object to manage the output of a Job Step""" 109 | return { 110 | "command": command, 111 | "directory": directory, 112 | "start": start or datetime.now(), 113 | "status": status or "Running", 114 | "output": "", 115 | } 116 | 117 | 118 | def end_execution( 119 | res: ExecutionResult, 120 | output: str = "", 121 | status: ExecutionStatus | None = None, 122 | ): 123 | """updates ExecutionResult object `res` fields and returns it""" 124 | assert res["start"] is not None 125 | 126 | res["end"] = datetime.now() 127 | res["duration"] = res["end"] - res["start"] 128 | res["status"] = status or "Success" 129 | res["output"] = output or res["output"] 130 | return res 131 | 132 | 133 | def compute_file_hash(file_path, algorithm="sha256", raise_exception=True): 134 | try: 135 | """Compute the hash of a file using the specified algorithm.""" 136 | hash_func = hashlib.new(algorithm) 137 | 138 | with open(file_path, "rb") as file: 139 | # read in 10MB chunks 140 | while chunk := file.read(10000000): 141 | hash_func.update(chunk) 142 | 143 | return hash_func.hexdigest() 144 | except FileNotFoundError: 145 | if raise_exception: 146 | raise 147 | return "File does not exist" 148 | except Exception: 149 | if raise_exception: 150 | raise 151 | return "Failed to compute hash" 152 | 153 | 154 | def decode_mariadb_filename(filename: str) -> str: 155 | """ 156 | Decode MariaDB encoded filenames that use @XXXX format for special characters. 157 | """ 158 | 159 | def _hex_to_char(match: re.Match) -> str: 160 | # Convert the hex value after @ to its character representation 161 | hex_value = match.group(1) 162 | return chr(int(hex_value, 16)) 163 | 164 | # Find @XXXX patterns and replace them with their character equivalents 165 | return re.sub(r"@([0-9A-Fa-f]{4})", _hex_to_char, filename) 166 | 167 | 168 | def get_mariadb_table_name_from_path(path: str) -> str: 169 | """ 170 | Extract the table name from a MariaDB table file path. 171 | """ 172 | # Extract the filename from the path 173 | filename = os.path.basename(path) 174 | if not filename: 175 | return "" 176 | # Remove the extension 177 | filename = os.path.splitext(filename)[0] 178 | # Decode the filename 179 | return decode_mariadb_filename(filename) 180 | 181 | 182 | def check_installed_pyspy(server_dir: str) -> bool: 183 | return os.path.exists(os.path.join(server_dir, "env/bin/py-spy")) 184 | 185 | 186 | def get_supervisor_processes_status() -> dict[str, str | dict[str, str]]: 187 | try: 188 | output = subprocess.check_output("sudo supervisorctl status all", shell=True) 189 | lines = output.decode("utf-8").strip().split("\n") 190 | 191 | flat_status = {} 192 | 193 | for line in lines: 194 | parts = line.split() 195 | if len(parts) < 2: 196 | continue 197 | name = parts[0].strip() 198 | state = parts[1].strip() 199 | 200 | if not name.startswith("agent:"): 201 | continue 202 | 203 | # Strip `agent:` prefix if present 204 | name = name[len("agent:") :] 205 | 206 | flat_status[name] = state 207 | 208 | nested_status = defaultdict(dict) 209 | 210 | for name, state in flat_status.items(): 211 | # Match pattern like worker-1, worker-2, etc. 212 | if "-" in name: 213 | group, sub = name.split("-", 1) 214 | nested_status[group][sub] = state 215 | else: 216 | nested_status[name] = state 217 | 218 | return dict(nested_status) 219 | except Exception: 220 | return {} 221 | -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | pre-commit 2 | black 3 | testcontainers[mysql]==3.7.1 4 | cryptography 5 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | redis: 5 | image: redis:5.0.8 6 | restart: always 7 | 8 | nginx: 9 | image: nginx:1.17.9 10 | restart: always 11 | 12 | web: 13 | image: frappe:agent 14 | restart: always 15 | 16 | worker: 17 | image: frappe:agent 18 | restart: always 19 | -------------------------------------------------------------------------------- /redis.conf: -------------------------------------------------------------------------------- 1 | dbfilename redis.rdb 2 | bind 127.0.0.1 3 | port 25025 4 | appendonly yes 5 | appendfilename "redis.aof" 6 | save 60 1 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | boto3==1.18.0 2 | certifi==2019.11.28 3 | chardet==3.0.4 4 | click==8.1.3 5 | filewarmer==0.0.17 6 | Flask==1.1.1 7 | gunicorn==20.0.4 8 | honcho==1.0.1 9 | idna==2.8 10 | ipython==8.12.3 11 | itsdangerous==1.1.0 12 | Jinja2==2.10.3 13 | MarkupSafe==1.1.1 14 | passlib==1.7.2 15 | peewee==3.13.1 16 | prometheus_client==0.20.0 17 | PyMySQL==0.9.3 18 | python-crontab==2.5.1 19 | python-dateutil==2.8.2 20 | requests==2.26.0 21 | rq==1.13.0 22 | urllib3==1.26.18 23 | Werkzeug==0.16.0 24 | wrapt==1.16.0 25 | docker==6.1.2 26 | filelock==3.13.1 27 | sentry-sdk[flask, rq]==2.1.1 28 | sql_metadata==2.15.0 29 | py-spy==0.4.0 30 | mariadb-binlog-indexer==0.0.8 31 | psutil==7.0.0 -------------------------------------------------------------------------------- /ruff.toml: -------------------------------------------------------------------------------- 1 | line-length = 110 2 | indent-width = 4 3 | target-version = "py38" 4 | 5 | [format] 6 | quote-style = "double" 7 | indent-style = "space" 8 | 9 | 10 | [lint] 11 | select = ["F", "E", "W", "I", "UP", "B", "RUF", "FA", "TCH", "C90", "RET", "SIM"] 12 | ignore = [ 13 | "UP015", # Unnecessary open mode parameters 14 | "SIM108", # Use ternary operator {contents} instead of if-else-block 15 | ] 16 | 17 | 18 | [lint.mccabe] 19 | # Flag errors (`C901`) whenever the complexity level exceeds 8 20 | max-complexity = 8 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | with open("requirements.txt") as f: 4 | install_requires = f.read().strip().split("\n") 5 | 6 | 7 | setup( 8 | name="agent", 9 | version="0.0.0", 10 | description="Frappe Press Agent", 11 | url="http://github.com/frappe/agent", 12 | author="Frappe Technologies", 13 | author_email="developers@frappe.io", 14 | packages=find_packages(), 15 | zip_safe=False, 16 | install_requires=install_requires, 17 | entry_points={ 18 | "console_scripts": [ 19 | "agent = agent.cli:cli", 20 | ], 21 | }, 22 | ) 23 | -------------------------------------------------------------------------------- /wait-for-it.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | URL=$1 4 | while : 5 | do 6 | if redis-cli -u $URL PING | grep -q PONG; then 7 | break 8 | fi 9 | sleep 1 10 | done 11 | --------------------------------------------------------------------------------