├── .dockerignore ├── .gitignore ├── .pylintrc ├── .travis.yml ├── Dockerfile ├── Dockerfile-with-code ├── LICENSE ├── MANIFEST.in ├── Makefile ├── Procfile ├── README.md ├── docs ├── best-practices.md ├── changelog.md ├── command-line.md ├── configuration.md ├── contributing.md ├── dashboard.md ├── dependencies.md ├── examples.md ├── get-started.md ├── index.md ├── io-monitoring.md ├── jobs-maintenance.md ├── jobs.md ├── license.md ├── memory-leaks.md ├── metrics.md ├── performance.md ├── queue-performance.md ├── queues.md ├── recurring-jobs.md ├── tests.md └── workers.md ├── examples ├── queue_performance │ ├── enqueue.py │ └── tasks.py ├── scheduler │ ├── README.md │ ├── config.py │ └── tasks.py ├── simple_crawler │ ├── README.md │ ├── crawler.py │ └── requirements.txt └── timed_set │ ├── README.md │ ├── config.py │ ├── enqueue_raw_jobs.py │ └── example.py ├── mkdocs.yml ├── mrq ├── __init__.py ├── agent.py ├── basetasks │ ├── __init__.py │ ├── cleaning.py │ ├── indexes.py │ ├── orchestrator.py │ └── utils.py ├── bin │ ├── __init__.py │ ├── mrq_agent.py │ ├── mrq_run.py │ └── mrq_worker.py ├── config.py ├── context.py ├── dashboard │ ├── __init__.py │ ├── app.py │ ├── static │ │ ├── bin │ │ │ ├── 0.bundle.js │ │ │ ├── 32941d6330044744c02493835b799e90.svg │ │ │ ├── 64f2d23d70cb2b2810031880f554b13c.png │ │ │ ├── 68ed1dac06bf0409c18ae7bc62889170.woff │ │ │ ├── 6c56b94fd0540844a7118cdff565b0ae.png │ │ │ ├── 7ad17c6085dee9a33787bac28fb23d46.eot │ │ │ ├── 8f88d990024975797f96ce7648dacd2f.png │ │ │ ├── 94b34ff5224ba38210d67623bb1a1504.png │ │ │ ├── bundle.js │ │ │ ├── d48475e6c742940f44e62622e16865b9.png │ │ │ └── e49d52e74b7689a0727def99da31f3eb.ttf │ │ ├── css │ │ │ ├── bootstrap-theme.min.css │ │ │ ├── datatables.css │ │ │ ├── jquery.circliful.css │ │ │ └── mrq-dashboard.css │ │ ├── fonts │ │ │ ├── glyphicons-halflings-regular.eot │ │ │ ├── glyphicons-halflings-regular.svg │ │ │ ├── glyphicons-halflings-regular.ttf │ │ │ └── glyphicons-halflings-regular.woff │ │ ├── images │ │ │ ├── favicon-old.ico │ │ │ ├── favicon.ico │ │ │ ├── sort_asc.png │ │ │ ├── sort_asc_disabled.png │ │ │ ├── sort_both.png │ │ │ ├── sort_desc.png │ │ │ └── sort_desc_disabled.png │ │ ├── js │ │ │ ├── app.js │ │ │ ├── main.js │ │ │ ├── models.js │ │ │ ├── router.js │ │ │ ├── vendor │ │ │ │ ├── backbone.min.js │ │ │ │ ├── backbone.queryparams.js │ │ │ │ ├── bootstrap.js │ │ │ │ ├── bootstrap.min.js │ │ │ │ ├── datatables.bs3.js │ │ │ │ ├── jquery-2.1.0.min.js │ │ │ │ ├── jquery.circliful.min.js │ │ │ │ ├── jquery.dataTables.min.js │ │ │ │ ├── jquery.sparkline.min.js │ │ │ │ ├── moment.min.js │ │ │ │ ├── require.js │ │ │ │ └── underscore.min.js │ │ │ └── views │ │ │ │ ├── agents.js │ │ │ │ ├── generic │ │ │ │ ├── datatablepage.js │ │ │ │ └── page.js │ │ │ │ ├── index.js │ │ │ │ ├── io.js │ │ │ │ ├── jobs.js │ │ │ │ ├── queues.js │ │ │ │ ├── root.js │ │ │ │ ├── scheduledjobs.js │ │ │ │ ├── status.js │ │ │ │ ├── taskexceptions.js │ │ │ │ ├── taskpaths.js │ │ │ │ ├── workergroups.js │ │ │ │ └── workers.js │ │ ├── package.json │ │ └── webpack.config.js │ ├── templates │ │ ├── index.html │ │ └── library.html │ ├── utils.py │ └── uwsgi-heroku.ini ├── exceptions.py ├── helpers.py ├── job.py ├── logger.py ├── monkey.py ├── processes.py ├── queue.py ├── queue_raw.py ├── queue_regular.py ├── redishelpers.py ├── scheduler.py ├── subpool.py ├── supervisor.py ├── task.py ├── utils.py ├── version.py └── worker.py ├── requirements-base.txt ├── requirements-dashboard.txt ├── requirements-dev.txt ├── requirements-heroku.txt ├── requirements.txt ├── scripts ├── git-set-file-times └── propagate_docs.py ├── setup.py └── tests ├── __init__.py ├── conftest.py ├── fixtures ├── config-io-hooks.py ├── config-logger-capped.py ├── config-logger.py ├── config-lostjobs.py ├── config-metric.py ├── config-multiredis.py ├── config-notify.py ├── config-raw1.py ├── config-retry1.py ├── config-scheduler-invalid1.py ├── config-scheduler-invalid2.py ├── config-scheduler-invalid3.py ├── config-scheduler-invalid4.py ├── config-scheduler1.py ├── config-scheduler2.py ├── config-scheduler3.py ├── config-scheduler5.py ├── config-scheduler6.py ├── config-scheduler7.py ├── config-scheduler8.py ├── config-shorttimeout.py ├── config1.py ├── config2.py ├── httpstatic │ ├── index.html │ └── nginx.conf └── standalone_script1.py ├── tasks ├── __init__.py ├── agent.py ├── concurrency.py ├── context.py ├── general.py ├── io.py ├── largefile.py ├── logger.py ├── mongodb.py └── redis.py ├── test_abort.py ├── test_agent.py ├── test_cancel.py ├── test_cli.py ├── test_config.py ├── test_context.py ├── test_disconnects.py ├── test_general.py ├── test_interrupts.py ├── test_io_hooks.py ├── test_jobaction.py ├── test_jobinspect.py ├── test_logger.py ├── test_memoryleaks.py ├── test_notify.py ├── test_parallel.py ├── test_pause.py ├── test_performance.py ├── test_processes.py ├── test_progress.py ├── test_queuesize.py ├── test_ratelimit.py ├── test_raw.py ├── test_retry.py ├── test_routes.py ├── test_scheduler.py ├── test_sorted.py ├── test_subpool.py ├── test_subqueues.py ├── test_timeout.py └── test_utils.py /.dockerignore: -------------------------------------------------------------------------------- 1 | venv 2 | pypy 3 | dist 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | var 14 | sdist 15 | develop-eggs 16 | .installed.cfg 17 | lib 18 | lib64 19 | __pycache__ 20 | .cache 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | 30 | # Translations 31 | *.mo 32 | 33 | # Mr Developer 34 | .mr.developer.cfg 35 | .project 36 | .pydevproject 37 | 38 | .idea/ 39 | 40 | venv 41 | pypy 42 | .DS_Store 43 | 44 | mrq-config.py 45 | dump.rdb 46 | supervisord.pid 47 | memory_traces 48 | mrq/dashboard/static/node_modules/ 49 | .vscode 50 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | 2 | [FORMAT] 3 | indent-string=' ' 4 | max-line-length=100 5 | 6 | [MESSAGES CONTROL] 7 | disable=star-args, invalid-name, missing-docstring, no-init, too-few-public-methods, too-many-locals, too-many-branches, locally-disabled, too-many-instance-attributes, too-many-arguments, too-many-public-methods, too-many-statements, cyclic-import 8 | 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | services: 3 | - docker 4 | env: 5 | - PYTHON_BIN=python 6 | - PYTHON_BIN=python3 7 | - PYTHON_BIN=/pypy/bin/pypy 8 | before_install: 9 | - docker ps 10 | - docker info 11 | - docker version 12 | - make docker 13 | script: 14 | - docker run -i -t -v `pwd`:/app:rw -w /app pricingassistant/mrq $PYTHON_BIN -m pylint 15 | --errors-only --init-hook="import sys; sys.path.append('.')" -d E1103 --rcfile .pylintrc 16 | mrq 17 | - docker run -i -t -v `pwd`:/app:rw -w /app pricingassistant/mrq $PYTHON_BIN -m pytest 18 | tests/ -v --junitxml=pytest-report.xml --cov mrq --cov-report term --timeout-method=thread 19 | --timeout=240 20 | notifications: 21 | slack: 22 | secure: gWywakbRDFB/fPCHO9gB8mpDuBn8tSQcza+JwJ4rFnr2icuGvmLIhu608g/W18XxqCz+MMhBeHizT7XTiNOW3+YSH/z3Wk4ARkN/BlL15sSrb5TUjb+4gOV9zPfuIs7Jh0xArxNnVjKdZ+dM/MBqYwa1oud53un2eF3Vnaipc6v34WWfFfkmOYecG5rFZSFYJlK/mYtujvzcGXavdFqEJb2jLg1JPKB69271QZ2VvrdxZd4Yt8LNbXri3NITZp2eabCNg5mSeIOxhPJnZcDfZh8IMJX0CssYt0Pdnm5+0ef1RMYQAEE+VnIJHaDlFiPdsKPuSk9JlhyzglxL8aGSyUECmp9N0B+xnrHblDsgYkCURyELciTUE+iCqgn7sIjweK6jqZySgyoAB3aXEvMTYhRreo5TmVwNgGpkikaipPuRPDVBMYzCjAc3rTYCz3AMlTt8kHlkR18TyocClyJAKvuuxyicvHBr+uNkNZvci5394lBGEyNrONvQ+xJCYikBOzCV0pk754oA13o1riLFTfFEJeHIBle9yUWya3AERWHOq/Fve3SzlYvE5Xhb2lPdW7y3/mRlhWOEqH9PDzB9n5hNyzNRU0uz2xOB6wRbMmJ/aDF8fAYcfqUS0/0hgeSyduRENRLXuvNj3X/UnATJgzmtDJZ49YQAGdXu2tMb2rg= 23 | on_success: change 24 | on_failure: always 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:buster-slim 2 | 3 | # 4 | # httpredir.debian.org is often unreliable 5 | # https://github.com/docker-library/buildpack-deps/issues/40 6 | # 7 | 8 | # RUN echo \ 9 | # 'deb ftp://ftp.us.debian.org/debian/ jessie main\n \ 10 | # deb ftp://ftp.us.debian.org/debian/ jessie-updates main\n \ 11 | # deb http://security.debian.org jessie/updates main\n' \ 12 | # > /etc/apt/sources.list 13 | 14 | RUN apt-get update && \ 15 | apt-get install -y --no-install-recommends \ 16 | curl \ 17 | gcc \ 18 | python-dev \ 19 | python-pip \ 20 | python-setuptools \ 21 | python3-pip \ 22 | python3-dev \ 23 | python3-setuptools \ 24 | make \ 25 | git \ 26 | vim \ 27 | bzip2 \ 28 | nginx redis-server \ 29 | g++ \ 30 | && \ 31 | apt-get clean -y && \ 32 | rm -rf /var/lib/apt/lists/* 33 | 34 | RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - 35 | RUN apt-get install -y --no-install-recommends nodejs 36 | 37 | RUN curl -sL https://www.mongodb.org/static/pgp/server-4.2.asc | apt-key add - 38 | RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.2 main" > /etc/apt/sources.list.d/mongodb-org-4.2.list 39 | RUN apt-get update && apt-get install -y --no-install-recommends mongodb-org 40 | 41 | # Download pypy 42 | RUN curl -sL 'https://github.com/squeaky-pl/portable-pypy/releases/download/pypy-7.2.0/pypy-7.2.0-linux_x86_64-portable.tar.bz2' > /pypy.tar.bz2 && tar jxvf /pypy.tar.bz2 && rm -rf /pypy.tar.bz2 && mv /pypy* /pypy 43 | 44 | # Upgrade pip 45 | RUN pip install --upgrade --ignore-installed pip 46 | RUN pip3 install --upgrade --ignore-installed pip 47 | RUN /pypy/bin/pypy -m ensurepip 48 | 49 | ADD requirements-heroku.txt /app/requirements-heroku.txt 50 | ADD requirements-base.txt /app/requirements-base.txt 51 | ADD requirements-dev.txt /app/requirements-dev.txt 52 | ADD requirements-dashboard.txt /app/requirements-dashboard.txt 53 | 54 | RUN python3 -m pip install -r /app/requirements-heroku.txt && \ 55 | python3 -m pip install -r /app/requirements-base.txt && \ 56 | python3 -m pip install -r /app/requirements-dev.txt && \ 57 | python3 -m pip install -r /app/requirements-dashboard.txt && \ 58 | rm -rf ~/.cache 59 | 60 | RUN python -m pip install -r /app/requirements-heroku.txt && \ 61 | python -m pip install -r /app/requirements-base.txt && \ 62 | python -m pip install -r /app/requirements-dev.txt && \ 63 | python -m pip install -r /app/requirements-dashboard.txt && \ 64 | rm -rf ~/.cache 65 | 66 | RUN /pypy/bin/pip install -r /app/requirements-heroku.txt && \ 67 | /pypy/bin/pip install -r /app/requirements-base.txt && \ 68 | /pypy/bin/pip install -r /app/requirements-dev.txt && \ 69 | /pypy/bin/pip install -r /app/requirements-dashboard.txt && \ 70 | rm -rf ~/.cache 71 | 72 | RUN mkdir -p /data/db 73 | 74 | RUN ln -s /app/mrq/bin/mrq_run.py /usr/bin/mrq-run 75 | RUN ln -s /app/mrq/bin/mrq_worker.py /usr/bin/mrq-worker 76 | RUN ln -s /app/mrq/bin/mrq_agent.py /usr/bin/mrq-agent 77 | RUN ln -s /app/mrq/dashboard/app.py /usr/bin/mrq-dashboard 78 | 79 | ENV PYTHONPATH /app 80 | 81 | VOLUME ["/data"] 82 | WORKDIR /app 83 | 84 | # Redis and MongoDB services 85 | EXPOSE 6379 27017 86 | 87 | # Dashboard, monitoring and docs 88 | EXPOSE 5555 20020 8000 89 | -------------------------------------------------------------------------------- /Dockerfile-with-code: -------------------------------------------------------------------------------- 1 | FROM pricingassistant/mrq-env:latest 2 | 3 | ADD ./mrq /app/mrq 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 pricingassistant 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.md 2 | include requirements* 3 | recursive-include mrq/dashboard/static * 4 | recursive-include mrq/dashboard/templates * 5 | recursive-exclude tests * -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | docker: 2 | docker build -t pricingassistant/mrq-env . 3 | docker build -t pricingassistant/mrq -f Dockerfile-with-code . 4 | 5 | docker_push: 6 | docker push pricingassistant/mrq-env:latest 7 | docker push pricingassistant/mrq:latest 8 | 9 | test: docker 10 | sh -c "docker run --rm -i -t -p 27017:27017 -p 6379:6379 -p 5555:5555 -p 20020:20020 -v `pwd`:/app:rw -w /app pricingassistant/mrq-env python -m pytest tests/ -v --instafail" 11 | 12 | test3: docker 13 | sh -c "docker run --rm -i -t -p 27017:27017 -p 6379:6379 -p 5555:5555 -p 20020:20020 -v `pwd`:/app:rw -w /app pricingassistant/mrq-env python3 -m pytest tests/ -v --instafail" 14 | 15 | testpypy: docker 16 | sh -c "docker run --rm -i -t -p 27017:27017 -p 6379:6379 -p 5555:5555 -p 20020:20020 -v `pwd`:/app:rw -w /app pricingassistant/mrq-env /pypy/bin/pypy -m pytest tests/ -v --instafail" 17 | 18 | shell: 19 | sh -c "docker run --rm -i -t -p 27017:27017 -p 6379:6379 -p 5555:5555 -p 20020:20020 -p 8000:8000 -v `pwd`:/app:rw -w /app pricingassistant/mrq-env bash" 20 | 21 | reshell: 22 | # Reconnect in the current taskqueue container 23 | sh -c 'docker exec -t -i `docker ps | grep pricingassistant/mrq-env | cut -f 1 -d " "` bash' 24 | 25 | shell_noport: 26 | sh -c "docker run --rm -i -t -v `pwd`:/app:rw -w /app pricingassistant/mrq-env bash" 27 | 28 | docs_serve: 29 | sh -c "docker run --rm -i -t -p 8000:8000 -v `pwd`:/app:rw -w /app pricingassistant/mrq-env mkdocs serve" 30 | 31 | lint: docker 32 | docker run -i -t -v `pwd`:/app:rw -w /app pricingassistant/mrq-env pylint -j 0 --init-hook="import sys; sys.path.append('.')" --rcfile .pylintrc mrq 33 | 34 | linterrors: docker 35 | docker run -i -t -v `pwd`:/app:rw -w /app pricingassistant/mrq-env pylint -j 0 --errors-only --init-hook="import sys; sys.path.append('.')" --rcfile .pylintrc mrq 36 | 37 | linterrors3: docker 38 | docker run -i -t -v `pwd`:/app:rw -w /app pricingassistant/mrq-env python3 -m pylint -j 0 --errors-only --init-hook="import sys; sys.path.append('.')" --rcfile .pylintrc mrq 39 | 40 | virtualenv: 41 | virtualenv venv --distribute --python=python2.7 42 | 43 | deps: 44 | pip install -r requirements.txt 45 | pip install -r requirements-dev.txt 46 | pip install -r requirements-dashboard.txt 47 | 48 | clean: 49 | find . -path ./venv -prune -o -name "*.pyc" -exec rm {} \; 50 | find . -name __pycache__ | xargs rm -r 51 | 52 | build_dashboard: 53 | cd mrq/dashboard/static && npm install && mkdir -p bin && npm run build 54 | 55 | dashboard: 56 | python mrq/dashboard/app.py 57 | 58 | stack: 59 | mongod & 60 | redis-server & 61 | python mrq/dashboard/app.py & 62 | 63 | pep8: 64 | autopep8 --max-line-length 99 -aaaaaaaa --diff --recursive mrq 65 | echo "Now run 'make autopep8' to apply." 66 | 67 | autopep8: 68 | autopep8 --max-line-length 99 -aaaaaaaa --in-place --recursive mrq 69 | 70 | pypi: linterrors linterrors3 71 | python setup.py sdist upload 72 | 73 | build_docs: 74 | python scripts/propagate_docs.py 75 | 76 | ensureindexes: 77 | mrq-run mrq.basetasks.indexes.EnsureIndexes 78 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: uwsgi mrq/dashboard/uwsgi-heroku.ini -------------------------------------------------------------------------------- /docs/best-practices.md: -------------------------------------------------------------------------------- 1 | The following is a list of recommendations that you should strongly consider in order to benefit from MRQ's design. 2 | 3 | ## Reentrant tasks 4 | 5 | From [Wikipedia](https://en.wikipedia.org/wiki/Reentrancy_(computing)): 6 | 7 | > A subroutine is called reentrant if it can be interrupted in the middle of its execution and then safely called again ("re-entered") before its previous invocations complete execution. 8 | 9 | This is a desired property of all tasks because by nature, code execution can be interrupted at any time. One single worker should always be considered unreliable. MRQ's job is to work around this issue by automatically requeueing interrupted jobs (see [Jobs maintenance](jobs-maintenance.md)) 10 | 11 | A good real-world example of this is the fact that Heroku dynos are restarted at least once a day. 12 | 13 | ## Monitoring failed jobs 14 | 15 | There are multiple, often unpredictable reasons jobs can fail and developer teams should monitor the list of failed tasks at all times. 16 | 17 | MRQ's Dashboard provides a dedicated view for failed tasks, conveniently grouped by Exception. 18 | 19 | ## Using the retry status 20 | 21 | All Exceptions shouldn't cause a failed job. For instance, doing an HTTP request might fail and most of the time, you will want to retry it at a later time. This is why it is useful to wrap code that can potentially raise an Exception in a try/except block, and call `retry_current_job` instead, like this: 22 | 23 | ```python 24 | from mrq.task import Task 25 | from mrq.context import retry_current_job, log 26 | import urllib2 27 | 28 | class SafeFetchTask(Task): 29 | def run(self, params): 30 | 31 | try: 32 | with urllib2.urlopen(params["url"]) as f: 33 | t = f.read() 34 | return len(t) 35 | 36 | except urllib2.HTTPError, e: 37 | log.warning("Got HTTP error %s, retrying...", e) 38 | retry_current_job() 39 | ``` 40 | 41 | Remember to add the base recurring jobs as explained in [Jobs maintenance](jobs-maintenance.md) to have `retry` jobs actuallt requeued. 42 | 43 | ## Using your own base Task class 44 | 45 | MRQ provides sensible defaults but in many cases, you will want to customize its behaviour and API. 46 | 47 | We recommend subclassing `mrq.task.Task` with your own base Task class, and have all your tasks subclass it instead. The `run_wrapped` method is a convenient entry point to wrap all further code. 48 | 49 | This is a simple example of a somewhat useful class `BaseTask`: 50 | 51 | ```python 52 | from mrq.task import Task 53 | from mrq.context import retry_current_job 54 | import urllib2 55 | 56 | class BaseTask(Task): 57 | 58 | retry_on_http_error = True 59 | 60 | def validate_params(self, params): 61 | """ Make sure some standard parameters are well-formatted """ 62 | if "url" in params: 63 | assert "://" in params["url"] 64 | 65 | def run_wrapped(self, params): 66 | """ Wrap all calls to tasks in init & safety code. """ 67 | 68 | self.validate_params(params) 69 | 70 | try: 71 | return self.run(params) 72 | 73 | # Intercept HTTPErrors in all tasks and retry them by default 74 | except urllib2.HTTPError, e: 75 | if self.retry_on_http_error: 76 | retry_current_job() 77 | 78 | ``` -------------------------------------------------------------------------------- /docs/changelog.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## Release 0.9.0 (upcoming) 4 | 5 | - First public beta release with support 6 | - API frozen for 1.0.0 -------------------------------------------------------------------------------- /docs/contributing.md: -------------------------------------------------------------------------------- 1 | We welcome all contributions on [our GitHub](https://github.com/pricingassistant/mrq)! -------------------------------------------------------------------------------- /docs/dashboard.md: -------------------------------------------------------------------------------- 1 | # Dashboard 2 | 3 | A strong focus was put on the tools and particularly the dashboard. After all, this will be your primary tool,which you will use most of the time! 4 | 5 | ![Queues view](http://i.imgur.com/H2nsgq2.png) 6 | 7 | ![Jobs view](http://i.imgur.com/SNsYhuQ.png) 8 | 9 | 10 | #### How to run 11 | 12 | To run the dashboard, make sure that mrq is properly installed. Then, use the 13 | following command: 14 | 15 | ```bash 16 | mrq-dashboard 17 | ``` 18 | 19 | The Dashboard will be running on `localhost` using port `5555`(http://localhost:5555). 20 | 21 | #### Main goal 22 | 23 | There are too many features on the dashboard to list, but the goal is to have complete visibility and control over what your workers are doing! 24 | -------------------------------------------------------------------------------- /docs/dependencies.md: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | 3 | ## Python 4 | 5 | MRQ has only been tested with Python 2.7+. 6 | 7 | Required external services dependencies are 8 | 9 | - [MongoDB >= 2.4](http://docs.mongodb.org/manual/installation/) 10 | - [Redis >= 2.6](http://redis.io/topics/quickstart) 11 | 12 | We use LUA scripting in Redis to boost performance and provide extra safety. 13 | 14 | You will need [Docker](http://docker.io) to run our unit tests. Our [Dockerfile](https://github.com/pricingassistant/mrq/blob/master/Dockerfile) is actually a good way to see a complete list of dependencies, including dev tools like graphviz for memleak images. 15 | 16 | You may want to convert your logs db to a capped collection : ie. run db. 17 | 18 | ``` 19 | runCommand({"convertToCapped": "mrq_jobs", "size": 10737418240}) 20 | ``` 21 | 22 | ## Javascript 23 | 24 | JS libraries used in the [Dashboard](dashboard.md): 25 | 26 | * [BackboneJS](http://backbonejs.org) 27 | * [UnderscoreJS](http://underscorejs.org) 28 | * [RequireJS](http://requirejs.org) 29 | * [MomentJS](http://momentjs.com) 30 | * [jQuery](http://jquery.com) 31 | * [Datatables](http://datatables.net) 32 | * [Datatables-Bootstrap3](https://github.com/Jowin/Datatables-Bootstrap3/) 33 | * [Twitter Bootstrap](https://github.com/twbs/bootstrap) 34 | 35 | # Credits 36 | 37 | Inspirations: 38 | 39 | * [RQ](http://python-rq.org/) 40 | * [Celery](www.celeryproject.org) 41 | 42 | 43 | # Useful third-party utils 44 | 45 | * http://superlance.readthedocs.org/en/latest/ 46 | -------------------------------------------------------------------------------- /docs/examples.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | Here are a few open source projects using MRQ: 4 | 5 | * [imgfab](https://github.com/sylvinus/imgfab): Create 3D galleries on Sketchfab with Blender. Tasks are queued by the user via a Flask REST API. 6 | * Add your own! -------------------------------------------------------------------------------- /docs/get-started.md: -------------------------------------------------------------------------------- 1 | # Get started 2 | 3 | This 5-minute tutorial will show you how to run your first jobs with MRQ. 4 | 5 | ## Installation 6 | 7 | - Make sure you have installed the [dependencies](dependencies.md) : Redis and MongoDB 8 | - Install MRQ with `pip install mrq` 9 | - Start a mongo server with `mongod &` 10 | - Start a redis server with `redis-server &` 11 | 12 | 13 | ## Write your first task 14 | 15 | Create a new directory and write a simple task in a file called `tasks.py` : 16 | 17 | ```makefile 18 | $ mkdir test-mrq && cd test-mrq 19 | $ touch __init__.py 20 | $ vim tasks.py 21 | ``` 22 | 23 | ```python 24 | from mrq.task import Task 25 | import urllib2 26 | 27 | 28 | class Fetch(Task): 29 | 30 | def run(self, params): 31 | 32 | with urllib2.urlopen(params["url"]) as f: 33 | t = f.read() 34 | return len(t) 35 | ``` 36 | 37 | ## Run it synchronously 38 | 39 | You can now run it from the command line using `mrq-run`: 40 | 41 | ```makefile 42 | $ mrq-run tasks.Fetch url http://www.google.com 43 | 44 | 2014-12-18 15:44:37.869029 [DEBUG] mongodb_jobs: Connecting to MongoDB at 127.0.0.1:27017/mrq... 45 | 2014-12-18 15:44:37.880115 [DEBUG] mongodb_jobs: ... connected. 46 | 2014-12-18 15:44:37.880305 [DEBUG] Starting tasks.Fetch({'url': 'http://www.google.com'}) 47 | 2014-12-18 15:44:38.158572 [DEBUG] Job None success: 0.278229s total 48 | 17655 49 | ``` 50 | 51 | ## Run it asynchronously 52 | 53 | Let's schedule the same task 3 times with different parameters: 54 | 55 | ```makefile 56 | $ mrq-run --queue fetches tasks.Fetch url http://www.google.com && 57 | mrq-run --queue fetches tasks.Fetch url http://www.yahoo.com && 58 | mrq-run --queue fetches tasks.Fetch url http://www.wordpress.com 59 | 60 | 2014-12-18 15:49:05.688627 [DEBUG] mongodb_jobs: Connecting to MongoDB at 127.0.0.1:27017/mrq... 61 | 2014-12-18 15:49:05.705400 [DEBUG] mongodb_jobs: ... connected. 62 | 2014-12-18 15:49:05.729364 [INFO] redis: Connecting to Redis at 127.0.0.1... 63 | 5492f771520d1887bfdf4b0f 64 | 2014-12-18 15:49:05.957912 [DEBUG] mongodb_jobs: Connecting to MongoDB at 127.0.0.1:27017/mrq... 65 | 2014-12-18 15:49:05.967419 [DEBUG] mongodb_jobs: ... connected. 66 | 2014-12-18 15:49:05.983925 [INFO] redis: Connecting to Redis at 127.0.0.1... 67 | 5492f771520d1887c2d7d2db 68 | 2014-12-18 15:49:06.182351 [DEBUG] mongodb_jobs: Connecting to MongoDB at 127.0.0.1:27017/mrq... 69 | 2014-12-18 15:49:06.193314 [DEBUG] mongodb_jobs: ... connected. 70 | 2014-12-18 15:49:06.209336 [INFO] redis: Connecting to Redis at 127.0.0.1... 71 | 5492f772520d1887c5b32881 72 | ``` 73 | 74 | You can see that instead of executing the tasks and returning their results right away, `mrq-run` has added them to the queue named `fetches` and printed their IDs. 75 | 76 | Now start MRQ's dasbhoard with `mrq-dashboard &` and go check your newly created queue and jobs on [localhost:5555](http://localhost:5555/#jobs) 77 | 78 | They are ready to be dequeued by a worker. Start one with `mrq-worker` and follow it on the dashboard as it executes the queued jobs in parallel. 79 | 80 | ```makefile 81 | $ mrq-worker fetches 82 | 83 | 2014-12-18 15:52:57.362209 [INFO] Starting Gevent pool with 10 worker greenlets (+ report, logs, adminhttp) 84 | 2014-12-18 15:52:57.388033 [INFO] redis: Connecting to Redis at 127.0.0.1... 85 | 2014-12-18 15:52:57.389488 [DEBUG] mongodb_jobs: Connecting to MongoDB at 127.0.0.1:27017/mrq... 86 | 2014-12-18 15:52:57.390996 [DEBUG] mongodb_jobs: ... connected. 87 | 2014-12-18 15:52:57.391336 [DEBUG] mongodb_logs: Connecting to MongoDB at 127.0.0.1:27017/mrq... 88 | 2014-12-18 15:52:57.392430 [DEBUG] mongodb_logs: ... connected. 89 | 2014-12-18 15:52:57.523329 [INFO] Fetching 1 jobs from ['fetches'] 90 | 2014-12-18 15:52:57.567311 [DEBUG] Starting tasks.Fetch({u'url': u'http://www.google.com'}) 91 | 2014-12-18 15:52:58.670492 [DEBUG] Job 5492f771520d1887bfdf4b0f success: 1.135268s total 92 | 2014-12-18 15:52:57.523329 [INFO] Fetching 1 jobs from ['fetches'] 93 | 2014-12-18 15:52:57.567747 [DEBUG] Starting tasks.Fetch({u'url': u'http://www.yahoo.com'}) 94 | 2014-12-18 15:53:01.897873 [DEBUG] Job 5492f771520d1887c2d7d2db success: 4.361895s total 95 | 2014-12-18 15:52:57.523329 [INFO] Fetching 1 jobs from ['fetches'] 96 | 2014-12-18 15:52:57.568080 [DEBUG] Starting tasks.Fetch({u'url': u'http://www.wordpress.com'}) 97 | 2014-12-18 15:53:00.685727 [DEBUG] Job 5492f772520d1887c5b32881 success: 3.149119s total 98 | 2014-12-18 15:52:57.523329 [INFO] Fetching 1 jobs from ['fetches'] 99 | 2014-12-18 15:52:57.523329 [INFO] Fetching 1 jobs from ['fetches'] 100 | ``` 101 | 102 | You can interrupt the worker with Ctrl-C once it is finished. 103 | 104 | ## Going further 105 | 106 | This was a preview on the very basic features of MRQ. What makes it actually useful is that: 107 | 108 | * You can run multiple workers in parallel. Each worker can also run multiple greenlets in parallel. 109 | * Workers can dequeue from multiple queues 110 | * You can queue jobs from your Python code to avoid using `mrq-run` from the command-line. 111 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Intro 2 | 3 | [MRQ](http://pricingassistant.github.io/mrq) is a distributed task queue for python built on top of mongo, redis and gevent. 4 | 5 | Full documentation is available on [readthedocs](http://mrq.readthedocs.org/en/latest/) 6 | 7 | # Why? 8 | 9 | MRQ is an opinionated task queue. It aims to be simple and beautiful like [RQ](http://python-rq.org) while having performances close to [Celery](http://celeryproject.org) 10 | 11 | MRQ was first developed at [Pricing Assistant](http://pricingassistant.com) and its initial feature set matches the needs of worker queues with heterogenous jobs (IO-bound & CPU-bound, lots of small tasks & a few large ones). 12 | 13 | # Main Features 14 | 15 | * **Simple code:** We originally switched from Celery to RQ because Celery's code was incredibly complex and obscure ([Slides](http://www.slideshare.net/sylvinus/why-and-how-pricing-assistant-migrated-from-celery-to-rq-parispy-2)). MRQ should be as easy to understand as RQ and even easier to extend. 16 | * **Great [dashboard](http://mrq.readthedocs.org/en/latest/dashboard/):** Have visibility and control on everything: queued jobs, current jobs, worker status, ... 17 | * **Per-job logs:** Get the log output of each task separately in the dashboard 18 | * **Gevent worker:** IO-bound tasks can be done in parallel in the same UNIX process for maximum throughput 19 | * **Job management:** You can retry, requeue, cancel jobs from the code or the dashboard. 20 | * **Performance:** Bulk job queueing, easy job profiling 21 | * **Easy [configuration](http://mrq.readthedocs.org/en/latest/configuration):** Every aspect of MRQ is configurable through command-line flags or a configuration file 22 | * **Job routing:** Like Celery, jobs can have default queues, timeout and ttl values. 23 | * **Builtin scheduler:** Schedule tasks by interval or by time of the day 24 | * **Strategies:** Sequential or parallel dequeue order, also a burst mode for one-time or periodic batch jobs. 25 | * **Subqueues:** Simple command-line pattern for dequeuing multiple sub queues, using auto discovery from worker side. 26 | * **Thorough [testing](http://mrq.readthedocs.org/en/latest/tests):** Edge-cases like worker interrupts, Redis failures, ... are tested inside a Docker container. 27 | * **Greenlet tracing:** See how much time was spent in each greenlet to debug CPU-intensive jobs. 28 | * **Integrated memory leak debugger:** Track down jobs leaking memory and find the leaks with objgraph. -------------------------------------------------------------------------------- /docs/io-monitoring.md: -------------------------------------------------------------------------------- 1 | # I/O Monitoring 2 | 3 | One of the most powerful features of MRQ is the ability to view in the Dashboard the current state of the running jobs. 4 | 5 | You can view their current callstack, and if the worker was started with `--trace_io`, the high-level details of the I/O operation they are waiting for, if any. 6 | 7 | This is done through gevent-style monkey patching of common I/O modules. Until further documentation, see [monkey.py](https://github.com/pricingassistant/mrq/blob/master/mrq/monkey.py) -------------------------------------------------------------------------------- /docs/jobs-maintenance.md: -------------------------------------------------------------------------------- 1 | # Jobs maintenance 2 | 3 | MRQ can provide strong guarantees that no job will be lost in the middle of a worker restart, database disconnect, etc... 4 | 5 | To do that, you should add these recurring scheduled jobs to your mrq-config.py: 6 | 7 | ``` 8 | SCHEDULER_TASKS = [ 9 | 10 | # This will requeue jobs in the 'retry' status, until they reach their max_retries. 11 | { 12 | "path": "mrq.basetasks.cleaning.RequeueRetryJobs", 13 | "params": {}, 14 | "interval": 60 15 | }, 16 | 17 | # This will requeue jobs marked as interrupted, for instance when a worker received SIGTERM 18 | { 19 | "path": "mrq.basetasks.cleaning.RequeueInterruptedJobs", 20 | "params": {}, 21 | "interval": 5 * 60 22 | }, 23 | 24 | # This will requeue jobs marked as started for a long time (more than their own timeout) 25 | # They can exist if a worker was killed with SIGKILL and not given any time to mark 26 | # its current jobs as interrupted. 27 | { 28 | "path": "mrq.basetasks.cleaning.RequeueStartedJobs", 29 | "params": {}, 30 | "interval": 3600 31 | }, 32 | 33 | # This will make sure MRQ's indexes are built 34 | { 35 | "path": "mrq.basetasks.indexes.EnsureIndexes", 36 | "params": {}, 37 | "interval": 24 * 3600 38 | } 39 | ] 40 | ``` 41 | 42 | Obviously this implies that all your jobs should be *idempotent*, meaning that they could be done multiple times, maybe partially, without breaking your app. This is a very good design to enforce for your whole task queue, though you can still manage locks yourself in your code that make sure a block of code will only run once. 43 | -------------------------------------------------------------------------------- /docs/license.md: -------------------------------------------------------------------------------- 1 | # License 2 | 3 | The MIT License (MIT) 4 | 5 | Copyright (c) 2014 pricingassistant 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy of 8 | this software and associated documentation files (the "Software"), to deal in 9 | the Software without restriction, including without limitation the rights to 10 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 11 | the Software, and to permit persons to whom the Software is furnished to do so, 12 | subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all 15 | copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 19 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 20 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 21 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 22 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /docs/metrics.md: -------------------------------------------------------------------------------- 1 | # Metrics & Graphite 2 | 3 | MRQ doesn't support sending metrics to Graphite out of the box but makes it extremely easy to do so. 4 | 5 | All you have to do is add this hook in your mrq-config file: 6 | 7 | ```python 8 | import graphiteudp # Install this via pip 9 | 10 | # Initialize the Graphite UDP Client 11 | _graphite_client = graphiteudp.GraphiteUDPClient(host, port, prefix, debug=False) 12 | _graphite_client.init() 13 | 14 | def METRIC_HOOK(name, incr=1, **kwargs): 15 | 16 | # You can use this to avoid sending too many different metrics 17 | whitelisted_metrics = ["queues.all.", "queues.default.", "jobs."] 18 | 19 | if any([name.startswith(m) for m in whitelisted_metrics]): 20 | _graphite_client.send(name, incr) 21 | ``` 22 | 23 | If you have another monitoring system you can plug anything in this hook to connect to it! -------------------------------------------------------------------------------- /docs/performance.md: -------------------------------------------------------------------------------- 1 | # Worker performance 2 | 3 | Performance is an explicit goal of MRQ as it was first developed at [Pricing Assistant](http://www.pricingassistant.com/) for crawling billions of web pages. 4 | 5 | ## Throughput tests 6 | 7 | On a regular Macbook Pro, we see 1300 jobs/second in a single worker process with very simple jobs that store results, to measure the overhead of MRQ. 8 | 9 | However what we are really measuring there is MongoDB's write performance. An install of MRQ with properly scaled MongoDB and Redis instances is be capable of much more. 10 | 11 | For more, see our tutorial on [Queue performance](queue-performance.md). 12 | 13 | ## PyPy support 14 | 15 | Earlier in its development MRQ was tested successfully on PyPy but we are waiting for better PyPy+gevent support to continue working on it, as performance was worse than CPython. 16 | 17 | ## Heroku 18 | 19 | On Heroku's 1X dynos with 512M RAM, we have found that for IO-bound jobs, `--processes 4 --greenlets 30` may be a good setting. 20 | -------------------------------------------------------------------------------- /docs/queues.md: -------------------------------------------------------------------------------- 1 | # Regular queues 2 | 3 | With regular queues, MRQ stores the tasks in MongoDB. 4 | 5 | You can transform a queue into a [pile](https://en.wikipedia.org/wiki/LIFO_(computing)) by appending `_reverse` to its name: 6 | 7 | ```makefile 8 | # Will dequeue the last jobs added to the queue "default" 9 | $ mrq-worker default_reverse 10 | ``` 11 | 12 | # Raw queues 13 | 14 | Raw queues give you more performance and some powerful features in exchange for a bit less visibility for individual queued jobs. In their case, only the parameters of a task are stored in serialized form in Redis when queued, and they are inserted in MongoDB only after being dequeued by a worker. 15 | 16 | There are 4 types of raw queues. The type of a queue is determined by a suffix in its name: 17 | 18 | * ```_raw``` : The simpliest raw queue, stored in a Redis LIST. 19 | * ```_set``` : Stored in a Redis SET. Gives you the ability to have "unique" tasks: only one (task, parameters) couple can be queued at a time. 20 | * ```_sorted_set``` : The most powerful MRQ queue type, stored in a Redis ZSET. Allows you to order (and re-order) the tasks to be dequeued. Like ```_set```, task parameters will be unique. 21 | * ```_timed_set``` : A special case of ```_sorted_set```, where tasks are sorted with a UNIX timestamp. This means you can schedule tasks to be executed at a precise time in the future. 22 | 23 | Raw queues need a special entry in the configuration to unserialize their "raw" parameters in a regular dict of parameters. They also need to be linked to a regular queue ("default" if none) for providing visibility and retries, after they are dequeued from the raw queue. 24 | 25 | This is an example of raw queue configuration: 26 | 27 | ```python 28 | RAW_QUEUES = { 29 | "myqueue_raw": { 30 | "job_factory": lambda rawparam: { 31 | "path": "tests.tasks.general.Add", 32 | "params": { 33 | "a": int(rawparam.split(" ")[0]), 34 | "b": int(rawparam.split(" ")[1]) 35 | } 36 | }, 37 | "retry_queue": "high" 38 | } 39 | } 40 | ``` 41 | 42 | This task adds two integers. To queue tasks, you can do from the code: 43 | 44 | ```python 45 | from mrq.job import queue_raw_jobs 46 | 47 | queue_raw_jobs("myqueue_raw", [ 48 | ["1 1"], 49 | ["42 8"] 50 | ]) 51 | ``` 52 | 53 | To run them, start a worker listening to both queues: 54 | 55 | ``` 56 | $ mrq-worker high myqueue_raw 57 | ``` 58 | 59 | Queueing on timed sets is a bit different, you can pass unix timestamps directly: 60 | 61 | ```python 62 | from mrq.job import queue_raw_jobs 63 | import time 64 | 65 | queue_raw_jobs("myqueue_timed_set", { 66 | "rawparam_xxx": time.time(), 67 | "rawparam_yyy": time.time() + 3600 # Do this in an hour 68 | }) 69 | ``` 70 | 71 | For more examples of raw queue configuration, check [the tests](https://github.com/pricingassistant/mrq/blob/master/tests/fixtures/config-raw1.py). 72 | 73 | You should also read our tutorial on [Queue performance](queue-performance.md) to get a good overview of the different queue types. 74 | -------------------------------------------------------------------------------- /docs/recurring-jobs.md: -------------------------------------------------------------------------------- 1 | # Recurring jobs 2 | 3 | MRQ provides a simple scheduler to help you run tasks every X units of time like a crontab does. 4 | 5 | See the [tests](https://github.com/pricingassistant/mrq/blob/master/tests/test_scheduler.py) 6 | 7 | Please note that scheduling jobs once (setting a precise time for them to be executed in the future) is supported by `timed_set` [raw queues](queues.md#raw-queues). 8 | 9 | Be sure to do `mrq-run mrq.basetasks.indexes.EnsureIndexes` at least once to build the indexes for MRQ, because the scheduler depends on a unique index on the `hash` field. -------------------------------------------------------------------------------- /docs/tests.md: -------------------------------------------------------------------------------- 1 | # Tests 2 | 3 | MRQ provides a [comprehensive test suite](https://github.com/pricingassistant/mrq/tree/master/tests). The goal is to test all edge cases in order to build a trusted foundation for running distributed code. 4 | 5 | Testing is done inside a Docker container for maximum repeatability. 6 | 7 | Therefore you need to [install docker](https://www.docker.io/gettingstarted/#h_installation) to run the tests. 8 | If you're not on an os that supports natively docker, don't forget to start up your VM and ssh into it. 9 | 10 | ``` 11 | $ make test 12 | ``` 13 | 14 | You can also open a shell inside the docker (just like you would enter in a virtualenv) with: 15 | 16 | ``` 17 | $ make docker 18 | $ make shell 19 | $ py.test tests/ -v 20 | ``` 21 | -------------------------------------------------------------------------------- /docs/workers.md: -------------------------------------------------------------------------------- 1 | ## Worker 2 | 3 | A worker is a unit of processing, that dequeues jobs and executes them. 4 | 5 | It is started with a list of queues to listen to, in a specific order. 6 | 7 | It can be started with concurrency options (multiple processes and / or multiple greenlets). We call this whole group a single 'worker' even though it is able to dequeue multiple jobs in parallel. 8 | 9 | If a worker is started with concurrency options, it will poll for waiting jobs and dispatch them to its related processes/greenlets. 10 | For example, if we decide to use the greenlets option, under the hood, the worker will be one python process that has a pool of greenlets which will be in charge of actually running tasks. 11 | 12 | 13 | ## Statuses 14 | 15 | At any time, a worker is in one of these statuses: 16 | 17 | * `init`: General worker initialization 18 | * `wait`: Waiting for new jobs from Redis 19 | * `spawn`: Got some new jobs, greenlets are being spawned 20 | * `full`: All the worker pool is busy executing jobs 21 | * `join`: Waiting for current jobs to finish, no new one will be accepted 22 | * `kill`: Killing all current jobs 23 | * `stop`: Worker is stopped, no jobs should remain -------------------------------------------------------------------------------- /examples/queue_performance/enqueue.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | from mrq.context import setup_context 4 | from mrq.job import queue_jobs, queue_raw_jobs 5 | 6 | setup_context() 7 | 8 | queue = sys.argv[1] 9 | n = int(sys.argv[2]) 10 | 11 | if queue == "square": 12 | queue_jobs("tasks.Square", [{"n": 42} for _ in range(n)], queue=queue) 13 | 14 | elif queue in ("square_raw", "square_nostorage_raw"): 15 | queue_raw_jobs(queue, [42 for _ in range(n)]) 16 | -------------------------------------------------------------------------------- /examples/queue_performance/tasks.py: -------------------------------------------------------------------------------- 1 | from mrq.task import Task 2 | import time 3 | 4 | 5 | class Square(Task): 6 | """ Returns the square of an integer """ 7 | def run(self, params): 8 | return int(params["n"]) ** 2 9 | 10 | 11 | class CPU(Task): 12 | """ A CPU-intensive task """ 13 | def run(self, params): 14 | for n in range(int(params["n"])): 15 | n ** n 16 | return params["a"] 17 | 18 | 19 | class IO(Task): 20 | """ An IO-intensive task """ 21 | def run(self, params): 22 | time.sleep(float(params["sleep"])) 23 | return params["a"] 24 | -------------------------------------------------------------------------------- /examples/scheduler/README.md: -------------------------------------------------------------------------------- 1 | Simple task scheduler with MRQ 2 | =========================== 3 | 4 | This is a simple exemple of a recurring task using MRQ. 5 | 6 | We have a simple task that print text to the terminal, and we want to launch it every 10 seconds. That is what this exemple is demonstrating. 7 | 8 | How to use 9 | ========== 10 | 11 | First, get into the docker image at the root of this directory: 12 | ``` 13 | docker run -t -i -v `pwd`:/src -w /src pricingassistant/mrq bash 14 | ``` 15 | Don't forget to `cd` in the directory of this exemple! 16 | 17 | Launch MongoDB & Redis if they are not already started: 18 | ``` 19 | $ mongod & 20 | $ redis-server & 21 | ``` 22 | 23 | Then launch a scheduler worker, feeding him the config. 24 | ``` 25 | mrq-worker --scheduler --config config.py 26 | ``` 27 | 28 | In the config. we have described two time the task, with different parameters 29 | You should then see the task printing its parameters every 10 seconds on the terminal. 30 | 31 | ``` 32 | 2018-03-09 11:16:03.927182 [DEBUG] Scheduler: added tasks.Print 1 None None None None [["x","Another test."]] 33 | 2018-03-09 11:16:03.930823 [DEBUG] Scheduler: added tasks.Print 1 None None None None [["x","Test."]] 34 | 2018-03-09 11:16:03.960213 [DEBUG] Scheduler: queued tasks.Print 1 None None None None [["x","Another test."]] 35 | 2018-03-09 11:16:03.970537 [DEBUG] Scheduler: queued tasks.Print 1 None None None None [["x","Test."]] 36 | 2018-03-09 11:16:04.887654 [DEBUG] Starting tasks.Print({u'x': u'Another test.'}) 37 | Hello world ! 38 | Another test. 39 | 2018-03-09 11:16:04.901856 [DEBUG] Job 5aa26cf322f9db001dfdbdb0 success: 0.014454s total 40 | 2018-03-09 11:16:04.903789 [DEBUG] Starting tasks.Print({u'x': u'Test.'}) 41 | Hello world ! 42 | Test. 43 | 2018-03-09 11:16:04.905972 [DEBUG] Job 5aa26cf322f9db001dfdbdb1 success: 0.002258s total 44 | 2018-03-09 11:16:14.985899 [DEBUG] Scheduler: queued tasks.Print 1 None None None None [["x","Another test."]] 45 | 2018-03-09 11:16:14.988513 [DEBUG] Scheduler: queued tasks.Print 1 None None None None [["x","Test."]] 46 | 2018-03-09 11:16:15.961432 [DEBUG] Starting tasks.Print({u'x': u'Another test.'}) 47 | Hello world ! 48 | Another test. 49 | 2018-03-09 11:16:15.964429 [DEBUG] Job 5aa26cfe22f9db001dfdbdb4 success: 0.004088s total 50 | 2018-03-09 11:16:15.967952 [DEBUG] Starting tasks.Print({u'x': u'Test.'}) 51 | Hello world ! 52 | Test. 53 | 2018-03-09 11:16:15.970359 [DEBUG] Job 5aa26cfe22f9db001dfdbdb5 success: 0.002809s total 54 | ``` 55 | 56 | Other features 57 | ============== 58 | 59 | It is also possible to launch tasks everyday, or on specifics days of the week or month. 60 | 61 | We advise you to look at the [tests](https://github.com/pricingassistant/mrq/blob/master/tests/test_scheduler.py) for these use-cases. -------------------------------------------------------------------------------- /examples/scheduler/config.py: -------------------------------------------------------------------------------- 1 | 2 | SCHEDULER_TASKS = [ 3 | { 4 | "path": "tasks.Print", 5 | "params": { 6 | "x": "Test." 7 | }, 8 | "interval": 1 9 | }, 10 | { 11 | "path": "tasks.Print", 12 | "params": { 13 | "x": "Another test." 14 | }, 15 | "interval": 1 16 | } 17 | ] 18 | 19 | SCHEDULER_INTERVAL = 10 20 | -------------------------------------------------------------------------------- /examples/scheduler/tasks.py: -------------------------------------------------------------------------------- 1 | 2 | from mrq.task import Task 3 | 4 | 5 | class Print(Task): 6 | 7 | def run(self, params): 8 | 9 | print("Hello world !") 10 | print(params["x"]) 11 | -------------------------------------------------------------------------------- /examples/simple_crawler/README.md: -------------------------------------------------------------------------------- 1 | Simple Web Crawler with MRQ 2 | =========================== 3 | 4 | This is a simple demo app that crawls a website, to demo some MRQ features. 5 | 6 | 7 | How to use 8 | ========== 9 | 10 | First, get into the docker image at the root of this directory: 11 | ``` 12 | docker run -t -i -v `pwd`:/src -w /src pricingassistant/mrq bash 13 | ``` 14 | Don't forget to `cd` in the directory of this exemple! 15 | 16 | Then install MRQ and the packages needed for this example: 17 | ``` 18 | $ cd examples/simple_crawler 19 | $ sudo apt-get install libxml2-dev libxslt1-dev libz-dev # Needed for lxml 20 | $ pip install -r requirements.txt 21 | ``` 22 | 23 | Launch MongoDB & Redis if they are not already started: 24 | ``` 25 | $ mongod & 26 | $ redis-server & 27 | ``` 28 | 29 | Queue the first task via the command line: 30 | 31 | ``` 32 | $ mrq-run --queue crawl crawler.Fetch '{"url": "http://docs.python-requests.org/"}' 33 | ``` 34 | 35 | Then start a worker with 3 (or more!) greenlets: 36 | 37 | ``` 38 | $ mrq-worker crawl --greenlets 3 39 | ``` 40 | 41 | You should also launch a dashboard to monitor the progress: 42 | ``` 43 | $ mrq-dashboard 44 | ``` 45 | 46 | We also included 2 utility tasks: 47 | ``` 48 | $ mrq-run crawler.Report 49 | $ mrq-run crawler.Reset 50 | ``` 51 | 52 | This is obviously a very simple crawler, production systems will be much more complex but it gives you an overview of MRQ and a good starting point. 53 | 54 | 55 | Expected result for crawler.Report 56 | ================================== 57 | 58 | As of 2018-03-06: 59 | 60 | ``` 61 | Crawl stats 62 | =========== 63 | URLs queued: 81 64 | URLs successfully crawled: 81 65 | URLs redirected: 1 66 | Bytes fetched: 4099658 67 | ``` 68 | -------------------------------------------------------------------------------- /examples/simple_crawler/crawler.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import lxml.html 3 | import datetime 4 | import re 5 | import urlparse 6 | from mrq.context import connections, log 7 | from mrq.job import queue_job 8 | from mrq.task import Task 9 | from mrq.queue import Queue 10 | 11 | 12 | class Fetch(Task): 13 | 14 | def run(self, params): 15 | 16 | collection = connections.mongodb_jobs.simple_crawler_urls 17 | 18 | response = requests.get(params["url"]) 19 | 20 | if response.status_code != 200: 21 | log.warning("Got status %s on page %s (Queued from %s)" % ( 22 | response.status_code, response.url, params.get("from") 23 | )) 24 | return False 25 | 26 | # Store redirects 27 | if response.url != params["url"]: 28 | collection.update({"_id": params["url"]}, {"$set": { 29 | "redirected_to": response.url, 30 | "fetched_date": datetime.datetime.now() 31 | }}) 32 | 33 | document = lxml.html.fromstring(response.content) 34 | 35 | document.make_links_absolute(response.url) 36 | 37 | queued_count = 0 38 | 39 | document_domain = urlparse.urlparse(response.url).netloc 40 | 41 | for (element, attribute, link, pos) in document.iterlinks(): 42 | 43 | link = re.sub("#.*", "", link or "") 44 | 45 | if not link: 46 | continue 47 | 48 | domain = urlparse.urlparse(link).netloc 49 | 50 | # Don't follow external links for this example 51 | if domain != document_domain: 52 | continue 53 | 54 | # We don't want to re-queue URLs twice. If we try to insert a duplicate, 55 | # pymongo will throw an error 56 | try: 57 | collection.insert({"_id": link}) 58 | except: 59 | continue 60 | 61 | queue_job("crawler.Fetch", { 62 | "url": link, 63 | "from": params["url"] 64 | }, queue="crawl") 65 | queued_count += 1 66 | 67 | stored_data = { 68 | "_id": response.url, 69 | "queued_urls": queued_count, 70 | "html_length": len(response.content), 71 | "fetched_date": datetime.datetime.now() 72 | } 73 | 74 | collection.update( 75 | {"_id": response.url}, 76 | stored_data, 77 | upsert=True 78 | ) 79 | 80 | return True 81 | 82 | 83 | class Report(Task): 84 | 85 | def run(self, params): 86 | 87 | collection = connections.mongodb_jobs.simple_crawler_urls 88 | 89 | print() 90 | print("Crawl stats") 91 | print("===========") 92 | print("URLs queued: %s" % collection.find().count()) 93 | print("URLs successfully crawled: %s" % collection.find({"fetched_date": {"$exists": True}}).count()) 94 | print("URLs redirected: %s" % collection.find({"redirected_to": {"$exists": True}}).count()) 95 | print("Bytes fetched: %s" % (list(collection.aggregate([ 96 | {"$group": {"_id": None, "sum": {"$sum": "$html_length"}}} 97 | ])) or [{}])[0].get("sum", 0)) 98 | print() 99 | 100 | 101 | class Reset(Task): 102 | 103 | def run(self, params): 104 | 105 | collection = connections.mongodb_jobs.simple_crawler_urls 106 | 107 | collection.remove({}) 108 | 109 | Queue("crawl").empty() 110 | -------------------------------------------------------------------------------- /examples/simple_crawler/requirements.txt: -------------------------------------------------------------------------------- 1 | lxml==3.4.2 2 | requests==2.20.0 -------------------------------------------------------------------------------- /examples/timed_set/README.md: -------------------------------------------------------------------------------- 1 | Simple Web Crawler with MRQ 2 | =========================== 3 | 4 | This is a simple demo app that uses raw timed set queues. 5 | 6 | 7 | How to use 8 | ========== 9 | 10 | First, get into a Python virtualenv (`make virtualenv`) or into the docker image at the root of this directory (`make ssh`) 11 | 12 | Then install MRQ and the packages needed for this example: 13 | ``` 14 | $ cd examples/timed_set 15 | ``` 16 | 17 | Launch MongoDB & Redis if they are not already started: 18 | ``` 19 | $ mongod & 20 | $ redis-server & 21 | ``` 22 | 23 | Enqueue raw jobs with python script: 24 | 25 | ``` 26 | $ python enqueue_raw_jobs.py example_timed_set 4 10 27 | ``` 28 | 29 | You can check redis entries: 30 | 31 | ``` 32 | $ redis-cli 33 | $ ZRANGE mrq:q:example_timed_set 0 -1 WITHSCORES 34 | ``` 35 | 36 | You should see the following lines : (except timestamp of course) 37 | 38 | ``` 39 | 1) "task_0" 40 | 2) "1520457493.2588561" 41 | 3) "task_1" 42 | 4) "1520457503.2588561" 43 | 5) "task_2" 44 | 6) "1520457513.2588561" 45 | 7) "task_3" 46 | 8) "1520457523.2588561" 47 | ``` 48 | 49 | You should also launch a dashboard to monitor the progress: 50 | ``` 51 | $ mrq-dashboard 52 | ``` 53 | 54 | Then spawn a worker listenig to your timed_set queue example_timed_set: 55 | ``` 56 | $ mrq-worker example_timed_set --config=/app/examples/timed_set/config.py 57 | ``` 58 | 59 | This is obviously a very simple example, production systems will be much more complex but it gives you an overview of timed set queues and a good starting point. 60 | 61 | 62 | Expected result for mrq-worker example_timed_set --config=/app/examples/timed_set/config.py 63 | ================================== 64 | 65 | ``` 66 | [DEBUG] Starting example.Print({'test': 'task_0'}) 67 | Hello World 68 | Given params test is task_0 69 | Bye 70 | 71 | [DEBUG] Starting example.Print({'test': 'task_1'}) 72 | Hello World 73 | Time of last tasks execution 10.04 seconds 74 | Given params test is task_1 75 | Bye 76 | 77 | [DEBUG] Starting example.Print({'test': 'task_2'}) 78 | Hello World 79 | Time of last tasks execution 10.03 seconds 80 | Given params test is task_2 81 | Bye 82 | 83 | [DEBUG] Starting example.Print({'test': 'task_3'}) 84 | Hello World 85 | Time of last tasks execution 10.03 seconds 86 | Given params test is task_3 87 | Bye 88 | ``` 89 | 90 | Limitation 91 | ================================== 92 | 93 | Note that the duration beetween enqueued tasks execution depends on when you start your worker. For example if you enqueue tasks every 10 seconds from now and you're waiting 20 seconds before spawning you worker, first 2 tasks we'll be executed directly as the expected execution time is in the past. Here is the expected output of that case : 94 | 95 | ``` 96 | [DEBUG] Starting example.Print({'test': 'task_0'}) 97 | Hello World 98 | Given params test is task_0 99 | Bye 100 | 101 | [DEBUG] Starting example.Print({'test': 'task_1'}) 102 | Hello World 103 | Last task was executed 3.13 seconds ago 104 | Given params test is task_1 105 | Bye 106 | 107 | [DEBUG] Starting example.Print({'test': 'task_2'}) 108 | Hello World 109 | Last task was executed 10.03 seconds ago 110 | Given params test is task_2 111 | Bye 112 | 113 | [DEBUG] Starting example.Print({'test': 'task_3'}) 114 | Hello World 115 | Last task was executed 10.03 seconds ago 116 | Given params test is task_3 117 | Bye 118 | ``` 119 | 120 | -------------------------------------------------------------------------------- /examples/timed_set/config.py: -------------------------------------------------------------------------------- 1 | RAW_QUEUES = { 2 | "example_timed_set": { 3 | "job_factory": lambda rawparam: { 4 | "path": "example.Print", 5 | "params": { 6 | "test": rawparam 7 | } 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /examples/timed_set/enqueue_raw_jobs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import time 4 | from mrq.context import setup_context 5 | from mrq.job import queue_raw_jobs 6 | 7 | setup_context() 8 | 9 | queue = sys.argv[1] 10 | n = int(sys.argv[2]) 11 | t = int(sys.argv[3]) 12 | 13 | if queue in ("example_timed_set"): 14 | now = time.time() 15 | # every 10 seconds 16 | queue_raw_jobs(queue, {"task_%s" % _: now + (_ + 1) * t for _ in range(n)}) 17 | -------------------------------------------------------------------------------- /examples/timed_set/example.py: -------------------------------------------------------------------------------- 1 | from mrq.task import Task 2 | from datetime import datetime 3 | import mrq.context as context 4 | import time 5 | 6 | 7 | class Print(Task): 8 | 9 | def run(self, params): 10 | 11 | print "Hello World" 12 | 13 | last_task = context.connections.redis.get("test:print") 14 | if last_task: 15 | print "Last task was executed %.2f seconds ago" % (time.time() - float(last_task)) 16 | 17 | context.connections.redis.set("test:print", time.time()) 18 | 19 | print "Given params test is", params["test"] 20 | 21 | print "Bye" 22 | 23 | 24 | class RemoveRedisEntry(Task): 25 | 26 | def run(self, params): 27 | 28 | context.connections.redis.delete("test:print") -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: MRQ 2 | theme: readthedocs 3 | dev_addr: '0.0.0.0:8000' 4 | pages: 5 | 6 | - ["index.md", "Introduction"] 7 | - ["get-started.md", "Get Started"] 8 | 9 | - ["jobs.md", "Basic usage", "Jobs & Tasks"] 10 | - ["workers.md", "Basic usage", "Workers"] 11 | - ["queues.md", "Basic usage", "Queues"] 12 | - ["configuration.md", "Basic usage", "Configuration"] 13 | - ["command-line.md", "Basic usage", "Command line"] 14 | 15 | - ["dashboard.md", "Visibility", "Dashboard"] 16 | - ["memory-leaks.md", "Visibility", "Hunting memory leaks"] 17 | - ["metrics.md", "Visibility", "Metrics"] 18 | - ["io-monitoring.md", "Visibility", "I/O Monitoring"] 19 | 20 | - ["performance.md", "Advanced", "Worker performance"] 21 | - ["queue-performance.md", "Advanced", "Queue performance"] 22 | - ["recurring-jobs.md", "Advanced", "Recurring jobs"] 23 | - ["jobs-maintenance.md", "Advanced", "Jobs maintenance"] 24 | - ["best-practices.md", "Advanced", "Best practices"] 25 | 26 | - ["contributing.md", "About MRQ", "Contributing"] 27 | - ["tests.md", "About MRQ", "Tests"] 28 | - ["changelog.md", "About MRQ", "Changelog"] 29 | - ["dependencies.md", "About MRQ", "Dependencies & credits"] 30 | - ["license.md", "About MRQ", "License"] 31 | -------------------------------------------------------------------------------- /mrq/__init__.py: -------------------------------------------------------------------------------- 1 | """ MRQ """ 2 | -------------------------------------------------------------------------------- /mrq/basetasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pricingassistant/mrq/5a1154dc6a60d8b6d6423937f6569ce8fb0347b6/mrq/basetasks/__init__.py -------------------------------------------------------------------------------- /mrq/basetasks/cleaning.py: -------------------------------------------------------------------------------- 1 | from future.builtins import str 2 | from mrq.queue import Queue 3 | from mrq.task import Task 4 | from mrq.job import Job 5 | from mrq.context import log, connections, run_task, get_current_config 6 | import datetime 7 | import time 8 | 9 | 10 | class RequeueInterruptedJobs(Task): 11 | 12 | """ Requeue jobs that were marked as status=interrupt when a worker got a SIGTERM. """ 13 | 14 | max_concurrency = 1 15 | 16 | def run(self, params): 17 | return run_task("mrq.basetasks.utils.JobAction", { 18 | "status": "interrupt", 19 | "action": "requeue_retry" 20 | }) 21 | 22 | 23 | class RequeueRetryJobs(Task): 24 | 25 | """ Requeue jobs that were marked as retry. """ 26 | 27 | max_concurrency = 1 28 | 29 | def run(self, params): 30 | print("IN") 31 | return run_task("mrq.basetasks.utils.JobAction", { 32 | "status": "retry", 33 | "dateretry": {"$lte": datetime.datetime.utcnow()}, 34 | "action": "requeue_retry" 35 | }) 36 | 37 | 38 | class RequeueStartedJobs(Task): 39 | 40 | """ Requeue jobs that were marked as status=started and never finished. 41 | 42 | That may be because the worker got a SIGKILL or was terminated abruptly. 43 | The timeout parameter of this task is in addition to the task's own timeout. 44 | """ 45 | 46 | max_concurrency = 1 47 | 48 | def run(self, params): 49 | 50 | additional_timeout = params.get("timeout", 300) 51 | 52 | stats = { 53 | "requeued": 0, 54 | "started": 0 55 | } 56 | 57 | # There shouldn't be that much "started" jobs so we can quite safely 58 | # iterate over them. 59 | 60 | fields = { 61 | "_id": 1, "datestarted": 1, "queue": 1, "path": 1, "retry_count": 1, "worker": 1, "status": 1 62 | } 63 | for job_data in connections.mongodb_jobs.mrq_jobs.find( 64 | {"status": "started"}, projection=fields): 65 | job = Job(job_data["_id"]) 66 | job.set_data(job_data) 67 | 68 | stats["started"] += 1 69 | 70 | expire_date = datetime.datetime.utcnow( 71 | ) - datetime.timedelta(seconds=job.timeout + additional_timeout) 72 | 73 | requeue = job_data["datestarted"] < expire_date 74 | 75 | if not requeue: 76 | # Check that the supposedly running worker still exists 77 | requeue = not connections.mongodb_jobs.mrq_workers.find_one( 78 | {"_id": job_data["worker"]}, projection={"_id": 1}) 79 | 80 | if requeue: 81 | log.debug("Requeueing job %s" % job.id) 82 | job.requeue() 83 | stats["requeued"] += 1 84 | 85 | return stats 86 | -------------------------------------------------------------------------------- /mrq/basetasks/indexes.py: -------------------------------------------------------------------------------- 1 | from mrq.task import Task 2 | from mrq.context import connections 3 | 4 | 5 | class EnsureIndexes(Task): 6 | 7 | def run(self, params): 8 | 9 | if connections.mongodb_logs: 10 | connections.mongodb_logs.mrq_logs.ensure_index( 11 | [("job", 1)], background=True) 12 | connections.mongodb_logs.mrq_logs.ensure_index( 13 | [("worker", 1)], background=True, sparse=True) 14 | 15 | connections.mongodb_jobs.mrq_workers.ensure_index( 16 | [("status", 1)], background=True) 17 | connections.mongodb_jobs.mrq_workers.ensure_index( 18 | [("datereported", 1)], background=True, expireAfterSeconds=3600) 19 | 20 | connections.mongodb_jobs.mrq_jobs.ensure_index( 21 | [("status", 1)], background=True) 22 | connections.mongodb_jobs.mrq_jobs.ensure_index( 23 | [("path", 1)], background=True) 24 | connections.mongodb_jobs.mrq_jobs.ensure_index( 25 | [("worker", 1)], background=True, sparse=True) 26 | connections.mongodb_jobs.mrq_jobs.ensure_index( 27 | [("queue", 1)], background=True) 28 | connections.mongodb_jobs.mrq_jobs.ensure_index( 29 | [("dateexpires", 1)], sparse=True, background=True, expireAfterSeconds=0) 30 | connections.mongodb_jobs.mrq_jobs.ensure_index( 31 | [("dateretry", 1)], sparse=True, background=True) 32 | connections.mongodb_jobs.mrq_jobs.ensure_index( 33 | [("datequeued", 1)], background=True) 34 | connections.mongodb_jobs.mrq_jobs.ensure_index( 35 | [("queue", 1), ("status", 1), ("datequeued", 1), ("_id", 1)], background=True) 36 | connections.mongodb_jobs.mrq_jobs.ensure_index( 37 | [("status", 1), ("queue", 1), ("path", 1)], background=True) 38 | 39 | connections.mongodb_jobs.mrq_scheduled_jobs.ensure_index( 40 | [("hash", 1)], unique=True, background=False) 41 | 42 | connections.mongodb_jobs.mrq_agents.ensure_index( 43 | [("datereported", 1)], background=True) 44 | connections.mongodb_jobs.mrq_agents.ensure_index( 45 | [("dateexpires", 1)], background=True, expireAfterSeconds=0) 46 | connections.mongodb_jobs.mrq_agents.ensure_index( 47 | [("worker_group", 1)], background=True) 48 | -------------------------------------------------------------------------------- /mrq/basetasks/orchestrator.py: -------------------------------------------------------------------------------- 1 | from future.builtins import str 2 | from mrq.queue import Queue 3 | from mrq.task import Task 4 | from mrq.job import Job 5 | from mrq.context import log, connections, run_task, get_current_config, subpool_map 6 | from collections import defaultdict 7 | import math 8 | import shlex 9 | import argparse 10 | from ..config import add_parser_args 11 | from ..utils import normalize_command 12 | import traceback 13 | import datetime 14 | import re 15 | 16 | 17 | class Orchestrate(Task): 18 | 19 | max_concurrency = 1 20 | 21 | def run(self, params): 22 | 23 | self.config = get_current_config() 24 | 25 | concurrency = int(params.get("concurrency", 5)) 26 | groups = self.fetch_worker_group_definitions() 27 | if len(groups) == 0: 28 | log.error("No worker group definition yet. Can't orchestrate!") 29 | return 30 | 31 | subpool_map(concurrency, self.orchestrate, groups) 32 | 33 | def redis_orchestrator_lock_key(self, worker_group): 34 | """ Returns the global redis key used to ensure only one agent orchestrator runs at a time """ 35 | return "%s:orchestratorlock:%s" % (get_current_config()["redis_prefix"], worker_group) 36 | 37 | def orchestrate(self, worker_group): 38 | try: 39 | self.do_orchestrate(worker_group) 40 | except Exception as e: 41 | log.error("Orchestration error! %s" % e) 42 | traceback.print_exc() 43 | 44 | def do_orchestrate(self, group): 45 | """ Manage the desired workers of *all* the agents in the given group """ 46 | 47 | log.debug("Starting orchestration run for worker group %s" % group["_id"]) 48 | 49 | agents = self.fetch_worker_group_agents(group) 50 | 51 | # Evaluate what workers are currently, rightfully there. They won't be touched. 52 | for agent in agents: 53 | desired_workers = self.get_desired_workers_for_agent(group, agent) 54 | agent["new_desired_workers"] = [] 55 | agent["new_desired_workers"] = desired_workers 56 | 57 | for agent in agents: 58 | if sorted(agent["new_desired_workers"]) != sorted(agent.get("desired_workers", [])): 59 | connections.mongodb_jobs.mrq_agents.update_one({"_id": agent["_id"]}, {"$set": { 60 | "desired_workers": agent["new_desired_workers"] 61 | }}) 62 | 63 | # Remember the date of the last successful orchestration (will be reported) 64 | self.dateorchestrated = datetime.datetime.utcnow() 65 | 66 | log.debug("Orchestration finished.") 67 | 68 | def redis_queuestats_key(self): 69 | """ Returns the global HSET redis key used to store queue stats """ 70 | return "%s:queuestats" % (get_current_config()["redis_prefix"]) 71 | 72 | def get_desired_workers_for_agent(self, group, agent): 73 | return group.get("commands", []) 74 | 75 | def fetch_worker_group_reports(self, worker_group, projection=None): 76 | return list(connections.mongodb_jobs.mrq_workers.find({ 77 | "config.worker_group": worker_group["_id"] 78 | }, projection=projection)) 79 | 80 | def fetch_worker_group_definitions(self): 81 | 82 | definitions = list(connections.mongodb_jobs.mrq_workergroups.find()) 83 | 84 | for definition in definitions: 85 | commands = [] 86 | # Prepend all commands by their worker group. 87 | for command in definition.get("commands", []): 88 | simplified_command, worker_count = normalize_command(command, definition["_id"]) 89 | commands.extend([simplified_command] * worker_count) 90 | definition["commands"] = commands 91 | 92 | return definitions 93 | 94 | def fetch_worker_group_agents(self, worker_group): 95 | return list(connections.mongodb_jobs.mrq_agents.find({"worker_group": worker_group["_id"], "status": "started"})) 96 | 97 | def get_config_for_profile(self, profile): 98 | parser = argparse.ArgumentParser() 99 | add_parser_args(parser, "worker") 100 | parts = shlex.split(profile["command"]) 101 | if "mrq-worker" in parts: 102 | parts = parts[parts.index("mrq-worker") + 1:] 103 | return parser.parse_args(parts) 104 | -------------------------------------------------------------------------------- /mrq/bin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pricingassistant/mrq/5a1154dc6a60d8b6d6423937f6569ce8fb0347b6/mrq/bin/__init__.py -------------------------------------------------------------------------------- /mrq/bin/mrq_agent.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | is_pypy = '__pypy__' in sys.builtin_module_names 5 | 6 | # Needed to make getaddrinfo() work in pymongo on Mac OS X 7 | # Docs mention it's a better choice for Linux as well. 8 | # This must be done asap in the worker 9 | if "GEVENT_RESOLVER" not in os.environ and not is_pypy: 10 | os.environ["GEVENT_RESOLVER"] = "ares" 11 | 12 | from gevent import monkey 13 | monkey.patch_all() 14 | 15 | import argparse 16 | 17 | sys.path.insert(0, os.getcwd()) 18 | 19 | from mrq import config 20 | from mrq.agent import Agent 21 | from mrq.context import set_current_config 22 | 23 | 24 | def main(): 25 | 26 | parser = argparse.ArgumentParser(description='Start a MRQ agent') 27 | 28 | cfg = config.get_config(parser=parser, config_type="agent", sources=("file", "env", "args")) 29 | 30 | set_current_config(cfg) 31 | 32 | agent = Agent() 33 | 34 | agent.work() 35 | 36 | sys.exit(agent.exitcode) 37 | 38 | 39 | if __name__ == "__main__": 40 | main() 41 | -------------------------------------------------------------------------------- /mrq/bin/mrq_run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | 4 | import os 5 | import sys 6 | is_pypy = '__pypy__' in sys.builtin_module_names 7 | 8 | # Needed to make getaddrinfo() work in pymongo on Mac OS X 9 | # Docs mention it's a better choice for Linux as well. 10 | # This must be done asap in the worker 11 | 12 | if "GEVENT_RESOLVER" not in os.environ and not is_pypy: 13 | os.environ["GEVENT_RESOLVER"] = "ares" 14 | 15 | # We must still monkey-patch the methods for job sub-pools. 16 | from gevent import monkey 17 | monkey.patch_all() 18 | 19 | import argparse 20 | import ujson as json 21 | import json as json_stdlib 22 | import datetime 23 | 24 | sys.path.insert(0, os.getcwd()) 25 | 26 | from mrq import config, utils 27 | from mrq.context import set_current_config, set_logger_config, set_current_job, connections 28 | from mrq.job import queue_job 29 | from mrq.utils import load_class_by_path, MongoJSONEncoder 30 | 31 | def main(): 32 | 33 | parser = argparse.ArgumentParser(description='Runs a task') 34 | 35 | cfg = config.get_config(parser=parser, config_type="run", sources=("file", "env", "args")) 36 | cfg["is_cli"] = True 37 | set_current_config(cfg) 38 | set_logger_config() 39 | 40 | if len(cfg["taskargs"]) == 1: 41 | params = json.loads(cfg["taskargs"][0]) # pylint: disable=no-member 42 | else: 43 | params = {} 44 | 45 | # mrq-run taskpath a 1 b 2 => {"a": "1", "b": "2"} 46 | for group in utils.group_iter(cfg["taskargs"], n=2): 47 | if len(group) != 2: 48 | print("Number of arguments wasn't even") 49 | sys.exit(1) 50 | params[group[0]] = group[1] 51 | 52 | if cfg["queue"]: 53 | ret = queue_job(cfg["taskpath"], params, queue=cfg["queue"]) 54 | print(ret) 55 | else: 56 | worker_class = load_class_by_path(cfg["worker_class"]) 57 | job = worker_class.job_class(None) 58 | job.set_data({ 59 | "path": cfg["taskpath"], 60 | "params": params, 61 | "queue": cfg["queue"] 62 | }) 63 | job.datestarted = datetime.datetime.utcnow() 64 | set_current_job(job) 65 | ret = job.perform() 66 | print(json_stdlib.dumps(ret, cls=MongoJSONEncoder)) # pylint: disable=no-member 67 | 68 | # This shouldn't be needed as the process will exit and close any remaining sockets 69 | # connections.redis.connection_pool.disconnect() 70 | 71 | if __name__ == "__main__": 72 | main() 73 | -------------------------------------------------------------------------------- /mrq/bin/mrq_worker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | from future.builtins import str 4 | import sys 5 | is_pypy = '__pypy__' in sys.builtin_module_names 6 | 7 | # Needed to make getaddrinfo() work in pymongo on Mac OS X 8 | # Docs mention it's a better choice for Linux as well. 9 | # This must be done asap in the worker 10 | if "GEVENT_RESOLVER" not in os.environ and not is_pypy: 11 | os.environ["GEVENT_RESOLVER"] = "ares" 12 | 13 | from gevent import monkey 14 | monkey.patch_all() 15 | 16 | import tempfile 17 | import signal 18 | import psutil 19 | import argparse 20 | import pipes 21 | 22 | sys.path.insert(0, os.getcwd()) 23 | 24 | from mrq import config 25 | from mrq.utils import load_class_by_path 26 | from mrq.context import set_current_config, set_logger_config 27 | 28 | 29 | def main(): 30 | 31 | parser = argparse.ArgumentParser(description='Start a MRQ worker') 32 | 33 | cfg = config.get_config(parser=parser, config_type="worker", sources=("file", "env", "args")) 34 | 35 | set_current_config(cfg) 36 | set_logger_config() 37 | 38 | # If we are launching with a --processes option and without MRQ_IS_SUBPROCESS, we are a manager process 39 | if cfg["processes"] > 0 and not os.environ.get("MRQ_IS_SUBPROCESS"): 40 | 41 | from mrq.supervisor import Supervisor 42 | 43 | command = " ".join(map(pipes.quote, sys.argv)) 44 | w = Supervisor(command, numprocs=cfg["processes"]) 45 | w.work() 46 | sys.exit(w.exitcode) 47 | 48 | # If not, start an actual worker 49 | else: 50 | 51 | worker_class = load_class_by_path(cfg["worker_class"]) 52 | w = worker_class() 53 | w.work() 54 | sys.exit(w.exitcode) 55 | 56 | if __name__ == "__main__": 57 | main() 58 | -------------------------------------------------------------------------------- /mrq/dashboard/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pricingassistant/mrq/5a1154dc6a60d8b6d6423937f6569ce8fb0347b6/mrq/dashboard/__init__.py -------------------------------------------------------------------------------- /mrq/dashboard/static/bin/64f2d23d70cb2b2810031880f554b13c.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pricingassistant/mrq/5a1154dc6a60d8b6d6423937f6569ce8fb0347b6/mrq/dashboard/static/bin/64f2d23d70cb2b2810031880f554b13c.png -------------------------------------------------------------------------------- /mrq/dashboard/static/bin/68ed1dac06bf0409c18ae7bc62889170.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pricingassistant/mrq/5a1154dc6a60d8b6d6423937f6569ce8fb0347b6/mrq/dashboard/static/bin/68ed1dac06bf0409c18ae7bc62889170.woff -------------------------------------------------------------------------------- /mrq/dashboard/static/bin/6c56b94fd0540844a7118cdff565b0ae.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pricingassistant/mrq/5a1154dc6a60d8b6d6423937f6569ce8fb0347b6/mrq/dashboard/static/bin/6c56b94fd0540844a7118cdff565b0ae.png -------------------------------------------------------------------------------- /mrq/dashboard/static/bin/7ad17c6085dee9a33787bac28fb23d46.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pricingassistant/mrq/5a1154dc6a60d8b6d6423937f6569ce8fb0347b6/mrq/dashboard/static/bin/7ad17c6085dee9a33787bac28fb23d46.eot -------------------------------------------------------------------------------- /mrq/dashboard/static/bin/8f88d990024975797f96ce7648dacd2f.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pricingassistant/mrq/5a1154dc6a60d8b6d6423937f6569ce8fb0347b6/mrq/dashboard/static/bin/8f88d990024975797f96ce7648dacd2f.png -------------------------------------------------------------------------------- /mrq/dashboard/static/bin/94b34ff5224ba38210d67623bb1a1504.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pricingassistant/mrq/5a1154dc6a60d8b6d6423937f6569ce8fb0347b6/mrq/dashboard/static/bin/94b34ff5224ba38210d67623bb1a1504.png -------------------------------------------------------------------------------- /mrq/dashboard/static/bin/bundle.js: -------------------------------------------------------------------------------- 1 | !function(n){function t(e){if(a[e])return a[e].exports;var r=a[e]={i:e,l:!1,exports:{}};return n[e].call(r.exports,r,r.exports,t),r.l=!0,r.exports}var e=window.webpackJsonp;window.webpackJsonp=function(t,a,i){for(var o,u,l=0,c=[];l'}if(a(this).data("type")!=undefined){i=a(this).data("type");if(i=="half"){a(this).append(''+D+s+"");a(this).find(".circle-text-half").css({"line-height":(F/1.45)+"px","font-size":t+"px"})}else{a(this).append(''+D+s+"");a(this).find(".circle-text").css({"line-height":F+"px","font-size":t+"px"})}}else{a(this).append(''+D+s+"");a(this).find(".circle-text").css({"line-height":F+"px","font-size":t+"px"})}}else{if(a(this).data("icon")!=undefined){}}if(a(this).data("info")!=undefined){E=a(this).data("info");if(a(this).data("type")!=undefined){i=a(this).data("type");if(i=="half"){a(this).append(''+E+"");a(this).find(".circle-info-half").css({"line-height":(F*0.9)+"px",})}else{a(this).append(''+E+"");a(this).find(".circle-info").css({"line-height":(F*1.25)+"px",})}}else{a(this).append(''+E+"");a(this).find(".circle-info").css({"line-height":(F*1.25)+"px",})}}a(this).width(F+"px");var h=a("").attr({width:F,height:F}).appendTo(a(this)).get(0);var f=h.getContext("2d");var p=h.width/2;var o=h.height/2;var A=e*360;var G=A*(Math.PI/180);var j=h.width/2.5;var z=2.3*Math.PI;var u=0;var C=false;var m=q===0?l:0;var n=Math.max(q,0);var r=Math.PI*2;var g=Math.PI/2;var i="";var w=false;if(a(this).data("type")!=undefined){i=a(this).data("type");if(i=="half"){var z=2*Math.PI;var u=3.13;var r=Math.PI*1;var g=Math.PI/0.996}}if(a(this).data("fill")!=undefined){w=a(this).data("fill")}else{w=c.fillColor}function k(x){f.clearRect(0,0,h.width,h.height);f.beginPath();f.arc(p,o,j,u,z,false);f.lineWidth=v-1;f.strokeStyle=d;f.stroke();if(w){f.fillStyle=w;f.fill()}f.beginPath();f.arc(p,o,j,-(g),((r)*x)-g,false);f.lineWidth=v;f.strokeStyle=B;f.stroke();if(m" + (source.datereported?moment.utc(source.datereported).fromNow():"Never") 71 | + "
" 72 | + "started " + moment.utc(source.datestarted).fromNow() + ""; 73 | } else { 74 | return source.datereported || ""; 75 | } 76 | } 77 | }, 78 | { 79 | "sTitle": "CPU", 80 | "sClass": "col-cpu", 81 | "sType":"numeric", 82 | "sWidth":"120px", 83 | "mData":function(source, type/*, val*/) { 84 | return source.total_cpu + " total
" + (source.free_cpu || "N/A") + " free"; 85 | } 86 | }, 87 | { 88 | "sTitle": "Memory", 89 | "sClass": "col-mem", 90 | "sType":"numeric", 91 | "sWidth":"130px", 92 | "mData":function(source, type/*, val*/) { 93 | return source.total_memory + " total
" + (source.free_memory || "N/A") + " free"; 94 | } 95 | }, { 96 | "sTitle": "Current Workers", 97 | "sClass": "col-mem", 98 | "sType":"string", 99 | "sWidth":"100%", 100 | "mData":function(source, type/*, val*/) { 101 | return "
" + source.current_workers.join("
") + "
"; 102 | } 103 | }, 104 | ], 105 | "aaSorting":[ [0,'asc'] ], 106 | }); 107 | 108 | this.initDataTable(datatableConfig); 109 | 110 | } 111 | }); 112 | 113 | }); 114 | -------------------------------------------------------------------------------- /mrq/dashboard/static/js/views/io.js: -------------------------------------------------------------------------------- 1 | define(["jquery", "underscore", "views/generic/datatablepage", "models"],function($, _, DataTablePage, Models) { 2 | 3 | return DataTablePage.extend({ 4 | 5 | el: '.js-page-io', 6 | 7 | template:"#tpl-page-io", 8 | 9 | events:{ 10 | }, 11 | 12 | renderDatatable:function() { 13 | 14 | var self = this; 15 | 16 | var unit_name = "ops"; 17 | 18 | var datatableConfig = { 19 | "aoColumns": [ 20 | 21 | { 22 | "sTitle": "Type", 23 | "sClass": "col-type", 24 | "sType": "string", 25 | "sWidth":"150px", 26 | "mData":function(source, type, val) { 27 | // console.log(source) 28 | return source.io.type; 29 | } 30 | }, 31 | { 32 | "sTitle": "Data", 33 | "sClass": "col-data", 34 | "sType":"string", 35 | "mData":function(source, type, val) { 36 | return "
"+_.escape(JSON.stringify(source.io.data, null, 0))+"
"; 37 | } 38 | }, 39 | { 40 | "sTitle": "Time", 41 | "sType":"string", 42 | "sWidth":"160px", 43 | "sClass": "col-jobs-time", 44 | "mData":function(source, type/*, val*/) { 45 | 46 | if (type == "display") { 47 | return "i/o started "+moment.utc(source.io.started*1000).fromNow(); 48 | } else { 49 | return source.io.started || ""; 50 | } 51 | } 52 | }, 53 | { 54 | "sTitle": "Path & ID", 55 | "sClass": "col-jobs-path", 56 | "sWidth":"300px", 57 | "mDataProp": "path", 58 | "mData": function ( source /*, val */) { 59 | return ""+source.path+""+ 60 | "

"+source.id+""; 61 | } 62 | }, 63 | { 64 | "sTitle": "Worker", 65 | "sType":"string", 66 | "sWidth":"200px", 67 | "sClass": "col-jobs-worker", 68 | "mData":function(source, type/*, val*/) { 69 | if (type == "display") { 70 | return source.worker?(""+source.worker+""):""; 71 | } else { 72 | return source.worker || ""; 73 | } 74 | } 75 | } 76 | ], 77 | "fnDrawCallback": function (oSettings) { 78 | 79 | _.each(oSettings.aoData,function(row) { 80 | var oData = row._aData; 81 | 82 | $(".col-jobs .inlinesparkline", row.nTr).sparkline("html", {"width": "100px", "height": "30px", "defaultPixelsPerValue": 1}); 83 | 84 | }); 85 | }, 86 | "aaSorting":[ [2,'desc'] ], 87 | "sPaginationType": "full_numbers", 88 | "iDisplayLength":100, 89 | "sDom":"iprtipl", 90 | "oLanguage":{ 91 | "sSearch":"", 92 | "sInfo": "Showing _START_ to _END_ of _TOTAL_ "+unit_name, 93 | "sEmptyTable":"No "+unit_name, 94 | "sInfoEmpty":"Showing 0 "+unit_name, 95 | "sInfoFiltered": "", 96 | "sLengthMenu": '' 101 | }, 102 | "sPaginationType": "bs_full", 103 | "bProcessing": true, 104 | "bServerSide": true, 105 | "bDeferRender": true, 106 | "bDestroy": true, 107 | "fnServerData": function (sSource, aoData, fnCallback) { 108 | self.loading = true; 109 | 110 | var params = {}; 111 | _.each(aoData, function(v) { 112 | params[v.name] = v.value; 113 | }); 114 | 115 | self.app.getJobsDataFromWorkers(function(err, data) { 116 | self.loading = false; 117 | self.trigger("loaded"); 118 | 119 | data = _.filter(data, function(row) { 120 | return (row.io || {}).type; 121 | }); 122 | 123 | if (!err) { 124 | fnCallback({ 125 | "aaData": data, 126 | "iTotalDisplayRecords": data.length, 127 | "sEcho": params.sEcho 128 | }) 129 | } 130 | }); 131 | } 132 | }; 133 | 134 | this.initDataTable(datatableConfig); 135 | } 136 | }); 137 | 138 | }); 139 | -------------------------------------------------------------------------------- /mrq/dashboard/static/js/views/root.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileoverview Defines the view container view that contains all views 3 | */ 4 | define(["views/generic/page", "jquery", 5 | "views/queues", "views/workers", "views/jobs", "views/scheduledjobs", "views/index", "views/taskpaths", "views/status", "views/taskexceptions", "views/io", "views/workergroups", "views/agents"], 6 | 7 | function( 8 | Page, $, 9 | QueuesView, WorkersView, JobsView, ScheduledJobsView, IndexView, TaskPathsView, StatusView, TaskExceptionsView, IOView, WorkerGroupsView, AgentsView 10 | ) { 11 | 12 | return Page.extend({ 13 | 14 | template:"#tpl-page-root", 15 | events:{ 16 | "change .js-store-select":"changestore" 17 | }, 18 | 19 | isTabVisible: true, 20 | 21 | init: function() { 22 | 23 | var self = this; 24 | 25 | // We want to reload once when the autorefresh rate changes 26 | $(".js-autorefresh").on("change", function() { 27 | self.trigger("visibilitychange"); 28 | }); 29 | 30 | 31 | // http://stackoverflow.com/questions/1060008/is-there-a-way-to-detect-if-a-browser-window-is-not-currently-active 32 | // (function() { 33 | // 34 | // var onchange = function(evt) { 35 | // 36 | // var prevVisible = self.isTabVisible; 37 | // var v = true, h = false, 38 | // evtMap = { 39 | // focus:v, focusin:v, pageshow:v, blur:h, focusout:h, pagehide:h 40 | // }; 41 | // 42 | // evt = evt || window.event; 43 | // if (evt.type in evtMap) 44 | // self.isTabVisible = evtMap[evt.type]; 45 | // else 46 | // self.isTabVisible = this[hidden] ? false : true; 47 | // 48 | // if (prevVisible != self.isTabVisible) { 49 | // self.trigger("visibilitychange"); 50 | // } 51 | // }; 52 | // 53 | // var hidden = "hidden"; 54 | // 55 | // // Standards: 56 | // if (hidden in document) 57 | // document.addEventListener("visibilitychange", onchange); 58 | // else if ((hidden = "mozHidden") in document) 59 | // document.addEventListener("mozvisibilitychange", onchange); 60 | // else if ((hidden = "webkitHidden") in document) 61 | // document.addEventListener("webkitvisibilitychange", onchange); 62 | // else if ((hidden = "msHidden") in document) 63 | // document.addEventListener("msvisibilitychange", onchange); 64 | // // IE 9 and lower: 65 | // else if ('onfocusin' in document) 66 | // document.onfocusin = document.onfocusout = onchange; 67 | // // All others: 68 | // else 69 | // window.onpageshow = window.onpagehide = window.onfocus = window.onblur = onchange; 70 | // 71 | // })(); 72 | }, 73 | 74 | /** 75 | * Shows a non blocking message to the user during a certain amount of time. 76 | * 77 | * @param {String} type : Type of alert ("warning", "error", "success", "info") 78 | * @param {String} message : Message to display 79 | * @param {Number} [timeout=5000] : Milliseconds before the alert is closed, if negative, will never close 80 | */ 81 | alert:function(type, message, timeout) { 82 | 83 | if (type=="clear") { 84 | $(".wizard-content .alert").alert('close'); 85 | return; 86 | } 87 | 88 | var html = '
'+ 89 | '×'+ 90 | '

'+message+'

'+ 91 | '
'; 92 | 93 | var $alert = $(".app-content").prepend(html).children().first(); 94 | $alert.alert(); 95 | 96 | if (timeout===undefined) { 97 | timeout = 5000; 98 | } 99 | if (timeout>0) { 100 | setTimeout(function() { 101 | $alert.alert('close'); 102 | },timeout); 103 | } 104 | }, 105 | 106 | render: function() { 107 | 108 | this.renderTemplate(); 109 | 110 | this.addChildPage('queues', new QueuesView()); 111 | this.addChildPage('io', new IOView()); 112 | this.addChildPage('workers', new WorkersView()); 113 | this.addChildPage('jobs', new JobsView()); 114 | this.addChildPage('scheduledjobs', new ScheduledJobsView()); 115 | this.addChildPage('taskpaths', new TaskPathsView()); 116 | this.addChildPage('taskexceptions', new TaskExceptionsView()); 117 | this.addChildPage('index', new IndexView()); 118 | this.addChildPage('status', new StatusView()); 119 | this.addChildPage('workergroups', new WorkerGroupsView()); 120 | this.addChildPage('agents', new AgentsView()); 121 | 122 | return this; 123 | } 124 | }); 125 | 126 | }); 127 | -------------------------------------------------------------------------------- /mrq/dashboard/static/js/views/scheduledjobs.js: -------------------------------------------------------------------------------- 1 | define(["jquery", "underscore", "views/generic/datatablepage", "models"],function($, _, DataTablePage, Models) { 2 | 3 | return DataTablePage.extend({ 4 | 5 | el: '.js-page-scheduledjobs', 6 | 7 | template:"#tpl-page-scheduledjobs", 8 | 9 | events:{ 10 | }, 11 | 12 | renderDatatable:function() { 13 | 14 | var self = this; 15 | 16 | var datatableConfig = self.getCommonDatatableConfig("scheduled_jobs"); 17 | 18 | _.extend(datatableConfig, { 19 | "aoColumns": [ 20 | 21 | { 22 | "sTitle": "Name", 23 | "sClass": "col-jobs-path", 24 | "mDataProp": "path", 25 | "fnRender": function ( o /*, val */) { 26 | return ""+o.aData.path+""+ 27 | "

"+o.aData._id+""; 28 | } 29 | }, 30 | 31 | { 32 | "sTitle": "Interval", 33 | "sType":"numeric", 34 | "sClass": "col-jobs-interval", 35 | "mData":function(source, type) { 36 | if (type == "display") { 37 | return moment.duration(source.interval*1000).humanize(); 38 | } 39 | return source.interval; 40 | } 41 | }, 42 | // { 43 | // "sTitle": "Daily time", 44 | // "sType":"string", 45 | // "sClass": "col-jobs-dailytime", 46 | // "mData":function(source, type) { 47 | // if (type == "display") { 48 | // return moment.duration(source.interval*1000).humanize(); 49 | // } 50 | // return source.interval; 51 | // } 52 | // }, 53 | { 54 | "sTitle": "Last Queued", 55 | "sType":"string", 56 | "sClass": "col-jobs-lastqueued", 57 | "mData":function(source, type) { 58 | return moment.utc(source.datelastqueued).fromNow(); 59 | } 60 | }, 61 | { 62 | "sTitle": "Params", 63 | "sClass": "col-jobs-params", 64 | "mDataProp": "params", 65 | "fnRender": function ( o /*, val */) { 66 | return "
"+_.escape(JSON.stringify(o.aData.params, null, 2))+"
"; 67 | } 68 | }, 69 | 70 | ], 71 | "aaSorting":[ [0,'asc'] ], 72 | }); 73 | 74 | this.initDataTable(datatableConfig); 75 | 76 | } 77 | }); 78 | 79 | }); 80 | -------------------------------------------------------------------------------- /mrq/dashboard/static/js/views/status.js: -------------------------------------------------------------------------------- 1 | define(["jquery", "underscore", "views/generic/datatablepage", "models", "moment"],function($, _, DataTablePage, Models, moment) { 2 | 3 | return DataTablePage.extend({ 4 | 5 | el: '.js-page-status', 6 | 7 | template:"#tpl-page-status", 8 | 9 | events:{ 10 | }, 11 | 12 | renderDatatable:function() { 13 | 14 | var self = this; 15 | 16 | var datatableConfig = self.getCommonDatatableConfig("status"); 17 | 18 | _.extend(datatableConfig, { 19 | "aoColumns": [ 20 | 21 | { 22 | "sTitle": "Status", 23 | "sClass": "col-status", 24 | "sType":"string", 25 | "sWidth":"150px", 26 | "mData":function(source, type/*, val*/) { 27 | return ""+source._id+""; 28 | } 29 | }, 30 | { 31 | "sTitle": "Jobs", 32 | "sClass": "col-jobs", 33 | "sType":"numeric", 34 | "sWidth":"120px", 35 | "mData":function(source, type/*, val*/) { 36 | var cnt = (source.jobs || 0); 37 | if (type == "display") { 38 | return ""+cnt+"" 39 | + "
" 40 | + ''; 41 | } else { 42 | return cnt; 43 | } 44 | } 45 | }, 46 | { 47 | "sTitle": "Speed", 48 | "sClass": "col-eta", 49 | "sType":"numeric", 50 | "mData":function(source, type, val) { 51 | console.log() 52 | return (Math.round(self.getCounterSpeed("index.status."+source._id) * 100) / 100) + " jobs/second"; 53 | } 54 | }, 55 | { 56 | "sTitle": "ETA", 57 | "sClass": "col-eta", 58 | "sType":"numeric", 59 | "mData":function(source, type, val) { 60 | return self.getCounterEta("index.status."+source._id, source.jobs || 0); 61 | } 62 | } 63 | 64 | ], 65 | "fnDrawCallback": function (oSettings) { 66 | 67 | _.each(oSettings.aoData,function(row) { 68 | var oData = row._aData; 69 | 70 | $(".col-jobs .inlinesparkline", row.nTr).sparkline("html", {"width": "100px", "height": "30px", "defaultPixelsPerValue": 1}); 71 | 72 | }); 73 | }, 74 | "aaSorting":[ [0,'asc'] ], 75 | }); 76 | 77 | this.initDataTable(datatableConfig); 78 | 79 | } 80 | }); 81 | 82 | }); 83 | -------------------------------------------------------------------------------- /mrq/dashboard/static/js/views/taskexceptions.js: -------------------------------------------------------------------------------- 1 | define(["jquery", "underscore", "views/generic/datatablepage", "models"],function($, _, DataTablePage, Models) { 2 | 3 | return DataTablePage.extend({ 4 | 5 | el: '.js-page-taskexceptions', 6 | 7 | template:"#tpl-page-taskexceptions", 8 | 9 | events:{ 10 | }, 11 | 12 | renderDatatable:function() { 13 | 14 | var self = this; 15 | 16 | var datatableConfig = self.getCommonDatatableConfig("taskexceptions"); 17 | 18 | _.extend(datatableConfig, { 19 | "aoColumns": [ 20 | 21 | { 22 | "sTitle": "Name", 23 | "sClass": "col-name", 24 | "sType": "string", 25 | "mData":function(source, type, val) { 26 | // console.log(source) 27 | return ""+source._id.path+""; 28 | } 29 | }, 30 | { 31 | "sTitle": "Exception", 32 | "sClass": "col-exception", 33 | "sType":"numeric", 34 | "mData":function(source, type, val) { 35 | return ""+source._id.exceptiontype+"" 36 | } 37 | }, 38 | { 39 | "sTitle": "Jobs", 40 | "sClass": "col-jobs", 41 | "sType":"numeric", 42 | "mData":function(source, type, val) { 43 | var cnt = source.jobs || 0; 44 | 45 | if (type == "display") { 46 | return ""+cnt+"" 47 | + "
" 48 | + ''; 49 | } else { 50 | return cnt; 51 | } 52 | } 53 | }, 54 | { 55 | "sTitle": "Speed", 56 | "sClass": "col-eta", 57 | "sType":"numeric", 58 | "mData":function(source, type, val) { 59 | return (Math.round(self.getCounterSpeed("taskexceptions."+source._id.path+" "+source._id.exceptiontype) * 100) / 100) + " jobs/second"; 60 | } 61 | }, 62 | { 63 | "sTitle": "ETA", 64 | "sClass": "col-eta", 65 | "sType":"numeric", 66 | "mData":function(source, type, val) { 67 | return self.getCounterEta("taskexceptions."+source._id.path+" "+source._id.exceptiontype, source.jobs || 0); 68 | } 69 | } 70 | 71 | ], 72 | "fnDrawCallback": function (oSettings) { 73 | 74 | _.each(oSettings.aoData,function(row) { 75 | var oData = row._aData; 76 | 77 | $(".col-jobs .inlinesparkline", row.nTr).sparkline("html", {"width": "100px", "height": "30px", "defaultPixelsPerValue": 1}); 78 | 79 | }); 80 | }, 81 | "aaSorting":[ [0,'asc'] ], 82 | }); 83 | 84 | this.initDataTable(datatableConfig); 85 | 86 | } 87 | }); 88 | 89 | }); 90 | -------------------------------------------------------------------------------- /mrq/dashboard/static/js/views/taskpaths.js: -------------------------------------------------------------------------------- 1 | define(["jquery", "underscore", "views/generic/datatablepage", "models"],function($, _, DataTablePage, Models) { 2 | 3 | return DataTablePage.extend({ 4 | 5 | el: '.js-page-taskpaths', 6 | 7 | template:"#tpl-page-taskpaths", 8 | 9 | events:{ 10 | }, 11 | 12 | renderDatatable:function() { 13 | 14 | var self = this; 15 | 16 | var datatableConfig = self.getCommonDatatableConfig("taskpaths"); 17 | 18 | _.extend(datatableConfig, { 19 | "aoColumns": [ 20 | 21 | { 22 | "sTitle": "Name", 23 | "sClass": "col-name", 24 | "sType": "string", 25 | "mData":function(source, type, val) { 26 | return ""+source._id+""; 27 | } 28 | }, 29 | { 30 | "sTitle": "Jobs", 31 | "sClass": "col-jobs", 32 | "sType":"numeric", 33 | "mData":function(source, type, val) { 34 | var cnt = source.jobs || 0; 35 | 36 | if (type == "display") { 37 | return ""+cnt+"" 38 | + "
" 39 | + ''; 40 | } else { 41 | return cnt; 42 | } 43 | } 44 | }, 45 | { 46 | "sTitle": "Speed", 47 | "sClass": "col-eta", 48 | "sType":"numeric", 49 | "mData":function(source, type, val) { 50 | return (Math.round(self.getCounterSpeed("taskpath."+source._id) * 100) / 100) + " jobs/second"; 51 | } 52 | }, 53 | { 54 | "sTitle": "ETA", 55 | "sClass": "col-eta", 56 | "sType":"numeric", 57 | "mData":function(source, type, val) { 58 | return self.getCounterEta("taskpath."+source._id, source.jobs || 0); 59 | } 60 | } 61 | 62 | ], 63 | "fnDrawCallback": function (oSettings) { 64 | 65 | _.each(oSettings.aoData,function(row) { 66 | var oData = row._aData; 67 | 68 | $(".col-jobs .inlinesparkline", row.nTr).sparkline("html", {"width": "100px", "height": "30px", "defaultPixelsPerValue": 1}); 69 | 70 | }); 71 | }, 72 | "aaSorting":[ [0,'asc'] ], 73 | }); 74 | 75 | this.initDataTable(datatableConfig); 76 | 77 | } 78 | }); 79 | 80 | }); 81 | -------------------------------------------------------------------------------- /mrq/dashboard/static/js/views/workergroups.js: -------------------------------------------------------------------------------- 1 | define(["jquery", "underscore", "models", "views/generic/page"],function($, _, Models, Page) { 2 | 3 | return Page.extend({ 4 | 5 | el: '.js-page-workergroups', 6 | 7 | template:"#tpl-page-workergroups", 8 | 9 | events:{ 10 | "click .submit": "submit" 11 | }, 12 | 13 | render: function() { 14 | var self = this; 15 | $.get("api/workergroups").done(function(data) { 16 | self.renderTemplate(); 17 | self.$("textarea").val(JSON.stringify(data["workergroups"], null, 8)); 18 | }); 19 | }, 20 | 21 | submit: function(el) { 22 | var self = this; 23 | 24 | self.$("button")[0].innerHTML = "Wait..."; 25 | 26 | var val = self.$("textarea").val(); 27 | 28 | $.post("api/workergroups", {"workergroups": val}).done(function(data) { 29 | if (data.status != "ok") { 30 | return alert("There was an error while saving!"); 31 | } 32 | self.$("button")[0].innerHTML = "Save"; 33 | }); 34 | } 35 | }); 36 | 37 | }); 38 | -------------------------------------------------------------------------------- /mrq/dashboard/static/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mrq-dashboard", 3 | "version": "0.9.1", 4 | "description": "PricingAssistant MRQ dashboard", 5 | "dependencies": {}, 6 | "engines": { 7 | "node": "^7.5.0" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/pricingassistant/mrq" 12 | }, 13 | "scripts": { 14 | "build": "webpack --config webpack.config.js" 15 | }, 16 | "author": "Pricing Assistant", 17 | "license": "MIT", 18 | "devDependencies": { 19 | "css-loader": "0.28.7", 20 | "file-loader": "0.11.2", 21 | "style-loader": "0.18.2", 22 | "uglifyjs-webpack-plugin": "0.4.6", 23 | "webpack": "3.1.0" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /mrq/dashboard/static/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const UglifyJSPlugin = require('uglifyjs-webpack-plugin'); 3 | const webpack = require('webpack'); 4 | 5 | module.exports = { 6 | entry: './js/main.js', 7 | resolve: { 8 | modules: ['./'], 9 | alias: { 10 | circliful: "js/vendor/jquery.circliful.min", 11 | jquery: "js/vendor/jquery-2.1.0.min", 12 | underscore: "js/vendor/underscore.min", 13 | backbone: "js/vendor/backbone.min", 14 | backbonequeryparams: "js/vendor/backbone.queryparams", 15 | bootstrap: "js/vendor/bootstrap.min", 16 | datatables: "js/vendor/jquery.dataTables.min", 17 | datatablesbs3: "js/vendor/datatables.bs3", 18 | moment: "js/vendor/moment.min", 19 | sparkline: "js/vendor/jquery.sparkline.min", 20 | } 21 | }, 22 | output: { 23 | path: path.resolve(__dirname, 'bin'), 24 | publicPath: '/static/bin/', 25 | filename: 'bundle.js' 26 | }, 27 | plugins: [ 28 | new webpack.IgnorePlugin(/^\.\/lang$/, /.*/), 29 | new webpack.ProvidePlugin({ 30 | $: "jquery", 31 | jQuery: "jquery", 32 | "window.jQuery": "jquery", 33 | _: "underscore", 34 | Backbone: "backbone", 35 | moment: "moment" 36 | }), 37 | new UglifyJSPlugin({compress: {warnings: false}}) 38 | ], 39 | module: { 40 | loaders: [ 41 | {test: /\.css$/, use: [ 'style-loader', 'css-loader' ]}, 42 | {test: /\.(png|woff|woff2|eot|ttf|svg)$/, loader: 'file-loader'} 43 | ] 44 | } 45 | }; 46 | -------------------------------------------------------------------------------- /mrq/dashboard/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | # from werkzeug import Response 3 | from functools import wraps 4 | from flask import request, Response 5 | from mrq.context import get_current_config 6 | from mrq.utils import MongoJSONEncoder 7 | 8 | 9 | def jsonify(*args, **kwargs): 10 | """ jsonify with support for MongoDB ObjectId 11 | """ 12 | return Response( 13 | json.dumps( 14 | dict( 15 | *args, 16 | **kwargs), 17 | cls=MongoJSONEncoder), 18 | mimetype='application/json') 19 | 20 | 21 | def check_auth(username, pwd): 22 | """This function is called to check if a username / 23 | password combination is valid. 24 | """ 25 | cfg = get_current_config() 26 | return username == cfg["dashboard_httpauth"].split( 27 | ":")[0] and pwd == cfg["dashboard_httpauth"].split(":")[1] 28 | 29 | 30 | def authenticate(): 31 | """Sends a 401 response that enables basic auth""" 32 | return Response( 33 | 'Could not verify your access level for that URL.\n' 34 | 'You have to login with proper credentials', 401, 35 | {'WWW-Authenticate': 'Basic realm="Login Required"'} 36 | ) 37 | 38 | 39 | def requires_auth(f): 40 | 41 | cfg = get_current_config() 42 | if not cfg["dashboard_httpauth"]: 43 | return f 44 | 45 | @wraps(f) 46 | def decorated(*args, **kwargs): 47 | auth = request.authorization 48 | if not auth or not check_auth(auth.username, auth.password): 49 | return authenticate() 50 | return f(*args, **kwargs) 51 | return decorated 52 | -------------------------------------------------------------------------------- /mrq/dashboard/uwsgi-heroku.ini: -------------------------------------------------------------------------------- 1 | [uwsgi] 2 | http-socket = :$(PORT) 3 | master = true 4 | processes = 4 5 | die-on-term = true 6 | module = mrq.dashboard.app:app 7 | memory-report = true 8 | gevent = 30 -------------------------------------------------------------------------------- /mrq/exceptions.py: -------------------------------------------------------------------------------- 1 | from gevent import GreenletExit 2 | import traceback 3 | 4 | 5 | # Inherits from BaseException to avoid being caught when not intended. 6 | class _MrqInterrupt(BaseException): 7 | 8 | original_exception = None 9 | 10 | def _get_exception_name(self): 11 | return self.__class__.__name__ 12 | 13 | def __str__(self): 14 | s = self._get_exception_name() 15 | if self.original_exception is not None: 16 | tb = "".join(traceback.format_exception(*self.original_exception)) # pylint: disable=not-an-iterable 17 | s += "\n---- Original exception: -----\n%s" % tb 18 | 19 | return s 20 | 21 | 22 | class TimeoutInterrupt(_MrqInterrupt): 23 | pass 24 | 25 | 26 | class AbortInterrupt(_MrqInterrupt): 27 | pass 28 | 29 | 30 | class RetryInterrupt(_MrqInterrupt): 31 | delay = None 32 | queue = None 33 | retry_count = 0 34 | 35 | def _get_exception_name(self): 36 | return "%s #%s: %s seconds, %s queue" % ( 37 | self.__class__.__name__, self.retry_count, self.delay, self.queue 38 | ) 39 | 40 | 41 | class MaxRetriesInterrupt(_MrqInterrupt): 42 | pass 43 | 44 | 45 | class StopRequested(GreenletExit): 46 | """ Thrown in the mail greenlet to stop dequeuing jobs. """ 47 | pass 48 | 49 | 50 | class JobInterrupt(GreenletExit): 51 | """ Interrupts that stop a job in its execution, e.g. when responding to a SIGTERM. """ 52 | pass 53 | 54 | 55 | class MaxConcurrencyInterrupt(_MrqInterrupt): 56 | pass 57 | -------------------------------------------------------------------------------- /mrq/helpers.py: -------------------------------------------------------------------------------- 1 | """ Helpers are util functions which use the context """ 2 | from .context import connections, get_current_config 3 | import time 4 | 5 | 6 | def ratelimit(key, limit, per=1, redis=None): 7 | """ Returns an integer with the number of available actions for the 8 | current period in seconds. If zero, rate was already reached. """ 9 | 10 | if redis is None: 11 | redis = connections.redis 12 | 13 | # http://redis.io/commands/INCR 14 | now = int(time.time()) 15 | 16 | k = "ratelimit:%s:%s" % (key, now // per) 17 | 18 | with redis.pipeline(transaction=True) as pipeline: 19 | pipeline.incr(k, 1) 20 | pipeline.expire(k, per + 10) 21 | value = pipeline.execute() 22 | 23 | current = int(value[0]) - 1 24 | 25 | if current >= limit: 26 | return 0 27 | else: 28 | return limit - current 29 | 30 | 31 | def metric(name, incr=1, **kwargs): 32 | cfg = get_current_config() 33 | if cfg.get("metric_hook"): 34 | return cfg.get("metric_hook")(name, incr=incr, **kwargs) 35 | -------------------------------------------------------------------------------- /mrq/logger.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from future.builtins import object 3 | from future.utils import iteritems 4 | 5 | from collections import defaultdict 6 | import logging 7 | import datetime 8 | import sys 9 | import pymongo 10 | PY3 = sys.version_info > (3,) 11 | 12 | 13 | def _encode_if_unicode(string): 14 | 15 | if PY3: 16 | return string 17 | 18 | if isinstance(string, unicode): # pylint: disable=undefined-variable 19 | return string.encode("utf-8", "replace") 20 | else: 21 | return string 22 | 23 | 24 | def _decode_if_str(string): 25 | 26 | if PY3: 27 | return str(string) 28 | 29 | if isinstance(string, str): 30 | return string.decode("utf-8", "replace") 31 | else: 32 | return unicode(string) # pylint: disable=undefined-variable 33 | 34 | 35 | class MongoHandler(logging.Handler): 36 | 37 | """ Job/Worker-aware log handler. 38 | 39 | We used the standard logging module before but it suffers from memory leaks 40 | when creating lots of logger objects. 41 | """ 42 | 43 | def __init__(self, worker=None, mongodb_logs_size=16 * 1024 * 1024): 44 | super(MongoHandler, self).__init__() 45 | 46 | self.buffer = {} 47 | self.collection = None 48 | self.mongodb_logs_size = mongodb_logs_size 49 | 50 | self.reset() 51 | self.set_collection() 52 | # Import here to avoid import loop 53 | # pylint: disable=cyclic-import 54 | from .context import get_current_job, get_current_worker 55 | self.get_current_job = get_current_job 56 | self.worker = worker 57 | 58 | def set_collection(self): 59 | from .context import get_current_config, connections 60 | config = get_current_config() 61 | collection = config["mongodb_logs"] 62 | 63 | if collection == "1": 64 | self.collection = connections.mongodb_logs.mrq_logs 65 | if self.collection and self.mongodb_logs_size: 66 | if "mrq_logs" in connections.mongodb_logs.collection_names() and not self.collection.options().get("capped"): 67 | connections.mongodb_logs.command({"convertToCapped": "mrq_logs", "size": self.mongodb_logs_size}) 68 | elif "mrq_logs" not in connections.mongodb_logs.collection_names(): 69 | try: 70 | connections.mongodb_logs.create_collection("mrq_logs", capped=True, size=self.mongodb_logs_size) 71 | except pymongo.errors.OperationFailure: # The collection might have been created in the meantime 72 | pass 73 | 74 | def reset(self): 75 | self.buffer = { 76 | "workers": defaultdict(list), 77 | "jobs": defaultdict(list) 78 | } 79 | 80 | def emit(self, record): 81 | log_entry = self.format(record) 82 | if self.collection is False: 83 | return 84 | log_entry = _decode_if_str(log_entry) 85 | 86 | if self.worker is not None: 87 | self.buffer["workers"][self.worker].append(log_entry) 88 | 89 | if record.name == "mrq.current": 90 | job_object = self.get_current_job() 91 | if job_object: 92 | self.buffer["jobs"][job_object.id].append(log_entry) 93 | 94 | def flush(self): 95 | # We may log some stuff before we are even connected to Mongo! 96 | if not self.collection: 97 | return 98 | 99 | inserts = [{ 100 | "worker": k, 101 | "logs": "\n".join(v) + "\n" 102 | } for k, v in iteritems(self.buffer["workers"])] + [{ 103 | "job": k, 104 | "logs": "\n".join(v) + "\n" 105 | } for k, v in iteritems(self.buffer["jobs"])] 106 | 107 | if len(inserts) == 0: 108 | return 109 | self.reset() 110 | 111 | try: 112 | self.collection.insert(inserts) 113 | except Exception as e: # pylint: disable=broad-except 114 | self.emit("Log insert failed: %s" % e) 115 | -------------------------------------------------------------------------------- /mrq/redishelpers.py: -------------------------------------------------------------------------------- 1 | from future.builtins import range 2 | from .utils import memoize 3 | from . import context 4 | 5 | 6 | def redis_key(name, *args): 7 | prefix = context.get_current_config()["redis_prefix"] 8 | if name == "known_subqueues": 9 | return "%s:ksq:%s" % (prefix, args[0].root_id) 10 | elif name == "queue": 11 | return "%s:q:%s" % (prefix, args[0].id) 12 | elif name == "started_jobs": 13 | return "%s:s:started" % prefix 14 | elif name == "paused_queues": 15 | return "%s:s:paused" % prefix 16 | elif name == "notify": 17 | return "%s:notify:%s" % (prefix, args[0].root_id) 18 | 19 | 20 | @memoize 21 | def redis_zaddbyscore(): 22 | """ Increments multiple keys in a sorted set & returns them """ 23 | 24 | return context.connections.redis.register_script(""" 25 | local zset = KEYS[1] 26 | local min = ARGV[1] 27 | local max = ARGV[2] 28 | local offset = ARGV[3] 29 | local count = ARGV[4] 30 | local score = ARGV[5] 31 | 32 | local data = redis.call('zrangebyscore', zset, min, max, 'LIMIT', offset, count) 33 | for i, member in pairs(data) do 34 | redis.call('zadd', zset, score, member) 35 | end 36 | 37 | return data 38 | """) 39 | 40 | 41 | @memoize 42 | def redis_zpopbyscore(): 43 | """ Pops multiple keys by score """ 44 | 45 | return context.connections.redis.register_script(""" 46 | local zset = KEYS[1] 47 | local min = ARGV[1] 48 | local max = ARGV[2] 49 | local offset = ARGV[3] 50 | local count = ARGV[4] 51 | 52 | local data = redis.call('zrangebyscore', zset, min, max, 'LIMIT', offset, count) 53 | if #data > 0 then 54 | redis.call('zremrangebyrank', zset, 0, #data - 1) 55 | end 56 | 57 | return data 58 | """) 59 | 60 | 61 | @memoize 62 | def redis_lpopsafe(): 63 | """ Safe version of LPOP that also adds the key in a "started" zset """ 64 | 65 | return context.connections.redis.register_script(""" 66 | local key = KEYS[1] 67 | local zset_started = KEYS[2] 68 | local count = ARGV[1] 69 | local now = ARGV[2] 70 | local left = ARGV[3] 71 | local data = {} 72 | local current = nil 73 | 74 | for i=1, count do 75 | if left == '1' then 76 | current = redis.call('lpop', key) 77 | else 78 | current = redis.call('rpop', key) 79 | end 80 | if current == false then 81 | return data 82 | end 83 | data[i] = current 84 | redis.call('zadd', zset_started, now, current) 85 | end 86 | 87 | return data 88 | """) 89 | 90 | 91 | def redis_group_command(command, cnt, redis_key): 92 | with context.connections.redis.pipeline(transaction=False) as pipe: 93 | for _ in range(cnt): 94 | getattr(pipe, command)(redis_key) 95 | return [x for x in pipe.execute() if x] 96 | -------------------------------------------------------------------------------- /mrq/subpool.py: -------------------------------------------------------------------------------- 1 | from itertools import count as itertools_count 2 | import traceback 3 | import time 4 | import gevent 5 | 6 | 7 | def subpool_map(pool_size, func, iterable): 8 | """ Starts a Gevent pool and run a map. Takes care of setting current_job and cleaning up. """ 9 | 10 | from .context import get_current_job, set_current_job, log 11 | 12 | if not pool_size: 13 | return [func(*args) for args in iterable] 14 | 15 | counter = itertools_count() 16 | 17 | current_job = get_current_job() 18 | 19 | def inner_func(*args): 20 | """ As each call to 'func' will be done in a random greenlet of the subpool, we need to 21 | register their IDs with set_current_job() to make get_current_job() calls work properly 22 | inside 'func'. 23 | """ 24 | next(counter) 25 | if current_job: 26 | set_current_job(current_job) 27 | 28 | try: 29 | ret = func(*args) 30 | except Exception as exc: 31 | trace = traceback.format_exc() 32 | exc.subpool_traceback = trace 33 | raise 34 | 35 | if current_job: 36 | set_current_job(None) 37 | return ret 38 | 39 | def inner_iterable(): 40 | """ This will be called inside the pool's main greenlet, which ID also needs to be registered """ 41 | if current_job: 42 | set_current_job(current_job) 43 | 44 | for x in iterable: 45 | yield x 46 | 47 | if current_job: 48 | set_current_job(None) 49 | 50 | start_time = time.time() 51 | pool = gevent.pool.Pool(size=pool_size) 52 | ret = pool.map(inner_func, inner_iterable()) 53 | pool.join(raise_error=True) 54 | total_time = time.time() - start_time 55 | 56 | log.debug("SubPool ran %s greenlets in %0.6fs" % (counter, total_time)) 57 | 58 | return ret 59 | 60 | 61 | def subpool_imap(pool_size, func, iterable, flatten=False, unordered=False, buffer_size=None): 62 | """ Generator version of subpool_map. Should be used with unordered=True for optimal performance """ 63 | 64 | from .context import get_current_job, set_current_job, log 65 | 66 | if not pool_size: 67 | for args in iterable: 68 | yield func(*args) 69 | 70 | counter = itertools_count() 71 | 72 | current_job = get_current_job() 73 | 74 | def inner_func(*args): 75 | """ As each call to 'func' will be done in a random greenlet of the subpool, we need to 76 | register their IDs with set_current_job() to make get_current_job() calls work properly 77 | inside 'func'. 78 | """ 79 | next(counter) 80 | if current_job: 81 | set_current_job(current_job) 82 | 83 | try: 84 | ret = func(*args) 85 | except Exception as exc: 86 | trace = traceback.format_exc() 87 | exc.subpool_traceback = trace 88 | raise 89 | 90 | if current_job: 91 | set_current_job(None) 92 | return ret 93 | 94 | def inner_iterable(): 95 | """ This will be called inside the pool's main greenlet, which ID also needs to be registered """ 96 | if current_job: 97 | set_current_job(current_job) 98 | 99 | for x in iterable: 100 | yield x 101 | 102 | if current_job: 103 | set_current_job(None) 104 | 105 | start_time = time.time() 106 | pool = gevent.pool.Pool(size=pool_size) 107 | 108 | if unordered: 109 | iterator = pool.imap_unordered(inner_func, inner_iterable(), maxsize=buffer_size or pool_size) 110 | else: 111 | iterator = pool.imap(inner_func, inner_iterable()) 112 | 113 | for x in iterator: 114 | if flatten: 115 | for y in x: 116 | yield y 117 | else: 118 | yield x 119 | 120 | pool.join(raise_error=True) 121 | total_time = time.time() - start_time 122 | 123 | log.debug("SubPool ran %s greenlets in %0.6fs" % (counter, total_time)) 124 | -------------------------------------------------------------------------------- /mrq/supervisor.py: -------------------------------------------------------------------------------- 1 | from future.builtins import object 2 | 3 | from .context import get_current_config, connections, log 4 | from bson import ObjectId 5 | from collections import defaultdict 6 | from .processes import ProcessPool, Process 7 | 8 | 9 | class Supervisor(Process): 10 | """ Manages several mrq-worker single processes """ 11 | 12 | def __init__(self, command, numprocs=1): 13 | self.numprocs = numprocs 14 | self.command = command 15 | self.pool = ProcessPool() 16 | 17 | def work(self): 18 | 19 | self.install_signal_handlers() 20 | 21 | self.pool.set_commands([self.command] * self.numprocs) 22 | 23 | self.pool.start() 24 | 25 | self.pool.wait() 26 | 27 | def shutdown_now(self): 28 | self.pool.terminate() 29 | 30 | def shutdown_graceful(self): 31 | self.pool.stop(timeout=None) 32 | -------------------------------------------------------------------------------- /mrq/task.py: -------------------------------------------------------------------------------- 1 | from future.builtins import object 2 | 3 | 4 | class Task(object): 5 | 6 | # Are we the first task that a Job called? 7 | is_main_task = False 8 | max_concurrency = 0 9 | 10 | # Default write concern values when setting status=success 11 | # http://docs.mongodb.org/manual/reference/write-concern/ 12 | status_success_update_w = None 13 | status_success_update_j = None 14 | 15 | def __init__(self): 16 | pass 17 | 18 | def run_wrapped(self, params): 19 | """ Override this method to provide your own wrapping code """ 20 | return self.run(params) 21 | 22 | def run(self, params): 23 | """ Override this method with the main code of all your tasks """ 24 | raise NotImplementedError 25 | -------------------------------------------------------------------------------- /mrq/version.py: -------------------------------------------------------------------------------- 1 | VERSION = "0.9.10" 2 | 3 | if __name__ == "__main__": 4 | import sys 5 | sys.stdout.write(VERSION) 6 | -------------------------------------------------------------------------------- /requirements-base.txt: -------------------------------------------------------------------------------- 1 | argparse>=1.1 2 | redis==2.10.6 3 | pymongo==3.7.2 4 | gevent>=1.2.2 5 | ujson>=1.33 6 | hiredis>=0.1.5 7 | psutil>=5.1.2,<=5.6.3 # https://github.com/giampaolo/psutil/issues/1659#issuecomment-586032229 8 | objgraph>=1.8.1 9 | termcolor>=1.1.0 10 | subprocess32>=3.2.7; python_version < '3.2' and sys.platform != "win32" 11 | future>=0.15.2 12 | importlib>=1.0.3; python_version < '2.7' 13 | -------------------------------------------------------------------------------- /requirements-dashboard.txt: -------------------------------------------------------------------------------- 1 | Flask>=0.11.1 -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest==2.9.1 2 | pylint==1.7.1 3 | pytest-cov==2.4.0 4 | pytest-instafail==0.3.0 5 | pytest-html==1.14.2 6 | pytest-httpbin==1.0.0 7 | pytest-timeout==1.2.0 8 | 9 | subprocess32>=3.2.7; python_version < '3.2' 10 | # git+https://github.com/srlindsay/gevent-profiler@master#egg=gevent-profiler==0.2 11 | 12 | # Used to test IO tracing 13 | requests>=2.20.0 14 | 15 | mkdocs==0.15.3 16 | mistune>=0.8.1 17 | -------------------------------------------------------------------------------- /requirements-heroku.txt: -------------------------------------------------------------------------------- 1 | uwsgi>=2.0.2 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Heroku only reads requirements.txt 2 | # As the main usecase is to deploy the dashboard there, this is the default setting. 3 | -r requirements-heroku.txt 4 | 5 | -r requirements-base.txt 6 | -r requirements-dashboard.txt 7 | -------------------------------------------------------------------------------- /scripts/git-set-file-times: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | use strict; 3 | 4 | # sets mtime and atime of files to the latest commit time in git 5 | # 6 | # This is useful for serving static content (managed by git) 7 | # from a cluster of identically configured HTTP servers. HTTP 8 | # clients and content delivery networks can get consistent 9 | # Last-Modified headers no matter which HTTP server in the 10 | # cluster they hit. This should improve caching behavior. 11 | # 12 | # This does not take into account merges, but if you're updating 13 | # every machine in the cluster from the same commit (A) to the 14 | # same commit (B), the mtimes will be _consistent_ across all 15 | # machines if not necessarily accurate. 16 | # 17 | # THIS IS NOT INTENDED TO OPTIMIZE BUILD SYSTEMS SUCH AS 'make' 18 | # YOU HAVE BEEN WARNED! 19 | 20 | my %ls = (); 21 | my $commit_time; 22 | 23 | if ($ENV{GIT_DIR}) { 24 | chdir($ENV{GIT_DIR}) or die $!; 25 | } 26 | 27 | $/ = "\0"; 28 | open FH, 'git ls-files -z|' or die $!; 29 | while () { 30 | chomp; 31 | $ls{$_} = $_; 32 | } 33 | close FH; 34 | 35 | 36 | $/ = "\n"; 37 | open FH, "git log -m -r --name-only --no-color --pretty=raw -z @ARGV |" or die $!; 38 | while () { 39 | chomp; 40 | if (/^committer .*? (\d+) (?:[\-\+]\d+)$/) { 41 | $commit_time = $1; 42 | } elsif (s/\0\0commit [a-f0-9]{40}( \(from [a-f0-9]{40}\))?$// or s/\0$//) { 43 | my @files = delete @ls{split(/\0/, $_)}; 44 | @files = grep { defined $_ } @files; 45 | next unless @files; 46 | utime $commit_time, $commit_time, @files; 47 | } 48 | last unless %ls; 49 | 50 | } 51 | close FH; -------------------------------------------------------------------------------- /scripts/propagate_docs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import subprocess 4 | import sys 5 | import mistune 6 | 7 | CURRENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) 8 | 9 | 10 | """ This script overrides the README with parts contained in the docs : 11 | index.md => Intro, Why, Main Features 12 | get-started => Get Started 13 | """ 14 | 15 | res = subprocess.check_output(["git", "branch"]) 16 | if "* master" not in res: 17 | print "you have to be on master to run this !" 18 | sys.exit(1) 19 | 20 | 21 | def get_path(filename): 22 | return os.path.join(CURRENT_DIRECTORY, "..", "docs", "%s.md" % filename) 23 | 24 | readme_path = os.path.join(CURRENT_DIRECTORY, "..", "README.md") 25 | 26 | with open(readme_path, 'r') as readme_file: 27 | readme = readme_file.read() 28 | 29 | for filename, title_begin, title_end in ( 30 | ("index", "MRQ", "Dashboard Screenshots"), 31 | ("get-started", "Get Started", "More"), 32 | ): 33 | with open(get_path(filename), 'r') as f: 34 | raw = f.read() 35 | # remove first 2 lines 36 | cropped = "\n".join(raw.split("\n")[2:]) 37 | 38 | readme = re.sub( 39 | "(# %s\n\n)(.*)(\n\n^# %s)" % (title_begin, title_end), 40 | r"\1%s\3" % cropped, 41 | readme, 42 | flags=re.MULTILINE | re.DOTALL 43 | ) 44 | 45 | with open(readme_path, 'w') as readme_file: 46 | readme_file.write(readme) 47 | 48 | subprocess.call(["git", "add", "-p", "README.md"]) 49 | subprocess.call(["git", "commit"]) 50 | 51 | # II. index.html 52 | raw = "" 53 | for filename in ["index", "dashboard", "get-started"]: 54 | with open(get_path(filename), 'r') as f: 55 | raw += f.read() 56 | raw += "\n\n" 57 | subprocess.call(["git", "checkout", "gh-pages"]) 58 | html = mistune.markdown(raw) 59 | 60 | index_path = os.path.join("index.html") 61 | 62 | with open(index_path, 'r') as index_file: 63 | index = index_file.read() 64 | index = re.sub( 65 | r"(
)(.*)(
)", 66 | r"\1%s\3" % html, 67 | index, 68 | flags=re.MULTILINE | re.DOTALL 69 | ) 70 | with open(index_path, 'w') as index_file: 71 | index_file.write(index) 72 | subprocess.call(["git", "add", "-p", "index.html"]) 73 | subprocess.call(["git", "commit"]) 74 | # subprocess.call(["git", "checkout", "master"]) 75 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup # , find_packages 2 | import os 3 | import sys 4 | 5 | 6 | def use_package(package): 7 | if not package: 8 | return False 9 | if package.startswith(('#', 'git+')): 10 | return False 11 | if sys.version_info.major > 2 and 'python_version <' in package: 12 | return False 13 | if sys.version_info.major == 2 and 'python_version >' in package: 14 | return False 15 | 16 | return True 17 | 18 | 19 | def get_requirements(): 20 | reqs = [] 21 | for filename in ["requirements-base.txt", "requirements-dashboard.txt"]: 22 | with open(filename, "r") as f: 23 | reqs += [x.strip().split(";")[0] for x in f.readlines() if use_package(x.strip())] 24 | return reqs 25 | 26 | 27 | def get_version(): 28 | basedir = os.path.dirname(__file__) 29 | with open(os.path.join(basedir, 'mrq/version.py')) as f: 30 | locals = {} 31 | exec(f.read(), locals) 32 | return locals['VERSION'] 33 | raise RuntimeError('No version info found.') 34 | 35 | setup( 36 | name="mrq", 37 | include_package_data=True, 38 | packages=['mrq', 'mrq.basetasks', 'mrq.bin', 'mrq.dashboard'], # find_packages(exclude=['tests', 'tests.tasks']), 39 | version=get_version(), 40 | description="A simple yet powerful distributed worker task queue in Python", 41 | author="Pricing Assistant", 42 | license='MIT', 43 | author_email="contact@pricingassistant.com", 44 | url="http://github.com/pricingassistant/mrq", 45 | # download_url="http://chardet.feedparser.org/download/python3-chardet-1.0.1.tgz", 46 | keywords=["worker", "task", "distributed", "queue", "asynchronous", "redis", "mongodb", "job", "processing", "gevent"], 47 | platforms='any', 48 | entry_points={ 49 | 'console_scripts': [ 50 | 'mrq-worker = mrq.bin.mrq_worker:main', 51 | 'mrq-run = mrq.bin.mrq_run:main', 52 | 'mrq-agent = mrq.bin.mrq_agent:main', 53 | 'mrq-dashboard = mrq.dashboard.app:main' 54 | ] 55 | }, 56 | # dependency_links=[ 57 | # "http://github.com/mongodb/mongo-python-driver/archive/cb4adb2193a83413bc5545d89b7bbde4d6087761.zip#egg=pymongo-2.7rc1" 58 | # ], 59 | zip_safe=False, 60 | install_requires=get_requirements(), 61 | classifiers=[ 62 | "Programming Language :: Python", 63 | "Programming Language :: Python :: 2.7", 64 | "Programming Language :: Python :: 3.5", 65 | "Programming Language :: Python :: 3.6", 66 | "Programming Language :: Python :: 3.7", 67 | #'Development Status :: 1 - Planning', 68 | #'Development Status :: 2 - Pre-Alpha', 69 | #'Development Status :: 3 - Alpha', 70 | #'Development Status :: 4 - Beta', 71 | 'Development Status :: 5 - Production/Stable', 72 | #'Development Status :: 6 - Mature', 73 | #'Development Status :: 7 - Inactive', 74 | "Environment :: Other Environment", 75 | "Intended Audience :: Developers", 76 | "License :: OSI Approved :: MIT License", 77 | "Operating System :: OS Independent", 78 | "Topic :: Utilities" 79 | ], 80 | long_description=open("README.md").read() 81 | ) 82 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pricingassistant/mrq/5a1154dc6a60d8b6d6423937f6569ce8fb0347b6/tests/__init__.py -------------------------------------------------------------------------------- /tests/fixtures/config-io-hooks.py: -------------------------------------------------------------------------------- 1 | 2 | # Read from tests.tasks.general.GetIoHookEvents 3 | IO_EVENTS = [] 4 | 5 | 6 | def MONGODB_PRE_HOOK(event): 7 | event["hook"] = "mongodb_pre" 8 | IO_EVENTS.append(event) 9 | 10 | 11 | def MONGODB_POST_HOOK(event): 12 | event["hook"] = "mongodb_post" 13 | IO_EVENTS.append(event) 14 | 15 | 16 | def REDIS_PRE_HOOK(event): 17 | event["hook"] = "redis_pre" 18 | IO_EVENTS.append(event) 19 | 20 | 21 | def REDIS_POST_HOOK(event): 22 | event["hook"] = "redis_post" 23 | IO_EVENTS.append(event) 24 | -------------------------------------------------------------------------------- /tests/fixtures/config-logger-capped.py: -------------------------------------------------------------------------------- 1 | 2 | LOG_HANDLERS = { 3 | "mrq.logger.MongoHandler": { 4 | "mongodb_logs_size": 16 * 1024 * 1024 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /tests/fixtures/config-logger.py: -------------------------------------------------------------------------------- 1 | 2 | LOG_HANDLERS = { 3 | "logging.FileHandler": { 4 | "filename": "/tmp/mrq.log" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /tests/fixtures/config-lostjobs.py: -------------------------------------------------------------------------------- 1 | SIMULATE_ZOMBIE_JOBS = True 2 | -------------------------------------------------------------------------------- /tests/fixtures/config-metric.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | 3 | # Read from tests.tasks.general.GetMetrics 4 | TEST_GLOBAL_METRICS = defaultdict(int) 5 | 6 | 7 | def METRIC_HOOK(name, incr=1, **kwargs): 8 | TEST_GLOBAL_METRICS[name] += incr 9 | -------------------------------------------------------------------------------- /tests/fixtures/config-multiredis.py: -------------------------------------------------------------------------------- 1 | NAME = "testworker-multiredis" 2 | 3 | REDIS = "redis://127.0.0.1:6379" 4 | 5 | REDIS_SECOND = "redis://127.0.0.1:6379/1" 6 | -------------------------------------------------------------------------------- /tests/fixtures/config-notify.py: -------------------------------------------------------------------------------- 1 | 2 | QUEUES_CONFIG = { 3 | "withnotify": { 4 | "notify": True 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /tests/fixtures/config-raw1.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | RAW_QUEUES = { 4 | "pushback_timed_set": { 5 | "pushback_seconds": 24 * 3600, 6 | "job_factory": lambda rawparam: { 7 | "path": "tests.tasks.general.MongoInsert", 8 | "params": { 9 | "timed_set": rawparam 10 | } 11 | } 12 | }, 13 | "test_timed_set": { 14 | "dashboard_graph": lambda: { 15 | "start": time.time() - (24 * 3600), 16 | "stop": time.time() + (24 * 3600), 17 | "slices": 30, 18 | "include_inf": True, 19 | "exact": False 20 | }, 21 | "job_factory": lambda rawparam: { 22 | "path": "tests.tasks.general.MongoInsert", 23 | "params": { 24 | "timed_set": rawparam 25 | } 26 | } 27 | }, 28 | "test_raw": { 29 | "has_subqueues": True, 30 | "job_factory": lambda rawparam: { 31 | "path": "tests.tasks.general.MongoInsert", 32 | "params": { 33 | "raw": rawparam 34 | } 35 | } 36 | }, 37 | "test_set": { 38 | "has_subqueues": True, 39 | "job_factory": lambda rawparam: { 40 | "path": "tests.tasks.general.MongoInsert", 41 | "params": { 42 | "set": rawparam 43 | } 44 | } 45 | }, 46 | "test_sorted_set": { 47 | "job_factory": lambda rawparam: { 48 | "path": "tests.tasks.general.MongoInsert", 49 | "params": { 50 | "sorted_set": rawparam 51 | } 52 | } 53 | }, 54 | "testexception_raw": { 55 | "retry_queue": "testx", 56 | "job_factory": lambda rawparam: { 57 | "path": "tests.tasks.general.RaiseException", 58 | "params": { 59 | "message": rawparam 60 | } 61 | } 62 | }, 63 | "testretry_raw": { 64 | "retry_queue": "testx", 65 | "job_factory": lambda rawparam: { 66 | "path": "tests.tasks.general.Retry", 67 | "params": { 68 | "sleep": int(rawparam), 69 | "delay": 0 70 | } 71 | } 72 | }, 73 | "testperformance_raw": { 74 | "job_factory": lambda rawparam: { 75 | "path": "tests.tasks.general.Add", 76 | "params": { 77 | "a": int(rawparam), 78 | "b": 0, 79 | "sleep": 0 80 | } 81 | } 82 | }, 83 | "testperformance_set": { 84 | "job_factory": lambda rawparam: { 85 | "path": "tests.tasks.general.Add", 86 | "params": { 87 | "a": int(rawparam), 88 | "b": 0, 89 | "sleep": 0 90 | } 91 | } 92 | }, 93 | "testperformance_timed_set": { 94 | "job_factory": lambda rawparam: { 95 | "path": "tests.tasks.general.Add", 96 | "params": { 97 | "a": int(rawparam), 98 | "b": 0, 99 | "sleep": 0 100 | } 101 | } 102 | }, 103 | "testperformance_sorted_set": { 104 | "job_factory": lambda rawparam: { 105 | "path": "tests.tasks.general.Add", 106 | "params": { 107 | "a": int(rawparam), 108 | "b": 0, 109 | "sleep": 0 110 | } 111 | } 112 | }, 113 | "testperformance_efficiency_raw": { 114 | "job_factory": lambda rawparam: { 115 | "path": "tests.tasks.general.Add", 116 | "params": { 117 | "a": 1, 118 | "b": 2, 119 | "sleep": float(rawparam) 120 | } 121 | } 122 | }, 123 | "testperformance_efficiency_nostorage_raw": { 124 | "statuses_no_storage": ("started", "success"), 125 | "job_factory": lambda rawparam: { 126 | "path": "tests.tasks.general.Add", 127 | "params": { 128 | "a": 1, 129 | "b": 2, 130 | "sleep": float(rawparam) 131 | } 132 | } 133 | }, 134 | "teststarted_raw": { 135 | "retry_queue": "teststartedx", 136 | "job_factory": lambda rawparam: { 137 | "path": "tests.tasks.general.WaitForFlag", 138 | "params": { 139 | "flag": rawparam 140 | } 141 | } 142 | }, 143 | "testnostorage_raw": { 144 | "retry_queue": "testnostorage", 145 | "statuses_no_storage": ("started", "success"), 146 | "job_factory": lambda rawparam: { 147 | "path": rawparam.split(" ")[0], 148 | "params": { 149 | "sleep": float(rawparam.split(" ")[1]) 150 | } 151 | } 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /tests/fixtures/config-retry1.py: -------------------------------------------------------------------------------- 1 | 2 | TASKS = { 3 | "tests.tasks.general.Retry": { 4 | "max_retries": 1, 5 | "retry_delay": 1 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /tests/fixtures/config-scheduler-invalid1.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | 4 | SCHEDULER_TASKS = [ 5 | { 6 | "path": "tests.tasks.general.MongoInsert", 7 | "params": { 8 | "weekday": i 9 | }, 10 | "weekday": i 11 | } for i in range(7) 12 | ] 13 | 14 | SCHEDULER_INTERVAL = 1 15 | -------------------------------------------------------------------------------- /tests/fixtures/config-scheduler-invalid2.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | 4 | SCHEDULER_TASKS = [ 5 | { 6 | "path": "tests.tasks.general.MongoInsert", 7 | "params": { 8 | "monthday": i 9 | }, 10 | "monthday": i 11 | } for i in range(7) 12 | ] 13 | 14 | SCHEDULER_INTERVAL = 1 15 | -------------------------------------------------------------------------------- /tests/fixtures/config-scheduler-invalid3.py: -------------------------------------------------------------------------------- 1 | 2 | SCHEDULER_TASKS = [ 3 | { 4 | "path": "tests.tasks.general.MongoInsert", 5 | "params": { 6 | "a": 1 7 | }, 8 | "interval": 5 9 | }, 10 | { 11 | "path": "tests.tasks.general.MongoInsert", 12 | "params": { 13 | "a": 1 14 | }, 15 | "interval": 5 16 | }, 17 | { 18 | "path": "tests.tasks.general.MongoInsert", 19 | "params": { 20 | "a": 3 21 | }, 22 | "interval": 5 23 | } 24 | ] 25 | 26 | SCHEDULER_INTERVAL = 1 27 | -------------------------------------------------------------------------------- /tests/fixtures/config-scheduler-invalid4.py: -------------------------------------------------------------------------------- 1 | 2 | SCHEDULER_TASKS = [ 3 | { 4 | "path": "tests.tasks.general.MongoInsert", 5 | "params": { 6 | "a": 1 7 | } 8 | } 9 | ] 10 | 11 | SCHEDULER_INTERVAL = 1 12 | -------------------------------------------------------------------------------- /tests/fixtures/config-scheduler1.py: -------------------------------------------------------------------------------- 1 | 2 | SCHEDULER_TASKS = [ 3 | { 4 | "path": "tests.tasks.general.MongoInsert", 5 | "params": { 6 | "a": 1 7 | }, 8 | "interval": 5 9 | }, 10 | { 11 | "path": "tests.tasks.general.MongoInsert", 12 | "params": { 13 | "a": 2 14 | }, 15 | "interval": 5 16 | }, 17 | { 18 | "path": "tests.tasks.general.MongoInsert", 19 | "params": { 20 | "a": 3 21 | }, 22 | "interval": 5 23 | }, 24 | { 25 | "path": "tests.tasks.general.MongoInsert", 26 | "params": { 27 | "a": 4 28 | }, 29 | "interval": 5 30 | } 31 | ] 32 | 33 | SCHEDULER_INTERVAL = 0.1 34 | -------------------------------------------------------------------------------- /tests/fixtures/config-scheduler2.py: -------------------------------------------------------------------------------- 1 | 2 | SCHEDULER_TASKS = [ 3 | { 4 | "path": "tests.tasks.general.MongoInsert", 5 | "params": { 6 | "a": 1 7 | }, 8 | "interval": 5 9 | }, 10 | { 11 | "path": "tests.tasks.general.MongoInsert", 12 | "params": { 13 | "a": 20 14 | }, 15 | "interval": 5 16 | }, 17 | { 18 | "path": "tests.tasks.general.MongoInsert", 19 | "params": { 20 | "a": 3 21 | }, 22 | "interval": 10 23 | }, 24 | { 25 | "path": "tests.tasks.general.MongoInsert2", 26 | "params": { 27 | "a": 4 28 | }, 29 | "interval": 5 30 | } 31 | ] 32 | 33 | SCHEDULER_INTERVAL = 1 34 | -------------------------------------------------------------------------------- /tests/fixtures/config-scheduler3.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | 4 | SCHEDULER_TASKS = [ 5 | { 6 | "path": "tests.tasks.general.MongoInsert", 7 | "params": { 8 | "a": 1 9 | }, 10 | "dailytime": datetime.datetime.fromtimestamp(float(os.environ.get("MRQ_TEST_SCHEDULER_TIME"))).time() 11 | }, 12 | { 13 | "path": "tests.tasks.general.MongoInsert", 14 | "params": { 15 | "a": 1, 16 | "b": "test", 17 | "c": 3.0 18 | }, 19 | "dailytime": datetime.datetime.fromtimestamp(float(os.environ.get("MRQ_TEST_SCHEDULER_TIME"))).time() 20 | } 21 | ] 22 | 23 | SCHEDULER_INTERVAL = 1 24 | -------------------------------------------------------------------------------- /tests/fixtures/config-scheduler5.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | 4 | SCHEDULER_INTERVAL = 1 5 | SCHEDULER_TASKS = [] 6 | 7 | for i in range(7): 8 | SCHEDULER_TASKS.extend([ 9 | { 10 | "path": "tests.tasks.general.MongoInsert", 11 | "params": { 12 | "weekday": i, 13 | "later": False 14 | }, 15 | "weekday": i, 16 | "dailytime": datetime.datetime.fromtimestamp(float(os.environ.get("MRQ_TEST_SCHEDULER_TIME"))).time() 17 | }, 18 | { 19 | "path": "tests.tasks.general.MongoInsert", 20 | "params": { 21 | "weekday": i, 22 | "later": True 23 | }, 24 | "weekday": i, 25 | "dailytime": datetime.datetime.fromtimestamp(float(os.environ.get("MRQ_TEST_SCHEDULER_TIME")) + 1000).time() 26 | }]) 27 | -------------------------------------------------------------------------------- /tests/fixtures/config-scheduler6.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | 4 | SCHEDULER_TASKS = [ 5 | { 6 | "path": "tests.tasks.general.MongoInsert", 7 | "params": { 8 | "monthday": i + 1 9 | }, 10 | "monthday": i + 1, 11 | "dailytime": datetime.datetime.fromtimestamp(float(os.environ.get("MRQ_TEST_SCHEDULER_TIME"))).time() 12 | } for i in range(31) 13 | ] 14 | 15 | SCHEDULER_INTERVAL = 1 16 | -------------------------------------------------------------------------------- /tests/fixtures/config-scheduler7.py: -------------------------------------------------------------------------------- 1 | 2 | SCHEDULER_TASKS = [ 3 | { 4 | "path": "tests.tasks.general.MongoInsert", 5 | "interval": 5 6 | }, 7 | { 8 | "path": "tests.tasks.general.MongoInsert", 9 | "interval": 6 10 | }, 11 | { 12 | "path": "tests.tasks.general.Add", 13 | "interval": 5 14 | } 15 | ] 16 | 17 | SCHEDULER_INTERVAL = 1 18 | -------------------------------------------------------------------------------- /tests/fixtures/config-scheduler8.py: -------------------------------------------------------------------------------- 1 | NAME = "testworker" 2 | 3 | TASKS = { 4 | "tests.tasks.general.Retry": { 5 | "default_ttl": 2 * 24 * 3600, 6 | "queue": "tests" 7 | } 8 | } 9 | 10 | QUEUES = ["high", "default", "low", "tests"] 11 | -------------------------------------------------------------------------------- /tests/fixtures/config-shorttimeout.py: -------------------------------------------------------------------------------- 1 | TASKS = { 2 | "tests.tasks.general.Add": { 3 | "timeout": 200 4 | } 5 | } 6 | 7 | NAME = "worker-shorttimeout" 8 | -------------------------------------------------------------------------------- /tests/fixtures/config1.py: -------------------------------------------------------------------------------- 1 | NAME = "testworker" 2 | 3 | TASKS = { 4 | "tests.tasks.general.TimeoutFromConfig": { 5 | "timeout": 2, 6 | "queue": "tests" 7 | } 8 | } 9 | 10 | QUEUES = ["high", "default", "low", "tests"] 11 | -------------------------------------------------------------------------------- /tests/fixtures/config2.py: -------------------------------------------------------------------------------- 1 | NAME = "testworker" 2 | 3 | # There configs should be added transparently. 4 | ADDITIONAL_UNEXPECTED_CONFIG = "1" 5 | 6 | MONGODB_JOBS = "mongodb://127.0.0.1:27017/mrq?connectTimeoutMS=4242" 7 | 8 | LOG_LEVEL = "INFO" 9 | -------------------------------------------------------------------------------- /tests/fixtures/httpstatic/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Oh hai. 4 | 5 | -------------------------------------------------------------------------------- /tests/fixtures/httpstatic/nginx.conf: -------------------------------------------------------------------------------- 1 | daemon off; 2 | worker_processes 1; 3 | 4 | events { 5 | worker_connections 768; 6 | # multi_accept on; 7 | } 8 | 9 | http { 10 | server { 11 | listen 8081; 12 | server_name localhost; 13 | 14 | index index.html; 15 | root /app/tests/fixtures/httpstatic/; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/fixtures/standalone_script1.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from mrq.context import setup_context, run_task, get_current_config 3 | 4 | # Autoconfigure MRQ's environment 5 | setup_context() 6 | 7 | print(run_task("tests.tasks.general.Add", {"a": 41, "b": 1})) 8 | 9 | print(get_current_config()["name"]) -------------------------------------------------------------------------------- /tests/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pricingassistant/mrq/5a1154dc6a60d8b6d6423937f6569ce8fb0347b6/tests/tasks/__init__.py -------------------------------------------------------------------------------- /tests/tasks/agent.py: -------------------------------------------------------------------------------- 1 | from mrq.task import Task 2 | from mrq.context import connections 3 | 4 | 5 | class Autoscale(Task): 6 | 7 | def run(self, params): 8 | 9 | connections.mongodb_jobs.tests_inserts.insert({"params": params}, manipulate=False) 10 | 11 | return params 12 | -------------------------------------------------------------------------------- /tests/tasks/concurrency.py: -------------------------------------------------------------------------------- 1 | import time 2 | from mrq.task import Task 3 | from mrq.context import log 4 | from .general import Add 5 | 6 | class LockedAdd(Add): 7 | 8 | max_concurrency = 1 9 | 10 | def run(self, params): 11 | log.info("adding", params) 12 | res = params.get("a", 0) + params.get("b", 0) 13 | 14 | if params.get("sleep", 0): 15 | log.info("sleeping %d", params.get("sleep", 0)) 16 | time.sleep(params.get("sleep", 0)) 17 | 18 | return res 19 | -------------------------------------------------------------------------------- /tests/tasks/context.py: -------------------------------------------------------------------------------- 1 | from mrq.task import Task 2 | from mrq.context import log, get_current_job, get_current_worker, get_current_config 3 | import os 4 | 5 | 6 | class GetContext(Task): 7 | 8 | def run(self, params): 9 | log.info("Getting context info...") 10 | return { 11 | "job_id": get_current_job().id, 12 | "worker_id": get_current_worker().id, 13 | "config": get_current_config(), 14 | "environ": os.environ 15 | } 16 | -------------------------------------------------------------------------------- /tests/tasks/io.py: -------------------------------------------------------------------------------- 1 | from future import standard_library 2 | standard_library.install_aliases() 3 | 4 | # Evil workaround to disable SSL verification 5 | import ssl 6 | ctx = ssl.create_default_context() 7 | ctx.check_hostname = False 8 | ctx.verify_mode = ssl.CERT_NONE 9 | 10 | from mrq.task import Task 11 | from mrq.context import connections, log 12 | import urllib.request, urllib.error, urllib.parse 13 | from future.moves.urllib.request import urlopen 14 | 15 | 16 | class TestIo(Task): 17 | 18 | def run(self, params): 19 | 20 | log.info("I/O starting") 21 | ret = self._run(params) 22 | log.info("I/O finished") 23 | 24 | return ret 25 | 26 | def _run(self, params): 27 | 28 | if params["test"] == "mongodb-insert": 29 | 30 | return connections.mongodb_jobs.tests_inserts.insert({"params": params["params"]}, manipulate=False) 31 | 32 | elif params["test"] == "mongodb-find": 33 | 34 | cursor = connections.mongodb_jobs.tests_inserts.find({"test": "x"}) 35 | return list(cursor) 36 | 37 | elif params["test"] == "mongodb-count": 38 | 39 | return connections.mongodb_jobs.tests_inserts.count() 40 | 41 | elif params["test"] == "mongodb-full-getmore": 42 | 43 | connections.mongodb_jobs.tests_inserts.insert_many([{"a": 1}, {"a": 2}]) 44 | 45 | return list(connections.mongodb_jobs.tests_inserts.find(batch_size=1)) 46 | 47 | elif params["test"] == "redis-llen": 48 | 49 | return connections.redis.llen(params["params"]["key"]) 50 | 51 | elif params["test"] == "redis-lpush": 52 | 53 | return connections.redis.lpush(params["params"]["key"], "xxx") 54 | 55 | elif params["test"] == "urllib2-get": 56 | 57 | return urlopen(params["params"]["url"], context=ctx).read() 58 | 59 | elif params["test"] == "urllib2-post": 60 | 61 | return urlopen(params["params"]["url"], data="x=x", context=ctx).read() 62 | 63 | elif params["test"] == "requests-get": 64 | 65 | import requests 66 | return requests.get(params["params"]["url"], verify=False).text 67 | -------------------------------------------------------------------------------- /tests/tasks/logger.py: -------------------------------------------------------------------------------- 1 | from mrq.task import Task 2 | from mrq.context import log 3 | import sys 4 | PY3 = sys.version_info > (3,) 5 | 6 | 7 | class Simple(Task): 8 | 9 | def run(self, params): 10 | 11 | 12 | # Some systems may be configured like this. 13 | if not PY3 and params.get("utf8_sys_stdout"): 14 | import codecs 15 | import sys 16 | UTF8Writer = codecs.getwriter('utf8') 17 | sys.stdout = UTF8Writer(sys.stdout) 18 | if params["class_name"] == "unicode": 19 | log.info(u"caf\xe9") 20 | elif params["class_name"] == "string": 21 | log.info("cafe") 22 | elif params["class_name"] == "latin-1": 23 | log.info("caf\xe9") 24 | elif params["class_name"] == "bytes1": 25 | log.info("Mat\xc3\xa9riels d'entra\xc3\xaenement") 26 | 27 | return True 28 | -------------------------------------------------------------------------------- /tests/tasks/mongodb.py: -------------------------------------------------------------------------------- 1 | from mrq.task import Task 2 | from mrq.context import connections 3 | 4 | 5 | class MongoTimeout(Task): 6 | 7 | def run(self, params): 8 | 9 | res = connections.mongodb_jobs.eval(""" 10 | function() { 11 | var a; 12 | for (i=0;i<10000000;i++) {  13 | for (y=0;y<10000000;y++) { 14 | a = Math.max(y); 15 | } 16 | } 17 | return a; 18 | } 19 | """) 20 | 21 | return res 22 | -------------------------------------------------------------------------------- /tests/tasks/redis.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from builtins import range 3 | from mrq.task import Task 4 | from mrq.context import connections, subpool_map 5 | import gevent 6 | import time 7 | 8 | 9 | class MultiRedis(Task): 10 | 11 | def run(self, params): 12 | 13 | connections.redis.set("test", "xxx") 14 | connections.redis_second.set("test", "yyy") 15 | 16 | assert connections.redis.get("test") == "xxx" 17 | assert connections.redis_second.get("test") == "yyy" 18 | 19 | return "ok" 20 | 21 | 22 | class Disconnections(Task): 23 | 24 | def run(self, params): 25 | 26 | def get_clients(): 27 | return [c for c in connections.redis.client_list() if c.get("cmd") != "client"] 28 | 29 | def inner(i): 30 | print("Greenlet #%s, %s clients so far" % (id(gevent.getcurrent()), len(get_clients()))) 31 | return connections.redis.get("test") 32 | 33 | if params["subpool_size"]: 34 | subpool_map(params["subpool_size"], inner, list(range(0, params["subpool_size"] * 5))) 35 | else: 36 | inner(0) 37 | -------------------------------------------------------------------------------- /tests/test_abort.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from datetime import timedelta 3 | 4 | from mrq.queue import Queue 5 | 6 | 7 | def test_abort(worker): 8 | 9 | worker.start() 10 | 11 | worker.send_task("tests.tasks.general.Abort", {"a": 41}, accept_statuses=["abort"]) 12 | 13 | assert Queue("default").size() == 0 14 | 15 | db_jobs = list(worker.mongodb_jobs.mrq_jobs.find()) 16 | assert len(db_jobs) == 1 17 | 18 | job = db_jobs[0] 19 | assert job["status"] == "abort" 20 | assert job.get("dateexpires") is not None 21 | assert job["dateexpires"] < datetime.utcnow() + timedelta(hours=24) 22 | 23 | 24 | def test_abort_traceback_history(worker): 25 | 26 | worker.start() 27 | 28 | worker.send_task("tests.tasks.general.Abort", {"a": 41}, block=True, accept_statuses=["abort"]) 29 | 30 | job = worker.mongodb_jobs.mrq_jobs.find()[0] 31 | 32 | assert len(job["traceback_history"]) == 1 33 | assert not job["traceback_history"][0].get("original_traceback") 34 | 35 | worker.send_task("tests.tasks.general.AbortOnFailed", {"a": 41}, block=True, accept_statuses=["abort"]) 36 | 37 | job = worker.mongodb_jobs.mrq_jobs.find({"path": "tests.tasks.general.AbortOnFailed"})[0] 38 | 39 | assert len(job["traceback_history"]) == 1 40 | assert "InAbortException" in job["traceback_history"][0].get("original_traceback") 41 | -------------------------------------------------------------------------------- /tests/test_cancel.py: -------------------------------------------------------------------------------- 1 | from mrq.job import Job 2 | from mrq.queue import Queue 3 | import time 4 | 5 | 6 | def test_cancel_by_path(worker): 7 | 8 | # Start the worker with only one greenlet so that tasks execute 9 | # sequentially 10 | worker.start(flags="--greenlets 1") 11 | 12 | job_id1 = worker.send_task( 13 | "tests.tasks.general.MongoInsert", {"a": 41, "sleep": 2}, block=False) 14 | worker.wait_for_idle() 15 | 16 | job_id2 = worker.send_task( 17 | "tests.tasks.general.MongoInsert", {"a": 43}, block=False, queue="testMrq") 18 | 19 | worker.send_task("mrq.basetasks.utils.JobAction", { 20 | "path": "tests.tasks.general.MongoInsert", 21 | "status": "queued", 22 | "action": "cancel" 23 | }, block=False) 24 | worker.wait_for_idle() 25 | 26 | # Leave some time to unqueue job_id2 without executing. 27 | time.sleep(1) 28 | worker.stop(deps=False) 29 | 30 | job1 = Job(job_id1).fetch().data 31 | job2 = Job(job_id2).fetch().data 32 | 33 | assert job1["status"] == "success" 34 | assert job1["result"] == {"a": 41, "sleep": 2} 35 | 36 | assert job2["status"] == "cancel" 37 | assert job2["dateexpires"] > job2["dateupdated"] 38 | 39 | assert job2.get("result") is None 40 | 41 | assert worker.mongodb_jobs.tests_inserts.count() == 1 42 | 43 | assert Queue("default").size() == 0 44 | 45 | worker.stop_deps() 46 | -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | from mrq.job import Job 2 | 3 | 4 | def test_cli_run_blocking(worker): 5 | 6 | worker.start_deps() 7 | 8 | result = worker.send_task_cli("tests.tasks.general.Add", {"a": 41, "b": 1}, queue=False) 9 | 10 | assert result == 42 11 | 12 | worker.stop_deps() 13 | 14 | 15 | def test_cli_run_nonblocking(worker): 16 | 17 | worker.start() 18 | 19 | job_id1 = worker.send_task_cli( 20 | "tests.tasks.general.Add", {"a": 41, "b": 1}, queue="default") 21 | 22 | job1 = Job(job_id1).fetch() 23 | 24 | job1.wait(poll_interval=0.01) 25 | 26 | job1.fetch() 27 | 28 | assert job1.data["status"] == "success" 29 | assert job1.data["result"] == 42 30 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | 4 | def test_config(worker): 5 | """ Test different config passing options. """ 6 | 7 | # Default config values 8 | worker.start() 9 | 10 | cfg = json.loads( 11 | worker.send_task("tests.tasks.general.GetConfig", {}, block=True)) 12 | 13 | assert cfg["mongodb_jobs"] == "mongodb://127.0.0.1:27017/mrq" 14 | assert cfg.get("additional_unexpected_config") is None 15 | 16 | worker.stop() 17 | 18 | # Values from config file 19 | worker.start(flags="--config tests/fixtures/config2.py") 20 | 21 | cfg = json.loads( 22 | worker.send_task("tests.tasks.general.GetConfig", {}, block=True)) 23 | 24 | assert cfg["mongodb_jobs"] == "mongodb://127.0.0.1:27017/mrq?connectTimeoutMS=4242" 25 | assert cfg["name"] == "testworker" 26 | assert cfg.get("additional_unexpected_config") == "1" 27 | 28 | worker.stop() 29 | 30 | # CLI > Config file && CLI > ENV 31 | worker.start(flags="--config tests/fixtures/config2.py --name xxx", env={"MRQ_NAME": "yyy"}) 32 | 33 | cfg = json.loads( 34 | worker.send_task("tests.tasks.general.GetConfig", {}, block=True)) 35 | 36 | assert cfg["name"] == "xxx" 37 | assert cfg.get("additional_unexpected_config") == "1" 38 | 39 | worker.stop() 40 | 41 | # ENV > Config file 42 | worker.start(flags="--config tests/fixtures/config2.py", env={"MRQ_NAME": "yyy"}) 43 | 44 | cfg = json.loads( 45 | worker.send_task("tests.tasks.general.GetConfig", {}, block=True)) 46 | 47 | assert cfg["name"] == "yyy" 48 | assert cfg.get("additional_unexpected_config") == "1" 49 | 50 | worker.stop() 51 | 52 | 53 | # Command line flags with default values should be able to overwrite file config 54 | # Here we have LOG_LEVEL="INFO" in the file, and DEBUG is default 55 | worker.start(flags="--config tests/fixtures/config2.py") 56 | 57 | cfg = json.loads( 58 | worker.send_task("tests.tasks.general.GetConfig", {}, block=True)) 59 | 60 | assert cfg["log_level"] == "INFO" 61 | 62 | worker.stop() 63 | 64 | worker.start(flags="--config tests/fixtures/config2.py --log_level DEBUG") 65 | 66 | cfg = json.loads( 67 | worker.send_task("tests.tasks.general.GetConfig", {}, block=True)) 68 | 69 | assert cfg["log_level"] == "DEBUG" 70 | 71 | worker.stop() 72 | -------------------------------------------------------------------------------- /tests/test_context.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from collections import defaultdict 4 | import subprocess 5 | 6 | 7 | # Read from tests.tasks.general.GetMetrics 8 | TEST_LOCAL_METRICS = defaultdict(int) 9 | 10 | 11 | def METRIC_HOOK(name, incr=1, **kwargs): 12 | TEST_LOCAL_METRICS[name] += incr 13 | 14 | 15 | def _reset_local_metrics(): 16 | for k in list(TEST_LOCAL_METRICS.keys()): 17 | TEST_LOCAL_METRICS.pop(k) 18 | 19 | 20 | def test_context_get(worker): 21 | 22 | result = worker.send_task("tests.tasks.context.GetContext", {}) 23 | 24 | assert result["job_id"] 25 | assert result["worker_id"] 26 | assert result["config"]["redis"] 27 | 28 | 29 | def test_context_connections_redis(worker): 30 | 31 | worker.start(flags=" --config tests/fixtures/config-multiredis.py") 32 | 33 | assert worker.send_task("tests.tasks.redis.MultiRedis", {}) == "ok" 34 | 35 | 36 | def test_context_metric_success(worker): 37 | from mrq.context import get_current_config 38 | 39 | local_config = get_current_config() 40 | local_config["metric_hook"] = METRIC_HOOK 41 | _reset_local_metrics() 42 | 43 | worker.start(flags=" --config tests/fixtures/config-metric.py") 44 | 45 | result = worker.send_task("tests.tasks.general.Add", {"a": 41, "b": 1}) 46 | result = worker.send_task("tests.tasks.general.Add", {"a": 41, "b": 1}) 47 | 48 | assert result == 42 49 | 50 | metrics = json.loads( 51 | worker.send_task("tests.tasks.general.GetMetrics", {})) 52 | 53 | # GetMetrics is also a task! 54 | assert metrics.get("queues.default.dequeued") == 3 55 | assert metrics.get("queues.all.dequeued") == 3 56 | 57 | TEST_LOCAL_METRICS.get("jobs.status.queued") == 3 58 | assert metrics.get("jobs.status.started") == 3 59 | assert metrics.get("jobs.status.success") == 2 # At the time it is run, GetMetrics isn't success yet. 60 | 61 | local_config["metric_hook"] = None 62 | 63 | 64 | def test_context_metric_queue(worker): 65 | from mrq.context import get_current_config 66 | 67 | local_config = get_current_config() 68 | local_config["metric_hook"] = METRIC_HOOK 69 | _reset_local_metrics() 70 | 71 | worker.start(flags=" --config tests/fixtures/config-metric.py") 72 | 73 | # Will send 1 task inside! 74 | worker.send_task("tests.tasks.general.SendTask", { 75 | "path": "tests.tasks.general.Add", "params": {"a": 41, "b": 1}}) 76 | 77 | metrics = json.loads( 78 | worker.send_task("tests.tasks.general.GetMetrics", {})) 79 | 80 | # GetMetrics is also a task! 81 | assert metrics.get("queues.default.dequeued") == 3 82 | assert metrics.get("queues.all.dequeued") == 3 83 | assert metrics.get("jobs.status.started") == 3 84 | assert metrics.get("jobs.status.success") == 2 # At the time it is run, GetMetrics isn't success yet. 85 | 86 | TEST_LOCAL_METRICS.get("queues.default.enqueued") == 2 87 | TEST_LOCAL_METRICS.get("queues.all.enqueued") == 2 88 | 89 | local_config["metric_hook"] = None 90 | 91 | 92 | def test_context_metric_failed(worker): 93 | 94 | worker.start(flags=" --config tests/fixtures/config-metric.py") 95 | 96 | worker.send_task( 97 | "tests.tasks.general.RaiseException", {}, accept_statuses=["failed"]) 98 | 99 | metrics = json.loads( 100 | worker.send_task("tests.tasks.general.GetMetrics", {})) 101 | 102 | # GetMetrics is also a task! 103 | assert metrics.get("queues.default.dequeued") == 2 104 | assert metrics.get("queues.all.dequeued") == 2 105 | assert metrics.get("jobs.status.started") == 2 106 | assert metrics.get("jobs.status.failed") == 1 107 | assert metrics.get("jobs.status.success") is None 108 | 109 | 110 | def test_context_setup(): 111 | 112 | process = subprocess.Popen("python tests/fixtures/standalone_script1.py", 113 | shell=True, close_fds=True, env={"MRQ_NAME": "testname1", "PYTHONPATH": os.getcwd()}, cwd=os.getcwd(), stdout=subprocess.PIPE) 114 | 115 | out, err = process.communicate() 116 | 117 | assert out.endswith(b"42\ntestname1\n") 118 | -------------------------------------------------------------------------------- /tests/test_disconnects.py: -------------------------------------------------------------------------------- 1 | from builtins import range 2 | import time 3 | from mrq.job import Job 4 | import pytest 5 | 6 | 7 | @pytest.mark.parametrize(["p_service"], [["mongodb"], ["redis"]]) 8 | def test_disconnects_service_during_task(worker, p_service): 9 | """ Test what happens when mongodb disconnects during a job 10 | """ 11 | 12 | worker.start() 13 | 14 | if p_service == "mongodb": 15 | service = worker.fixture_mongodb 16 | elif p_service == "redis": 17 | service = worker.fixture_redis 18 | 19 | service_pid = service.process.pid 20 | 21 | job_id1 = worker.send_task("tests.tasks.general.Add", { 22 | "a": 41, "b": 1, "sleep": 5}, block=False, queue="default") 23 | 24 | time.sleep(2) 25 | 26 | service.stop() 27 | service.start() 28 | 29 | service_pid2 = service.process.pid 30 | 31 | # Make sure we did restart 32 | assert service_pid != service_pid2 33 | 34 | time.sleep(5) 35 | 36 | # Result should be there without issues 37 | assert Job(job_id1).fetch().data["result"] == 42 38 | 39 | 40 | @pytest.mark.parametrize(["gevent_count", "subpool_size", "iterations", "expected_clients"], [ 41 | (None, None, 1, 1), # single task opens a single connection 42 | (None, None, 2, 1), 43 | (None, 10, 1, 10), # single task with subpool of 10 opens 10 connections 44 | (None, 10, 2, 10), 45 | (None, 200, 1, 100), # single task with subpool of 200 opens 100 connections, we reach the max_connections limit 46 | (None, 200, 2, 100), 47 | (4, None, 1, 4), # 4 gevent workers with a single task each : 4 connections 48 | (4, None, 2, 4), 49 | (2, 2, 1, 4), # 2 gevent workers with 2 single tasks each : 4 connections 50 | (2, 2, 2, 4), 51 | 52 | ]) 53 | def test_redis_disconnections(gevent_count, subpool_size, iterations, expected_clients, worker): 54 | """ mrq.context.connections is not the actual connections pool that the worker uses. 55 | this worker's pool is not accessible from here, since it runs in a different thread. 56 | """ 57 | from mrq.context import connections 58 | 59 | worker.start_deps() 60 | 61 | gevent_count = gevent_count if gevent_count is not None else 1 62 | 63 | def get_clients(): 64 | lst = connections.redis.client_list() 65 | return [c for c in lst if c.get("cmd") != "client"] 66 | 67 | assert len(get_clients()) == 0 68 | 69 | # 1. start the worker and asserts that there is a redis client connected 70 | kwargs = {"flags": "--redis_max_connections 100", "deps": False} 71 | if gevent_count: 72 | kwargs["flags"] += " --greenlets %s" % gevent_count 73 | 74 | worker.start(**kwargs) 75 | 76 | for i in range(0, iterations): 77 | # sending tasks has the good side effect to wait for the worker to connect to redis 78 | worker.send_tasks("tests.tasks.redis.Disconnections", [{"subpool_size": subpool_size}] * gevent_count) 79 | 80 | clients = get_clients() 81 | 82 | cmd_get_clients = [x for x in clients if x.get("cmd") == "get"] 83 | 84 | # These can be 2 background greenlets doing redis ops (known queues, paused queues) 85 | assert len(cmd_get_clients) <= expected_clients + 2 86 | 87 | # 2. kill the worker and make sure that the connection was closed 88 | worker.stop(deps=False) # so that we still have access to redis 89 | 90 | assert len(get_clients()) == 0 91 | 92 | worker.stop_deps() 93 | -------------------------------------------------------------------------------- /tests/test_io_hooks.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import json 3 | 4 | 5 | def test_io_hooks_nothing(worker): 6 | 7 | worker.start(flags=" --config tests/fixtures/config-io-hooks.py") 8 | 9 | assert worker.send_task( 10 | "tests.tasks.general.Add", {"a": 41, "b": 1}) == 42 11 | 12 | events = json.loads( 13 | worker.send_task("tests.tasks.general.GetIoHookEvents", {})) 14 | 15 | job_events = [x for x in events if x.get("job")] 16 | 17 | for evt in job_events: 18 | print(evt) 19 | 20 | # Only update should be the result in mongodb. 21 | assert len(job_events) == 1 * 2 22 | 23 | assert job_events[0]["hook"] == "mongodb_pre" 24 | assert job_events[1]["hook"] == "mongodb_post" 25 | 26 | assert job_events[0]["method"] == "update" 27 | assert job_events[1]["method"] == "update" 28 | 29 | assert job_events[0]["collection"] == "mrq.mrq_jobs" 30 | assert job_events[1]["collection"] == "mrq.mrq_jobs" 31 | 32 | 33 | def test_io_hooks_redis(worker): 34 | 35 | worker.start(flags=" --config tests/fixtures/config-io-hooks.py") 36 | 37 | worker.send_task( 38 | "tests.tasks.io.TestIo", 39 | {"test": "redis-llen", "params": {"key": "xyz"}} 40 | ) 41 | 42 | events = json.loads( 43 | worker.send_task("tests.tasks.general.GetIoHookEvents", {})) 44 | 45 | job_events = [x for x in events if x.get("job")] 46 | 47 | for evt in job_events: 48 | print(evt) 49 | 50 | assert len(job_events) == 2 * 2 51 | 52 | assert job_events[0]["hook"] == "redis_pre" 53 | assert job_events[1]["hook"] == "redis_post" 54 | 55 | assert job_events[0]["key"] == "xyz" 56 | assert job_events[1]["key"] == "xyz" 57 | 58 | assert job_events[0]["command"] == "LLEN" 59 | assert job_events[1]["command"] == "LLEN" 60 | 61 | # Regular MongoDB update 62 | 63 | assert job_events[2]["hook"] == "mongodb_pre" 64 | assert job_events[3]["hook"] == "mongodb_post" 65 | 66 | assert job_events[2]["method"] == "update" 67 | assert job_events[3]["method"] == "update" 68 | 69 | assert job_events[2]["collection"] == "mrq.mrq_jobs" 70 | assert job_events[3]["collection"] == "mrq.mrq_jobs" 71 | 72 | 73 | def test_io_hooks_mongodb(worker): 74 | 75 | worker.start(flags=" --config tests/fixtures/config-io-hooks.py") 76 | 77 | ret = worker.send_task( 78 | "tests.tasks.io.TestIo", 79 | {"test": "mongodb-full-getmore"} 80 | ) 81 | 82 | print(ret) 83 | 84 | events = json.loads( 85 | worker.send_task("tests.tasks.general.GetIoHookEvents", {})) 86 | 87 | job_events = [x for x in events if x.get("job")] 88 | 89 | for evt in job_events: 90 | print(evt) 91 | 92 | # First, insert 93 | assert job_events[0]["hook"] == "mongodb_pre" 94 | assert job_events[1]["hook"] == "mongodb_post" 95 | 96 | assert job_events[0]["collection"] == "mrq.tests_inserts" 97 | assert job_events[1]["collection"] == "mrq.tests_inserts" 98 | 99 | assert job_events[0]["method"] == "insert_many" 100 | assert job_events[1]["method"] == "insert_many" 101 | 102 | # Then first query 103 | assert job_events[2]["hook"] == "mongodb_pre" 104 | assert job_events[3]["hook"] == "mongodb_post" 105 | 106 | assert job_events[2]["collection"] == "mrq.tests_inserts" 107 | assert job_events[3]["collection"] == "mrq.tests_inserts" 108 | 109 | assert job_events[2]["method"] == "find" 110 | assert job_events[3]["method"] == "find" 111 | 112 | # Then getmore query 113 | assert job_events[4]["hook"] == "mongodb_pre" 114 | assert job_events[5]["hook"] == "mongodb_post" 115 | 116 | assert job_events[4]["collection"] == "mrq.tests_inserts" 117 | assert job_events[5]["collection"] == "mrq.tests_inserts" 118 | 119 | assert job_events[4]["method"] == "cursor" 120 | assert job_events[5]["method"] == "cursor" 121 | 122 | # Then getmore query (can't understand why there are 2 more of those) 123 | assert job_events[6]["hook"] == "mongodb_pre" 124 | assert job_events[7]["hook"] == "mongodb_post" 125 | 126 | assert job_events[6]["collection"] == "mrq.tests_inserts" 127 | assert job_events[7]["collection"] == "mrq.tests_inserts" 128 | 129 | assert job_events[6]["method"] == "cursor" 130 | assert job_events[7]["method"] == "cursor" 131 | 132 | # Then getmore query 133 | assert job_events[8]["hook"] == "mongodb_pre" 134 | assert job_events[9]["hook"] == "mongodb_post" 135 | 136 | assert job_events[8]["collection"] == "mrq.tests_inserts" 137 | assert job_events[9]["collection"] == "mrq.tests_inserts" 138 | 139 | assert job_events[8]["method"] == "cursor" 140 | assert job_events[9]["method"] == "cursor" 141 | 142 | # Result MongoDB update 143 | 144 | assert job_events[10]["hook"] == "mongodb_pre" 145 | assert job_events[11]["hook"] == "mongodb_post" 146 | 147 | assert job_events[10]["method"] == "update" 148 | assert job_events[11]["method"] == "update" 149 | 150 | assert job_events[10]["collection"] == "mrq.mrq_jobs" 151 | assert job_events[11]["collection"] == "mrq.mrq_jobs" 152 | 153 | assert len(job_events) == 6 * 2 154 | -------------------------------------------------------------------------------- /tests/test_logger.py: -------------------------------------------------------------------------------- 1 | import time 2 | import pytest 3 | import os 4 | 5 | OPTS = [] 6 | for cls in ["string", "unicode", "latin-1", "bytes1"]: 7 | for utf8_sys_stdout in [True, False]: 8 | OPTS.append([cls, utf8_sys_stdout]) 9 | 10 | 11 | @pytest.mark.parametrize(["class_name", "utf8_sys_stdout"], OPTS) 12 | def test_supports_string_and_unicode(worker, class_name, utf8_sys_stdout): 13 | result = worker.send_task("tests.tasks.logger.Simple", { 14 | "class_name": class_name, 15 | "utf8_sys_stdout": utf8_sys_stdout 16 | }) 17 | 18 | # Force-flush the logs 19 | worker.stop(deps=False) 20 | assert result 21 | 22 | time.sleep(0.1) 23 | 24 | # Job logs 25 | db_logs = list(worker.mongodb_logs.mrq_logs.find({"job": {"$exists": True}})) 26 | 27 | assert len(db_logs) == 1 28 | if class_name == "unicode": 29 | assert u"caf\xe9" in db_logs[0]["logs"] 30 | elif class_name == "string": 31 | assert u"cafe" in db_logs[0]["logs"] 32 | elif class_name == "latin-1": 33 | assert "caf" in db_logs[0]["logs"] 34 | assert u"cafe" not in db_logs[0]["logs"] 35 | assert u"caf\xe9" not in db_logs[0]["logs"] 36 | 37 | # Worker logs 38 | # db_logs = list(worker.mongodb_logs.mrq_logs.find({"worker": db_workers[0]["_id"]})) 39 | # assert len(db_logs) >= 1 40 | # if class_name == "unicode": 41 | # assert u"caf\xe9" in db_logs 42 | # else: 43 | # assert u"cafe" in db_logs 44 | 45 | worker.stop_deps() 46 | 47 | def test_other_handlers(worker): 48 | worker.start(flags="--config tests/fixtures/config-logger.py") 49 | worker.send_task("tests.tasks.logger.Simple", { 50 | "class_name": "string" 51 | }) 52 | worker.stop(deps=False) 53 | assert os.path.isfile("/tmp/mrq.log") 54 | worker.stop_deps() 55 | os.unlink("/tmp/mrq.log") 56 | 57 | def test_log_level(worker): 58 | worker.start(flags="--log_level INFO --config tests/fixtures/config-logger.py") 59 | worker.send_task("tests.tasks.logger.Simple", { 60 | "class_name": "string" 61 | }, block=True) 62 | worker.stop(deps=False) 63 | assert os.path.isfile("/tmp/mrq.log") 64 | with open("/tmp/mrq.log") as f: 65 | lines = f.readlines() 66 | assert all(["DEBUG" not in line for line in lines]) 67 | worker.stop_deps() 68 | os.unlink("/tmp/mrq.log") 69 | 70 | 71 | def test_collection_is_capped(worker): 72 | result = worker.send_task("tests.tasks.logger.Simple", { 73 | "class_name": "string" 74 | }) 75 | 76 | # Force-flush the logs 77 | worker.stop(deps=False) 78 | assert worker.mongodb_logs.mrq_logs.options()["capped"] is True 79 | 80 | worker.stop_deps() 81 | -------------------------------------------------------------------------------- /tests/test_memoryleaks.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from builtins import range 3 | import time 4 | import pytest 5 | import os 6 | 7 | 8 | def test_max_memory_restart(worker): 9 | 10 | N = 10 11 | 12 | worker.start( 13 | flags=" --processes 1 --greenlets 1 --max_memory 50 --report_interval 1") 14 | 15 | worker.send_tasks( 16 | "tests.tasks.general.Leak", 17 | [{"size": 1000000, "sleep": 1} for _ in range(N)], 18 | queue="default", 19 | block=False 20 | ) 21 | 22 | i = 0 23 | while worker.mongodb_jobs.mrq_jobs.find({"status": "success"}).count() != N: 24 | time.sleep(1) 25 | i += 1 26 | if i % 5 == 0: 27 | os.system("ps -ef") 28 | 29 | # We must have been restarted at least once. 30 | assert worker.mongodb_jobs.mrq_workers.find().count() > 1 31 | 32 | 33 | def get_diff_after_jobs(worker, n_tasks, leak, sleep=0): 34 | 35 | time.sleep(3) 36 | 37 | mem_start = worker.get_report(with_memory=True)["process"]["mem"]["total"] 38 | 39 | worker.send_tasks( 40 | "tests.tasks.general.Leak", 41 | [{"size": leak, "sleep": sleep} for _ in range(n_tasks)], 42 | queue="default", 43 | block=True 44 | ) 45 | 46 | time.sleep(3) 47 | 48 | mem_stop = worker.get_report(with_memory=True)["process"]["mem"]["total"] 49 | 50 | diff = mem_stop - mem_start 51 | 52 | print("Memory diff for %s tasks was %s" % (n_tasks, diff)) 53 | 54 | return diff 55 | 56 | 57 | def test_memoryleaks_noleak(worker): 58 | 59 | return pytest.skip("Too flaky, investigate sources of noise") 60 | 61 | TRACE = "" 62 | # TRACE = "--trace_memory_type ObjectId" 63 | 64 | worker.start( 65 | flags="--trace_memory --greenlets 1 --mongodb_logs 0 --scheduler_interval 0 --subqueues_refresh_interval 0 --paused_queues_refresh_interval 0 --report_interval 10000 %s" % TRACE) 66 | 67 | # Send it once to add to imports 68 | get_diff_after_jobs(worker, 10, 0) 69 | 70 | diff100 = get_diff_after_jobs(worker, 100, 0) 71 | 72 | assert worker.mongodb_jobs.mrq_jobs.count() == 100 + 10 73 | 74 | diff200 = get_diff_after_jobs(worker, 200, 0) 75 | 76 | assert worker.mongodb_jobs.mrq_jobs.count() == 200 + 100 + 10 77 | 78 | # Most of the tasks should have mem_diff == 0 79 | assert worker.mongodb_jobs.mrq_jobs.find( 80 | {"memory_diff": 0}).count() > 310 * 0.95 81 | 82 | assert diff100 < 15000 83 | assert diff200 < 15000 84 | 85 | assert worker.mongodb_jobs.mrq_workers.find().count() == 1 86 | 87 | 88 | def test_memoryleaks_1mleak(worker): 89 | 90 | worker.start( 91 | flags="--trace_memory --greenlets 1 --mongodb_logs 0 --report_interval 10000") 92 | 93 | # Send it once to add to imports 94 | get_diff_after_jobs(worker, 10, 0) 95 | 96 | worker.mongodb_jobs.mrq_jobs.remove() 97 | 98 | # 1M leak! 99 | # sleep is needed so that psutil measurements are accurate :-/ 100 | diff1m = get_diff_after_jobs(worker, 10, 100000, sleep=0) 101 | 102 | assert diff1m > 900000 103 | 104 | assert worker.mongodb_jobs.mrq_jobs.find( 105 | {"memory_diff": {"$gte": 80000}}).count() == 10 106 | 107 | assert worker.mongodb_jobs.mrq_workers.find().count() == 1 108 | -------------------------------------------------------------------------------- /tests/test_notify.py: -------------------------------------------------------------------------------- 1 | from builtins import range 2 | import time 3 | import pytest 4 | from mrq.context import connections 5 | from mrq.job import Job 6 | 7 | 8 | def test_queue_notify(worker, worker2): 9 | 10 | worker.start(flags="--max_latency 30 --config tests/fixtures/config-notify.py", queues="withnotify withoutnotify") 11 | 12 | # Used to queue jobs in the same environment & config! 13 | worker2.start(flags="--config tests/fixtures/config-notify.py") 14 | 15 | id1 = worker2.send_task("tests.tasks.general.SendTask", { 16 | "params": {"a": 42, "b": 1}, 17 | "path": "tests.tasks.general.Add", 18 | "queue": "withnotify" 19 | }) 20 | 21 | time.sleep(2) 22 | 23 | assert Job(id1).fetch().data["status"] == "success" 24 | assert Job(id1).fetch().data["result"] == 43 25 | 26 | id2 = worker2.send_task("tests.tasks.general.SendTask", { 27 | "params": {"a": 43, "b": 1}, 28 | "path": "tests.tasks.general.Add", 29 | "queue": "withoutnotify" 30 | }) 31 | 32 | time.sleep(2) 33 | 34 | assert Job(id2).fetch().data["status"] == "queued" 35 | -------------------------------------------------------------------------------- /tests/test_parallel.py: -------------------------------------------------------------------------------- 1 | from builtins import range 2 | import time 3 | import pytest 4 | from mrq.context import connections 5 | 6 | 7 | @pytest.mark.parametrize(["p_flags"], [ 8 | ["--greenlets 50"], 9 | ["--processes 10 --greenlets 5"] 10 | ]) 11 | def test_parallel_100sleeps(worker, p_flags): 12 | 13 | worker.start(flags=p_flags) 14 | 15 | print("Worker started. Queueing sleeps") 16 | 17 | start_time = time.time() 18 | 19 | # This will sleep a total of 100 seconds 20 | result = worker.send_tasks( 21 | "tests.tasks.general.Add", [{"a": i, "b": 0, "sleep": 1} for i in range(100)]) 22 | 23 | total_time = time.time() - start_time 24 | 25 | print("Total time for 100 parallel sleeps: %s" % total_time) 26 | 27 | # But should be done quickly! 28 | assert total_time < 15 29 | 30 | # ... and return correct results 31 | assert result == list(range(100)) 32 | 33 | 34 | @pytest.mark.parametrize(["p_greenlets", "p_strategy"], [ 35 | [g, s] 36 | for g in [1, 2] 37 | for s in ["", "parallel", "burst"] 38 | ]) 39 | def test_dequeue_strategy(worker, p_greenlets, p_strategy): 40 | 41 | worker.start_deps(flush=True) 42 | 43 | worker.send_task( 44 | "tests.tasks.general.MongoInsert", {"a": 41, "sleep": 1}, queue="q1", block=False, start=False) 45 | worker.send_task( 46 | "tests.tasks.general.MongoInsert", {"a": 42, "sleep": 1}, queue="q2", block=False, start=False) 47 | worker.send_task( 48 | "tests.tasks.general.MongoInsert", {"a": 43, "sleep": 1}, queue="q1", block=False, start=False) 49 | worker.send_task( 50 | "tests.tasks.general.MongoInsert", {"a": 44, "sleep": 1}, queue="q2", block=False, start=False) 51 | worker.send_task( 52 | "tests.tasks.general.MongoInsert", {"a": 45, "sleep": 1}, queue="q3", block=False, start=False) 53 | 54 | time.sleep(0.5) 55 | 56 | flags = "--greenlets %s" % p_greenlets 57 | if p_strategy: 58 | flags += " --dequeue_strategy %s" % p_strategy 59 | 60 | print("Worker has flags %s" % flags) 61 | worker.start(flags=flags, queues="q1 q2", deps=False, block=False) 62 | 63 | gotit = worker.wait_for_idle(timeout=10) 64 | 65 | if p_strategy == "burst": 66 | assert not gotit # because worker should be stopped already 67 | else: 68 | assert gotit 69 | 70 | inserts = list(connections.mongodb_jobs.tests_inserts.find(sort=[("_id", 1)])) 71 | order = [row["params"]["a"] for row in inserts] 72 | 73 | if p_strategy == "parallel": 74 | assert set(order[0:2]) == set([41, 42]) 75 | assert set(order[2:4]) == set([43, 44]) 76 | else: 77 | assert set(order[0:2]) == set([41, 43]) 78 | assert set(order[2:4]) == set([42, 44]) 79 | -------------------------------------------------------------------------------- /tests/test_pause.py: -------------------------------------------------------------------------------- 1 | from mrq.job import Job 2 | import pytest 3 | from mrq.queue import Queue, send_task 4 | import time 5 | from mrq.context import set_current_config, get_config 6 | 7 | 8 | def test_pause_resume(worker): 9 | 10 | worker.start(flags="--paused_queues_refresh_interval=0.1") 11 | 12 | Queue("high").pause() 13 | 14 | assert Queue("high").is_paused() 15 | 16 | # wait for the paused_queues list to be refreshed 17 | time.sleep(2) 18 | 19 | job_id1 = send_task( 20 | "tests.tasks.general.MongoInsert", {"a": 41}, 21 | queue="high") 22 | 23 | job_id2 = send_task( 24 | "tests.tasks.general.MongoInsert", {"a": 43}, 25 | queue="low") 26 | 27 | time.sleep(5) 28 | 29 | job1 = Job(job_id1).fetch().data 30 | job2 = Job(job_id2).fetch().data 31 | 32 | assert job1["status"] == "queued" 33 | 34 | assert job2["status"] == "success" 35 | assert job2["result"] == {"a": 43} 36 | 37 | assert worker.mongodb_jobs.tests_inserts.count() == 1 38 | 39 | Queue("high").resume() 40 | 41 | Job(job_id1).wait(poll_interval=0.01) 42 | 43 | job1 = Job(job_id1).fetch().data 44 | 45 | assert job1["status"] == "success" 46 | assert job1["result"] == {"a": 41} 47 | 48 | assert worker.mongodb_jobs.tests_inserts.count() == 2 49 | 50 | 51 | def test_pause_refresh_interval(worker): 52 | 53 | """ Tests that a refresh interval of 0 disables the pause functionnality """ 54 | 55 | worker.start(flags="--paused_queues_refresh_interval=0") 56 | 57 | Queue("high").pause() 58 | 59 | assert Queue("high").is_paused() 60 | 61 | # wait for the paused_queues list to be refreshed 62 | time.sleep(2) 63 | 64 | job_id1 = send_task( 65 | "tests.tasks.general.MongoInsert", {"a": 41}, 66 | queue="high") 67 | 68 | time.sleep(5) 69 | 70 | job1 = Job(job_id1).fetch().data 71 | 72 | assert job1["status"] == "success" 73 | assert job1["result"] == {"a": 41} 74 | 75 | 76 | def test_pause_subqueue(worker): 77 | 78 | # set config in current context in order to have a subqueue delimiter 79 | set_current_config(get_config(config_type="worker")) 80 | 81 | worker.start(queues="high high/", flags="--subqueues_refresh_interval=1 --paused_queues_refresh_interval=1") 82 | 83 | Queue("high").pause() 84 | 85 | assert Queue("high/").is_paused() 86 | 87 | # wait for the paused_queues list to be refreshed 88 | time.sleep(2) 89 | 90 | job_id1 = send_task( 91 | "tests.tasks.general.MongoInsert", {"a": 41}, 92 | queue="high") 93 | 94 | job_id2 = send_task( 95 | "tests.tasks.general.MongoInsert", {"a": 43}, 96 | queue="high/subqueue") 97 | 98 | # wait a bit to make sure the jobs status will still be queued 99 | time.sleep(5) 100 | 101 | job1 = Job(job_id1).fetch().data 102 | job2 = Job(job_id2).fetch().data 103 | 104 | assert job1["status"] == "queued" 105 | assert job2["status"] == "queued" 106 | 107 | assert worker.mongodb_jobs.tests_inserts.count() == 0 108 | 109 | Queue("high/").resume() 110 | 111 | Job(job_id1).wait(poll_interval=0.01) 112 | 113 | Job(job_id2).wait(poll_interval=0.01) 114 | 115 | job1 = Job(job_id1).fetch().data 116 | job2 = Job(job_id2).fetch().data 117 | 118 | assert job1["status"] == "success" 119 | assert job1["result"] == {"a": 41} 120 | 121 | assert job2["status"] == "success" 122 | assert job2["result"] == {"a": 43} 123 | 124 | assert worker.mongodb_jobs.tests_inserts.count() == 2 125 | -------------------------------------------------------------------------------- /tests/test_processes.py: -------------------------------------------------------------------------------- 1 | from mrq.processes import ProcessPool 2 | import time 3 | 4 | 5 | def test_processpool_sleeps(): 6 | 7 | pool = ProcessPool(watch_interval=1) 8 | 9 | pool.start() 10 | 11 | pool.set_commands(["/bin/sleep 1", "/bin/sleep 10"]) 12 | 13 | time.sleep(2) 14 | 15 | pool.watch_processes() 16 | pids1 = {p["command"]: p["pid"] for p in pool.processes} 17 | 18 | time.sleep(2) 19 | 20 | pool.watch_processes() 21 | assert len(pool.processes) == 2 22 | 23 | pids2 = {p["command"]: p["pid"] for p in pool.processes} 24 | 25 | assert pids1["/bin/sleep 1"] != pids2["/bin/sleep 1"] 26 | assert pids1["/bin/sleep 10"] == pids2["/bin/sleep 10"] 27 | 28 | pool.set_commands(["/bin/sleep 2", "/bin/sleep 10"]) 29 | 30 | time.sleep(3) 31 | 32 | pool.watch_processes() 33 | assert len(pool.processes) == 2 34 | 35 | pids3 = {p["command"]: p["pid"] for p in pool.processes} 36 | assert pids3["/bin/sleep 2"] != pids2["/bin/sleep 1"] 37 | assert pids1["/bin/sleep 10"] == pids3["/bin/sleep 10"] 38 | -------------------------------------------------------------------------------- /tests/test_progress.py: -------------------------------------------------------------------------------- 1 | import time 2 | import pytest 3 | 4 | 5 | @pytest.mark.parametrize(["p_save"], [ 6 | [True], 7 | [False] 8 | ]) 9 | def test_progress(worker, p_save): 10 | 11 | worker.start(flags="--report_interval 1") 12 | 13 | assert worker.send_task( 14 | "tests.tasks.general.Progress", {"save": p_save}, block=False) 15 | 16 | time.sleep(5) 17 | 18 | assert worker.mongodb_jobs.mrq_jobs.find()[0]["progress"] > 0.2 19 | assert worker.mongodb_jobs.mrq_jobs.find()[0]["progress"] < 0.6 20 | 21 | time.sleep(7) 22 | 23 | assert worker.mongodb_jobs.mrq_jobs.find()[0]["progress"] == 1 24 | -------------------------------------------------------------------------------- /tests/test_queuesize.py: -------------------------------------------------------------------------------- 1 | 2 | def test_job_queue(worker): 3 | from mrq.context import connections 4 | worker.start() 5 | worker.send_task("tests.tasks.general.Wait", {}, block=False) 6 | assert int(connections.redis.get("queuesize:%s" % "default")) == 1 7 | worker.wait_for_idle() 8 | assert int(connections.redis.get("queuesize:%s" % "default")) == 0 9 | 10 | def test_job_failed(worker): 11 | from mrq.context import connections 12 | worker.start() 13 | worker.send_task("tests.tasks.general.RaiseException", {}, block=False) 14 | worker.wait_for_idle() 15 | assert int(connections.redis.get("queuesize:%s" % "default")) == 0 16 | 17 | def test_job_requeue(worker): 18 | from mrq.context import connections 19 | from mrq.job import Job 20 | 21 | worker.start() 22 | job_id = worker.send_task("tests.tasks.general.RaiseException", {}, block=False) 23 | worker.wait_for_idle() 24 | assert int(connections.redis.get("queuesize:%s" % "default")) == 0 25 | 26 | Job(job_id).requeue() 27 | assert int(connections.redis.get("queuesize:%s" % "default")) == 1 28 | -------------------------------------------------------------------------------- /tests/test_ratelimit.py: -------------------------------------------------------------------------------- 1 | from builtins import range 2 | from mrq.helpers import ratelimit 3 | import time 4 | 5 | 6 | def test_helpers_ratelimit(worker): 7 | 8 | worker.start_deps() 9 | 10 | assert ratelimit("k3", 1, per=1) == 1 11 | assert ratelimit("k3", 1, per=1) == 0 12 | assert ratelimit("k3", 1, per=1) == 0 13 | 14 | for i in range(0, 10): 15 | r = ratelimit("k", 10, per=1) 16 | assert r == 10 - i 17 | 18 | assert ratelimit("k", 10, per=1) == 0 19 | assert ratelimit("k2", 5, per=1) == 5 20 | 21 | # We *could* have failures there if we go over a second but we've not seen 22 | # it much so far. 23 | for i in range(0, 100): 24 | assert ratelimit("k", 10, per=1) == 0 25 | 26 | # TODO: test the "per" argument a bit better. 27 | time.sleep(1) 28 | 29 | assert ratelimit("k", 10, per=1) == 10 30 | assert ratelimit("k", 10, per=1) == 9 31 | 32 | # This is actually another counter. 33 | assert ratelimit("k", 10, per=10) == 10 34 | assert ratelimit("k", 10, per=10) == 9 35 | 36 | worker.stop_deps() 37 | -------------------------------------------------------------------------------- /tests/test_routes.py: -------------------------------------------------------------------------------- 1 | from builtins import str 2 | from builtins import bool 3 | import json 4 | import pytest 5 | 6 | 7 | def test_routes_taskexceptions(worker, api): 8 | 9 | task_path = "tests.tasks.general.RaiseException" 10 | worker.send_task(task_path, { 11 | "message": "xyz"}, block=True, accept_statuses=["failed"]) 12 | 13 | data, _ = api.GET("/api/datatables/taskexceptions?sEcho=1") 14 | assert len(data["aaData"]) == 1 15 | assert data["aaData"][0]["_id"]["path"] == task_path 16 | 17 | 18 | def test_routes_status(worker, api): 19 | 20 | worker.send_task("tests.tasks.general.RaiseException", { 21 | "message": "xyz"}, block=True, accept_statuses=["failed"]) 22 | 23 | data, _ = api.GET("/api/datatables/status?sEcho=1") 24 | assert len(data["aaData"]) == 1 25 | 26 | 27 | def test_routes_taskpaths(worker, api): 28 | 29 | task_path = "tests.tasks.general.RaiseException" 30 | worker.send_task(task_path, { 31 | "message": "xyz"}, block=True, accept_statuses=["failed"]) 32 | 33 | data, _ = api.GET("/api/datatables/taskpaths?sEcho=1") 34 | 35 | assert len(data["aaData"]) == 1 36 | assert data["aaData"][0]["_id"] == task_path 37 | assert data["aaData"][0]["jobs"] == 1 38 | 39 | 40 | def test_routes_workers(worker, api): 41 | 42 | worker.start() 43 | 44 | data, _ = api.GET("/workers") 45 | assert len(data) == 1 46 | 47 | 48 | def test_routes_traceback(worker, api): 49 | 50 | worker.send_task("tests.tasks.general.RaiseException", { 51 | "message": "xyz"}, block=True, accept_statuses=["failed"]) 52 | 53 | job = worker.mongodb_jobs.mrq_jobs.find_one() 54 | assert job 55 | 56 | data, _ = api.GET("/api/job/%s/traceback" % job["_id"]) 57 | assert "xyz" in data["traceback"] 58 | 59 | 60 | def test_routes_result(worker, api): 61 | 62 | worker.send_task("tests.tasks.general.ReturnParams", { 63 | "message": "xyz"}, block=True) 64 | 65 | job = worker.mongodb_jobs.mrq_jobs.find_one() 66 | assert job 67 | 68 | data, _ = api.GET("/api/job/%s/result" % job["_id"]) 69 | assert data["result"]["message"] == "xyz" 70 | 71 | 72 | def test_routes_jobaction(worker, api): 73 | 74 | worker.send_task("tests.tasks.general.ReturnParams", { 75 | "message": "xyz"}, block=False, queue="tmp") 76 | 77 | job = worker.mongodb_jobs.mrq_jobs.find_one() 78 | assert job 79 | 80 | params = { 81 | "action": "cancel", 82 | "id": str(job["_id"]) 83 | } 84 | data, _ = api.POST("/api/jobaction", data=json.dumps(params)) 85 | 86 | assert "job_id" in data 87 | assert bool(data["job_id"]) 88 | 89 | 90 | def test_routes_datatables(worker, api): 91 | 92 | worker.start(flags="--config tests/fixtures/config-raw1.py") 93 | 94 | task_path = "tests.tasks.general.RaiseException" 95 | worker.send_task(task_path, { 96 | "message": "xyz"}, block=True, accept_statuses=["failed"], queue="low") 97 | worker.send_task(task_path, { 98 | "message": "xyz"}, block=True, accept_statuses=["failed"], queue="high") 99 | 100 | unit = "workers" 101 | data, _ = api.GET("/api/datatables/%s?sEcho=1" % unit) 102 | assert len(data["aaData"]) == 1 103 | 104 | unit = "jobs" 105 | data, _ = api.GET("/api/datatables/%s?sEcho=1" % unit) 106 | assert len(data["aaData"]) == 2 107 | assert data["aaData"][0]["path"] == task_path 108 | assert data["aaData"][0]["status"] == "failed" 109 | assert "xyz" in data["aaData"][0]["traceback"] 110 | 111 | unit = "queues" 112 | data, _ = api.GET("/api/datatables/%s?sEcho=1" % unit) 113 | assert len(data["aaData"]) == 2 114 | 115 | # TODO: test unit = "scheduled_jobs" 116 | 117 | 118 | def test_routes_logs(worker, api): 119 | 120 | task_path = "tests.tasks.general.RaiseException" 121 | worker.send_task(task_path, { 122 | "message": "xyz"}, block=True, accept_statuses=["failed"]) 123 | 124 | job = worker.mongodb_jobs.mrq_jobs.find_one() 125 | assert job 126 | 127 | data, _ = api.GET("/api/logs?job=%s" % job["_id"]) 128 | 129 | assert bool(data["last_log_id"]) 130 | # TODO: test actual logs 131 | -------------------------------------------------------------------------------- /tests/test_sorted.py: -------------------------------------------------------------------------------- 1 | from mrq.job import Job 2 | import datetime 3 | from mrq.queue import Queue 4 | import time 5 | import pytest 6 | 7 | 8 | def test_sorted_graph(worker): 9 | 10 | p_queue = "test_sorted_set" 11 | 12 | worker.start_deps() 13 | 14 | assert Queue(p_queue).size() == 0 15 | 16 | worker.send_raw_tasks(p_queue, { 17 | "000": -1, 18 | "aaa": 1, 19 | "aaa2": 1.5, 20 | "bbb": 2, 21 | "ccc": 4 22 | }, start=False, block=False) 23 | time.sleep(0.5) 24 | 25 | assert Queue(p_queue).size() == 5 26 | assert Queue(p_queue).get_sorted_graph( 27 | 1, 4, slices=3, include_inf=True) == [1, 2, 1, 0, 1] 28 | assert Queue(p_queue).get_sorted_graph( 29 | 1, 4, slices=3, include_inf=False) == [2, 1, 0] 30 | 31 | worker.stop_deps() 32 | 33 | 34 | # def test_sorted_graph_dashboardtest(worker): 35 | 36 | # p_queue = "test_timed_set" 37 | 38 | # worker.start_deps() 39 | 40 | # assert Queue(p_queue).size() == 0 41 | 42 | # now = time.time() 43 | 44 | # worker.send_raw_tasks(p_queue, { 45 | # "000": now - 3600 * 6, 46 | # "aaa": now, 47 | # "aaa2": now + 3600 * 12, 48 | # "bbb": now + 3600000, 49 | # "ccc": now 50 | # }, start=False) 51 | # time.sleep(10000) 52 | 53 | # worker.stop_deps() 54 | -------------------------------------------------------------------------------- /tests/test_subpool.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from future import standard_library 3 | standard_library.install_aliases() 4 | from builtins import range 5 | from bson import ObjectId 6 | import urllib.request, urllib.error, urllib.parse 7 | import json 8 | import time 9 | import os 10 | import pytest 11 | 12 | 13 | @pytest.mark.parametrize(["use_worker"], [[False], [True]]) 14 | def test_subpool_simple(worker, use_worker): 15 | 16 | # Check that a subpool can be used both in an outside of a Job context 17 | if use_worker: 18 | worker.start() 19 | else: 20 | from tests.tasks.general import SubPool 21 | 22 | def run(params): 23 | if use_worker: 24 | return worker.send_task("tests.tasks.general.SubPool", params) 25 | else: 26 | return SubPool().run(params) 27 | 28 | # Check that sequential sleeps work 29 | start_time = time.time() 30 | result = run({ 31 | "pool_size": 1, "inner_params": [1, 1] 32 | }) 33 | total_time = time.time() - start_time 34 | 35 | assert result == [1, 1] 36 | assert total_time > 2 37 | 38 | # py.test doesn't use gevent so we don't get the benefits of the hub 39 | if use_worker: 40 | 41 | # Parallel sleeps 42 | start_time = time.time() 43 | result = run({ 44 | "pool_size": 20, "inner_params": [1] * 20 45 | }) 46 | total_time = time.time() - start_time 47 | 48 | assert result == [1] * 20 49 | assert total_time < 2 50 | 51 | 52 | @pytest.mark.parametrize(["p_imap"], [ 53 | [True], 54 | [False] 55 | ]) 56 | def test_subpool_exception(worker, p_imap): 57 | 58 | # An exception in the subpool is raised outside the pool 59 | worker.send_task("tests.tasks.general.SubPool", { 60 | "pool_size": 20, "inner_params": ["exception"], "imap": p_imap 61 | }, accept_statuses=["failed"]) 62 | 63 | job = worker.mongodb_jobs.mrq_jobs.find_one() 64 | assert job 65 | assert job["status"] == "failed" 66 | assert "__INNER_EXCEPTION_LINE__" in job["traceback"] 67 | 68 | 69 | @pytest.mark.parametrize(["p_size"], [ 70 | [0], 71 | [1], 72 | [2], 73 | [100] 74 | ]) 75 | def test_subpool_import(worker, p_size): 76 | """ This tests that the patch_import() function does its job of preventing a gevent crash 77 | like explained in https://code.google.com/p/gevent/issues/detail?id=108 """ 78 | 79 | # Large file import 80 | worker.send_task("tests.tasks.general.SubPool", { 81 | "pool_size": p_size, "inner_params": ["import-large-file"] * p_size 82 | }, accept_statuses=["success"]) 83 | 84 | 85 | def test_subpool_imap(): 86 | 87 | from mrq.context import subpool_imap 88 | 89 | def iterator(n): 90 | for i in range(0, n): 91 | if i == 5: 92 | raise Exception("Iterator exception!") 93 | yield i 94 | 95 | def inner_func(i): 96 | time.sleep(1) 97 | print("inner_func: %s" % i) 98 | if i == 4: 99 | raise Exception("Inner exception!") 100 | return i * 2 101 | 102 | with pytest.raises(Exception): 103 | for res in subpool_imap(10, inner_func, iterator(10)): 104 | print("Got %s" % res) 105 | 106 | for res in subpool_imap(2, inner_func, iterator(1)): 107 | print("Got %s" % res) 108 | 109 | with pytest.raises(Exception): 110 | for res in subpool_imap(2, inner_func, iterator(5)): 111 | print("Got %s" % res) 112 | -------------------------------------------------------------------------------- /tests/test_subqueues.py: -------------------------------------------------------------------------------- 1 | import time 2 | import pytest 3 | from mrq.job import Job 4 | from mrq.queue import Queue 5 | 6 | 7 | @pytest.mark.parametrize(["queues", "enqueue_on"], [ 8 | [["main/", "second/"], ["main/", "main/sub", "main/sub/nested", "second/x"]], 9 | [["prefix/main/"], ["prefix/main/", "prefix/main/sub", "prefix/main/sub/nested"]], 10 | ]) 11 | def test_matchable_subqueues(worker, queues, enqueue_on): 12 | worker.start(queues=" ".join(queues), flags="--subqueues_refresh_interval=0.1") 13 | 14 | job_ids = [] 15 | 16 | for subqueue in enqueue_on: 17 | job_id = worker.send_task("tests.tasks.general.GetTime", {}, queue=subqueue, block=False) 18 | job_ids.append(job_id) 19 | 20 | for i, job_id in enumerate(job_ids): 21 | print("Checking queue %s" % enqueue_on[i]) 22 | assert Job(job_id).wait(poll_interval=0.01, timeout=5) 23 | 24 | 25 | @pytest.mark.parametrize(["queue", "enqueue_on"], [ 26 | ["main/", ["/main", "main_", "/", "main", "other"]], 27 | ["prefix/main/", ["prefix", "prefix/other", "prefix/main"]], 28 | ]) 29 | def test_unmatchable_subqueues(worker, queue, enqueue_on): 30 | worker.start(queues=queue, flags="--subqueues_refresh_interval=0.1") 31 | 32 | job_ids = [] 33 | 34 | for subqueue in enqueue_on: 35 | job_id = worker.send_task("tests.tasks.general.GetTime", {}, queue=subqueue, block=False) 36 | job_ids.append(job_id) 37 | 38 | time.sleep(2) 39 | results = [Job(j).fetch().data.get("status") for j in job_ids] 40 | 41 | # ensure tasks are not consumed by a worker 42 | assert results == ["queued"] * len(results) 43 | 44 | 45 | def test_refresh_interval(worker): 46 | 47 | """ Tests that a refresh interval of 0 disables the subqueue detection """ 48 | 49 | worker.start(queues="test/", flags="--subqueues_refresh_interval=0") 50 | 51 | time.sleep(2) 52 | 53 | job_id1 = worker.send_task( 54 | "tests.tasks.general.GetTime", {"a": 41}, 55 | queue="test/subqueue", block=False) 56 | 57 | time.sleep(5) 58 | 59 | job1 = Job(job_id1).fetch().data 60 | 61 | assert job1["status"] == "queued" 62 | -------------------------------------------------------------------------------- /tests/test_timeout.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | def test_timeout_normal(worker): 5 | 6 | worker.start(flags="--config tests/fixtures/config1.py") 7 | 8 | r = worker.send_task( 9 | "tests.tasks.general.TimeoutFromConfig", {"a": 1, "b": 2}, block=True) 10 | assert r == 3 11 | 12 | r = worker.send_task("tests.tasks.general.TimeoutFromConfig", { 13 | "a": 1, "b": 2, "sleep": 1000}, block=True, accept_statuses=["timeout"]) 14 | assert r != 3 15 | 16 | r = worker.send_task("tests.tasks.general.TimeoutFromConfig", { 17 | "a": 1, "b": 2, "sleep": 1000, "broadexcept": True}, block=True, accept_statuses=["timeout"]) 18 | assert r != 3 19 | 20 | def test_timeout_global_config(worker): 21 | 22 | worker.start(env={"MRQ_DEFAULT_JOB_TIMEOUT": "1"}) 23 | 24 | assert worker.send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 0}) == 42 25 | assert worker.send_task("tests.tasks.general.Add", {"a": 41, "b": 1, "sleep": 2}, block=True, accept_statuses=["timeout"]) != 42 26 | 27 | 28 | def test_timeout_subpool(worker): 29 | 30 | worker.start(env={"MRQ_DEFAULT_JOB_TIMEOUT": "1"}) 31 | 32 | jobs = worker.send_task("tests.tasks.general.ListJobsByGreenlets", {}, block=True) 33 | 34 | other_jobs = [j for j in jobs["job_ids"] if j != jobs["current_job_id"]] 35 | assert len(other_jobs) == 0 36 | # Only one greenlet in current job 37 | assert jobs["job_ids"] == [jobs["current_job_id"]] 38 | 39 | r = worker.send_task("tests.tasks.general.SubPool", { 40 | "pool_size": 3, 41 | "inner_params": [1000, 1001, 1002, 1003, 1004] 42 | }, block=True, accept_statuses=["timeout"]) 43 | assert r is None 44 | 45 | time.sleep(1) 46 | 47 | jobs = worker.send_task("tests.tasks.general.ListJobsByGreenlets", {}, block=True) 48 | 49 | other_jobs = [j for j in jobs["job_ids"] if j != jobs["current_job_id"]] 50 | print("Leftover greenlets: %s" % other_jobs) 51 | assert len(other_jobs) == 0 52 | # Only one greenlet in current job 53 | assert jobs["job_ids"] == [jobs["current_job_id"]] 54 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | from mrq.utils import MovingETA 2 | 3 | 4 | def test_movingeta(): 5 | 6 | eta = MovingETA(2) 7 | 8 | assert eta.next(10, t=0) is None 9 | assert eta.next(5, t=1) == 1 10 | assert eta.next(4, t=2) == 4 11 | assert eta.next(2, t=10) == 8 12 | 13 | eta = MovingETA(3) 14 | assert eta.next(10, t=0) is None 15 | assert eta.next(9, t=1) == 9 16 | assert eta.next(8, t=2) == 8 17 | assert eta.next(7, t=3) == 7 18 | assert 3 < eta.next(5, t=4) < 4 19 | 20 | eta = MovingETA(2) 21 | assert eta.next(0, t=0) is None 22 | assert eta.next(0, t=1) is None 23 | --------------------------------------------------------------------------------