├── .git-blame-ignore-revs ├── .github └── workflows │ ├── publish.yml │ └── tests.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGES.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── conftest.py ├── pyproject.toml ├── scrapyd_client ├── __init__.py ├── __main__.py ├── deploy.py ├── exceptions.py ├── pyclient.py └── utils.py └── tests ├── __init__.py ├── conftest.py ├── test_errors.py ├── test_projects.py ├── test_pyclient_idempotent.py ├── test_pyclient_stateful.py ├── test_schedule.py ├── test_scrapyd_deploy.py ├── test_spiders.py ├── test_targets.py └── test_utils.py /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | b8f03726fdf364456b00e4ff47247ee699df2186 # flake8 2 | 29d6b6907fdc3c49f6bc35ca41a2a89e972e6db6 # black 3 | cdf4e55f9fae36c05decd6bef7ee5fcb1a467ca1 # ruff 4 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish to PyPI 2 | on: push 3 | jobs: 4 | publish: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v4 8 | - uses: actions/setup-python@v5 9 | with: 10 | python-version: 3.8 11 | - run: pip install --upgrade build 12 | - run: python -m build --sdist --wheel 13 | - name: Publish to TestPyPI 14 | uses: pypa/gh-action-pypi-publish@release/v1 15 | with: 16 | password: ${{ secrets.TEST_PYPI_TOKEN }} 17 | repository-url: https://test.pypi.org/legacy/ 18 | skip-existing: true 19 | - name: Publish to PyPI 20 | if: startsWith(github.ref, 'refs/tags') 21 | uses: pypa/gh-action-pypi-publish@release/v1 22 | with: 23 | password: ${{ secrets.PYPI_TOKEN }} 24 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: [push, pull_request] 3 | jobs: 4 | tests: 5 | if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository 6 | runs-on: ${{ matrix.os }} 7 | strategy: 8 | matrix: 9 | os: [macos-latest, ubuntu-latest, windows-latest] 10 | python-version: [3.9, '3.10', '3.11', '3.12', '3.13', pypy-3.10] 11 | # It is difficult to install libxml2 and libxslt development packages on Windows. 12 | # https://www.lfd.uci.edu/~gohlke/pythonlibs/ distributes a wheel, but the URL changes. 13 | exclude: 14 | - os: windows-latest 15 | python-version: pypy-3.10 16 | steps: 17 | - uses: actions/checkout@v4 18 | - uses: actions/setup-python@v5 19 | with: 20 | python-version: ${{ matrix.python-version }} 21 | cache: pip 22 | cache-dependency-path: pyproject.toml 23 | - if: matrix.os == 'ubuntu-latest' && matrix.python-version == 'pypy-3.10' 24 | run: | 25 | sudo apt-get update 26 | sudo apt-get install libxml2-dev libxslt-dev 27 | - run: pip install .[test] 28 | - run: coverage run --source=scrapyd_client -m pytest 29 | - if: matrix.os == 'ubuntu-latest' 30 | run: bash <(curl -s https://codecov.io/bash) 31 | - run: pip install -U check-manifest setuptools 32 | - run: check-manifest 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *egg-info 3 | dist 4 | build 5 | htmlcov 6 | .cache/ 7 | .coverage 8 | venv/ 9 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | autoupdate_schedule: quarterly 3 | repos: 4 | - repo: https://github.com/astral-sh/ruff-pre-commit 5 | rev: v0.6.3 6 | hooks: 7 | - id: ruff 8 | - id: ruff-format 9 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | History 2 | ------- 3 | 4 | 2.0.3 (2025-05-15) 5 | ~~~~~~~~~~~~~~~~~~ 6 | 7 | Added 8 | ^^^^^ 9 | 10 | - ``scrapyd_client.ScrapyClient.cancel`` 11 | - ``scrapyd_client.ScrapyClient.daemonstatus`` 12 | - ``scrapyd_client.ScrapyClient.delproject`` 13 | - ``scrapyd_client.ScrapyClient.delversion`` 14 | - ``scrapyd_client.ScrapyClient.versions`` 15 | 16 | Fixed 17 | ^^^^^ 18 | 19 | - ``scrapyd-client schedule`` no longer errors when parsing ``--arg`` arguments. (Regression in 2.0.0: "The ``scrapyd-client schedule`` subcommand accepts multiple ``--arg setting=...`` arguments.") 20 | 21 | 2.0.2 (2025-03-08) 22 | ~~~~~~~~~~~~~~~~~~ 23 | 24 | Fixed 25 | ^^^^^ 26 | 27 | - ``scrapyd_client.ScrapyClient`` uses the target's URL instead of the target's name when constructing URLs. (Regression in 2.0.0: "``scrapyd-client`` raises an error if no Scrapy project is found, like ``scrapyd-deploy``, instead of assuming a target at ``http://localhost:6800``") 28 | 29 | 2.0.1 (2025-02-12) 30 | ~~~~~~~~~~~~~~~~~~ 31 | 32 | Added 33 | ^^^^^ 34 | 35 | - ``scrapyd_client.ScrapyClient.status`` 36 | 37 | 2.0.0 (2024-10-11) 38 | ~~~~~~~~~~~~~~~~~~ 39 | 40 | Added 41 | ^^^^^ 42 | 43 | - Add ``scrapyd-client targets`` subcommand. 44 | - ``scrapyd-client`` can be called as ``python -m scrapyd_client``. 45 | - Add support for Python 3.12. 46 | 47 | Changed 48 | ^^^^^^^ 49 | 50 | Changes to commands: 51 | 52 | - **BREAKING CHANGE:** ``scrapyd-client`` subcommands accept the ``--target`` (``-t``) option, not the ``scrapyd-client`` command. The target is the name of a target in the ``scrapy.cfg`` file, like ``scrapyd-deploy``, instead of a URL. 53 | - **BREAKING CHANGE:** ``scrapyd-client`` raises an error if no Scrapy project is found, like ``scrapyd-deploy``, instead of assuming a target at ``http://localhost:6800``. 54 | - The ``scrapyd-client schedule`` subcommand accepts multiple ``--arg setting=...`` arguments. (@mxdev88) 55 | - The ``scrapyd_client.ScrapyClient.schedule`` methods accept ``args`` as a list, instead of as a dict. 56 | - The ``scrapyd-deploy --debug`` option prints the subprocess' standard output and standard error, instead of writing to ``stdout`` and ``stderr`` files. 57 | 58 | Changes to modules: 59 | 60 | - **BREAKING CHANGE:** Move exceptions from ``scrapyd_client.utils`` to ``scrapyd_client.exceptions``. 61 | - **BREAKING CHANGE:** Move ``DEFAULT_TARGET_URL`` and ``HEADERS`` from ``scrapyd_client.utils`` to ``scrapyd_client.pyclient``. 62 | - **BREAKING CHANGE:** Merge ``scrapyd_client.commands`` and ``scrapyd_client.cli`` into ``scrapyd_client.__main__``. 63 | 64 | Fixed 65 | ^^^^^ 66 | 67 | - Run ``clean`` separately from building the egg. (Setuptools caches the Python packages from before ``clean`` runs.) 68 | - Add ``pip`` requirement for ``uberegg``. 69 | 70 | Removed 71 | ^^^^^^^ 72 | 73 | - **BREAKING CHANGE:** Remove the ``scrapyd-client --username`` (``-u``) and ``--password`` (``-p``) options, in favor of using the ``scrapy.cfg`` file. 74 | - **BREAKING CHANGE:** Remove the ``scrapyd-deploy --list-targets`` (``-l``) option, in favor of ``scrapyd-client targets``. 75 | - **BREAKING CHANGE:** Remove the ``scrapyd-deploy --list-projects`` (``-L``) option, in favor of ``scrapyd-client projects``. 76 | - **BREAKING CHANGE:** Remove the ``get_request`` and ``post_request`` functions from ``scrapyd_client.utils``. 77 | - **BREAKING CHANGE:** Remove the ``scrapyd_client.lib`` module, in favor of ``scrapyd_client.ScrapydClient``. 78 | - Remove ``urllib3`` and ``w3lib`` requirements. 79 | - Drop support for Python 3.7, 3.8. 80 | 81 | 1.2.3 (2023-01-30) 82 | ~~~~~~~~~~~~~~~~~~ 83 | 84 | Added 85 | ^^^^^ 86 | 87 | - Add ``scrapyd-client --username`` and ``--password`` options. (@mxdev88) 88 | - Add ``ScrapydClient``, a Python client to interact with Scrapyd. (@mxdev88) 89 | - Expand environment variables in the ``scrapy.cfg`` file. (@mxdev88) 90 | - Add support for Python 3.10, 3.11. (@Laerte) 91 | 92 | 1.2.2 (2022-05-03) 93 | ~~~~~~~~~~~~~~~~~~ 94 | 95 | Fixed 96 | ^^^^^ 97 | 98 | - Fix ``FileNotFoundError`` when using ``scrapyd-deploy --deploy-all-targets``. 99 | 100 | 1.2.1 (2022-05-02) 101 | ~~~~~~~~~~~~~~~~~~ 102 | 103 | Added 104 | ^^^^^ 105 | 106 | - Add ``scrapyd-deploy --include-dependencies`` option to install project dependencies from a ``requirements.txt`` file. (@mxdev88) 107 | 108 | Changed 109 | ^^^^^^^ 110 | 111 | Fixed 112 | ^^^^^ 113 | 114 | - Remove temporary directories created by ``scrapyd-deploy --deploy-all-targets``. 115 | - Address ``w3lib`` deprecation warning, by adding ``urllib3`` requirement. 116 | - Address Python deprecation warnings. 117 | 118 | 1.2.0 (2021-10-01) 119 | ~~~~~~~~~~~~~~~~~~ 120 | 121 | Added 122 | ^^^^^ 123 | 124 | - Add support for Scrapy 2.5. 125 | - Add support for Python 3.7, 3.8, 3.9, PyPy3.7. 126 | 127 | Removed 128 | ^^^^^^^ 129 | 130 | - Remove ``scrapyd_client.utils.get_config``, which was a compatibility wrapper for Python 2.7. 131 | - Drop support for Python 2.7, 3.4, 3.5. 132 | 133 | 1.2.0a1 (2017-08-24) 134 | ~~~~~~~~~~~~~~~~~~~~ 135 | 136 | Added 137 | ^^^^^ 138 | 139 | - Add ``scrapyd-client`` console script with ``deploy``, ``projects``, ``spiders`` and ``schedule`` subcommands. 140 | - Install ``scrapyd-deploy`` as a console script. 141 | 142 | 1.1.0 (2017-02-10) 143 | ~~~~~~~~~~~~~~~~~~ 144 | 145 | Added 146 | ^^^^^ 147 | 148 | - Add ``scrapyd-deploy --deploy-all-targets`` (``-a``) option to deploy to all targets. 149 | - Add support for Python 3. 150 | 151 | Fixed 152 | ^^^^^ 153 | 154 | - Fix returncode on egg deploy error. 155 | 156 | Removed 157 | ^^^^^^^ 158 | 159 | - Drop support for Python 2.6. 160 | 161 | 1.0.1 (2015-04-09) 162 | ~~~~~~~~~~~~~~~~~~ 163 | 164 | Initial release. 165 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Scrapy developers. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of Scrapy nor the names of its contributors may be used 15 | to endorse or promote products derived from this software without 16 | specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.py 2 | include *.rst 3 | include LICENSE 4 | recursive-include tests *.py 5 | exclude .git-blame-ignore-revs 6 | exclude .pre-commit-config.yaml 7 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | Scrapyd-client 3 | ============== 4 | 5 | |PyPI Version| |Build Status| |Coverage Status| |Python Version| 6 | 7 | Scrapyd-client is a client for Scrapyd_. It provides: 8 | 9 | Command line tools: 10 | 11 | - ``scrapyd-deploy``, to deploy your project to a Scrapyd server 12 | - ``scrapyd-client``, to interact with your project once deployed 13 | 14 | Python client: 15 | 16 | - ``ScrapydClient``, to interact with Scrapyd within your python code 17 | 18 | It is configured using the `Scrapy configuration file`_. 19 | 20 | .. _Scrapyd: https://scrapyd.readthedocs.io 21 | .. |PyPI Version| image:: https://img.shields.io/pypi/v/scrapyd-client.svg 22 | :target: https://pypi.org/project/scrapyd-client/ 23 | .. |Build Status| image:: https://github.com/scrapy/scrapyd-client/workflows/Tests/badge.svg 24 | .. |Coverage Status| image:: https://codecov.io/gh/scrapy/scrapyd-client/branch/master/graph/badge.svg 25 | :target: https://codecov.io/gh/scrapy/scrapyd-client 26 | .. |Python Version| image:: https://img.shields.io/pypi/pyversions/scrapyd-client.svg 27 | :target: https://pypi.org/project/scrapyd-client/ 28 | 29 | 30 | scrapyd-deploy 31 | -------------- 32 | 33 | Deploying your project to a Scrapyd server involves: 34 | 35 | #. `Eggifying `__ your project. 36 | #. Uploading the egg to the Scrapyd server through the `addversion.json `__ webservice. 37 | 38 | The ``scrapyd-deploy`` tool automates the process of building the egg and pushing it to the target Scrapyd server. 39 | 40 | Deploying a project 41 | ~~~~~~~~~~~~~~~~~~~ 42 | 43 | #. Change (``cd``) to the root of your project (the directory containing the ``scrapy.cfg`` file) 44 | #. Eggify your project and upload it to the target: 45 | 46 | .. code-block:: shell 47 | 48 | scrapyd-deploy -p 49 | 50 | If you don't have a ``setup.py`` file in the root of your project, one will be created. If you have one, it must set the ``entry_points`` keyword argument in the ``setup()`` function call, for example: 51 | 52 | .. code-block:: python 53 | 54 | setup( 55 | name = 'project', 56 | version = '1.0', 57 | packages = find_packages(), 58 | entry_points = {'scrapy': ['settings = projectname.settings']}, 59 | ) 60 | 61 | If the command is successful, you should see a JSON response, like: 62 | 63 | .. code-block:: none 64 | 65 | Deploying myproject-1287453519 to http://localhost:6800/addversion.json 66 | Server response (200): 67 | {"status": "ok", "spiders": ["spider1", "spider2"]} 68 | 69 | To save yourself from having to specify the target and project, you can configure your defaults in the `Scrapy configuration file`_. 70 | 71 | Versioning 72 | ~~~~~~~~~~ 73 | 74 | By default, ``scrapyd-deploy`` uses the current timestamp for generating the project version. You can pass a custom version using ``--version``: 75 | 76 | .. code-block:: shell 77 | 78 | scrapyd-deploy -p --version 79 | 80 | See `Scrapyd's documentation `__ on how it determines the latest version. 81 | 82 | If you use Mercurial or Git, you can use ``HG`` or ``GIT`` respectively as the argument supplied to 83 | ``--version`` to use the current revision as the version. You can save yourself having to specify 84 | the version parameter by adding it to your target's entry in ``scrapy.cfg``: 85 | 86 | .. code-block:: ini 87 | 88 | [deploy] 89 | ... 90 | version = HG 91 | 92 | Note: The ``version`` keyword argument in the ``setup()`` function call in the ``setup.py`` file has no meaning to Scrapyd. 93 | 94 | Include dependencies 95 | ~~~~~~~~~~~~~~~~~~~~ 96 | 97 | #. Create a `requirements.txt `__ file at the root of your project, alongside the ``scrapy.cfg`` file 98 | #. Use the ``--include-dependencies`` option when building or deploying your project: 99 | 100 | .. code-block:: bash 101 | 102 | scrapyd-deploy --include-dependencies 103 | 104 | Alternatively, you can install the dependencies directly on the Scrapyd server. 105 | 106 | Include data files 107 | ~~~~~~~~~~~~~~~~~~ 108 | 109 | #. Create a ``setup.py`` file at the root of your project, alongside the ``scrapy.cfg`` file, if you don't have one: 110 | 111 | .. code-block:: shell 112 | 113 | scrapyd-deploy --build-egg=/dev/null 114 | 115 | #. Set the ``package_data`` and ``include_package_data` keyword arguments in the ``setup()`` function call in the ``setup.py`` file. For example: 116 | 117 | .. code-block:: python 118 | 119 | from setuptools import setup, find_packages 120 | 121 | setup( 122 | name = 'project', 123 | version = '1.0', 124 | packages = find_packages(), 125 | entry_points = {'scrapy': ['settings = projectname.settings']}, 126 | package_data = {'projectname': ['path/to/*.json']}, 127 | include_package_data = True, 128 | ) 129 | 130 | Local settings 131 | ~~~~~~~~~~~~~~ 132 | 133 | You may want to keep certain settings local and not have them deployed to Scrapyd. 134 | 135 | #. Create a ``local_settings.py`` file at the root of your project, alongside the ``scrapy.cfg`` file 136 | #. Add the following to your project's settings file: 137 | 138 | .. code-block:: python 139 | 140 | try: 141 | from local_settings import * 142 | except ImportError: 143 | pass 144 | 145 | ``scrapyd-deploy`` doesn't deploy anything outside of the project module, so the ``local_settings.py`` file won't be deployed. 146 | 147 | Troubleshooting 148 | ~~~~~~~~~~~~~~~ 149 | 150 | - Problem: A settings file for local development is being included in the egg. 151 | 152 | Solution: See `Local settings`_. Or, exclude the module from the egg. If using scrapyd-client's default ``setup.py`` file, change the ``find_package()`` call: 153 | 154 | .. code-block:: python 155 | 156 | setup( 157 | name = 'project', 158 | version = '1.0', 159 | packages = find_packages(), 160 | entry_points = {'scrapy': ['settings = projectname.settings']}, 161 | ) 162 | 163 | to: 164 | 165 | .. code-block:: python 166 | 167 | setup( 168 | name = 'project', 169 | version = '1.0', 170 | packages = find_packages(exclude=["myproject.devsettings"]), 171 | entry_points = {'scrapy': ['settings = projectname.settings']}, 172 | ) 173 | 174 | - Problem: Code using ``__file__`` breaks when run in Scrapyd. 175 | 176 | Solution: Use `pkgutil.get_data `__ instead. For example, change: 177 | 178 | .. code-block:: python 179 | 180 | path = os.path.dirname(os.path.realpath(__file__)) # BAD 181 | open(os.path.join(path, "tools", "json", "test.json"), "rb").read() 182 | 183 | to: 184 | 185 | .. code-block:: python 186 | 187 | import pkgutil 188 | pkgutil.get_data("projectname", "tools/json/test.json") 189 | 190 | - Be careful when writing to disk in your project, as Scrapyd will most likely be running under a 191 | different user which may not have write access to certain directories. If you can, avoid writing 192 | to disk and always use `tempfile `__ for temporary files. 193 | 194 | - If you use a proxy, use the ``HTTP_PROXY``, ``HTTPS_PROXY``, ``NO_PROXY`` and/or ``ALL_PROXY`` environment variables, 195 | as documented by the `requests `__ package. 196 | 197 | scrapyd-client 198 | -------------- 199 | 200 | For a reference on each subcommand invoke ``scrapyd-client --help``. 201 | 202 | Where filtering with wildcards is possible, it is facilitated with `fnmatch `__. 203 | The ``--project`` option can be omitted if one is found in a ``scrapy.cfg``. 204 | 205 | deploy 206 | ~~~~~~ 207 | 208 | This is a wrapper around `scrapyd-deploy`_. 209 | 210 | targets 211 | ~~~~~~~ 212 | 213 | Lists all targets: 214 | 215 | scrapyd-client targets 216 | 217 | projects 218 | ~~~~~~~~ 219 | 220 | Lists all projects of a Scrapyd instance:: 221 | 222 | # lists all projects on the default target 223 | scrapyd-client projects 224 | # lists all projects from a custom URL 225 | scrapyd-client -t http://scrapyd.example.net projects 226 | 227 | schedule 228 | ~~~~~~~~ 229 | 230 | Schedules one or more spiders to be executed:: 231 | 232 | # schedules any spider 233 | scrapyd-client schedule 234 | # schedules all spiders from the 'knowledge' project 235 | scrapyd-client schedule -p knowledge \* 236 | # schedules any spider from any project whose name ends with '_daily' 237 | scrapyd-client schedule -p \* \*_daily 238 | # schedules spider1 in project1 specifying settings 239 | scrapyd-client schedule -p project1 spider1 --arg 'setting=DOWNLOADER_MIDDLEWARES={"my.middleware.MyDownloader": 610}' 240 | 241 | spiders 242 | ~~~~~~~ 243 | 244 | Lists spiders of one or more projects:: 245 | 246 | # lists all spiders 247 | scrapyd-client spiders 248 | # lists all spiders from the 'knowledge' project 249 | scrapyd-client spiders -p knowledge 250 | 251 | ScrapydClient 252 | ------------- 253 | 254 | Interact with Scrapyd within your python code. 255 | 256 | .. code-block:: python 257 | 258 | from scrapyd_client import ScrapydClient 259 | client = ScrapydClient() 260 | 261 | for project in client.projects(): 262 | print(client.jobs(project=project)) 263 | 264 | 265 | Scrapy configuration file 266 | ------------------------- 267 | 268 | Targets 269 | ~~~~~~~ 270 | 271 | You can define a Scrapyd target in your project's ``scrapy.cfg`` file. Example: 272 | 273 | .. code-block:: ini 274 | 275 | [deploy] 276 | url = http://scrapyd.example.com/api/scrapyd 277 | username = scrapy 278 | password = secret 279 | project = projectname 280 | 281 | You can now deploy your project without the ```` argument or ``-p `` option:: 282 | 283 | scrapyd-deploy 284 | 285 | If you have multiple targets, add the target name in the section name. Example: 286 | 287 | .. code-block:: ini 288 | 289 | [deploy:targetname] 290 | url = http://scrapyd.example.com/api/scrapyd 291 | 292 | [deploy:another] 293 | url = http://other.example.com/api/scrapyd 294 | 295 | If you are working with CD frameworks, you do not need to commit your secrets to your repository. You can use environment variable expansion like so: 296 | 297 | .. code-block:: ini 298 | 299 | [deploy] 300 | url = $SCRAPYD_URL 301 | username = $SCRAPYD_USERNAME 302 | password = $SCRAPYD_PASSWORD 303 | 304 | or using this syntax: 305 | 306 | .. code-block:: ini 307 | 308 | [deploy] 309 | url = ${SCRAPYD_URL} 310 | username = ${SCRAPYD_USERNAME} 311 | password = ${SCRAPYD_PASSWORD} 312 | 313 | To deploy to one target, run:: 314 | 315 | scrapyd-deploy targetname -p 316 | 317 | To deploy to all targets, use the ``-a`` option:: 318 | 319 | scrapyd-deploy -a -p 320 | 321 | While your target needs to be defined with its URL in ``scrapy.cfg``, 322 | you can use `netrc `__ for username and password, like so:: 323 | 324 | machine scrapyd.example.com 325 | login scrapy 326 | password secret 327 | -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from scrapy.utils import conf 3 | 4 | 5 | @pytest.fixture(autouse=True) 6 | def only_closest_scrapy_cfg(monkeypatch): 7 | """Avoids a developer's own configuration files interfering with tests.""" 8 | 9 | def get_sources(use_closest=True): 10 | if use_closest: 11 | return [conf.closest_scrapy_cfg()] 12 | return [] 13 | 14 | monkeypatch.setattr(conf, "get_sources", get_sources) 15 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.2"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "scrapyd-client" 7 | version = "2.0.3" 8 | authors = [{name = "Scrapy developers", email = "info@scrapy.org"}] 9 | description = "A client for Scrapyd" 10 | readme = "README.rst" 11 | license = {text = "BSD"} 12 | urls = {Homepage = "https://github.com/scrapy/scrapyd-client"} 13 | classifiers = [ 14 | "Framework :: Scrapy", 15 | "Development Status :: 5 - Production/Stable", 16 | "Environment :: Console", 17 | "Intended Audience :: Developers", 18 | "Topic :: Internet :: WWW/HTTP", 19 | "License :: OSI Approved :: BSD License", 20 | "Operating System :: OS Independent", 21 | "Programming Language :: Python :: 3.9", 22 | "Programming Language :: Python :: 3.10", 23 | "Programming Language :: Python :: 3.11", 24 | "Programming Language :: Python :: 3.12", 25 | "Programming Language :: Python :: 3.13", 26 | "Programming Language :: Python :: Implementation :: CPython", 27 | "Programming Language :: Python :: Implementation :: PyPy", 28 | ] 29 | dependencies = [ 30 | "pip", # uberegg 31 | "requests", 32 | "scrapy>=0.17", 33 | "setuptools", 34 | "uberegg>=0.1.1", 35 | ] 36 | 37 | [project.optional-dependencies] 38 | test = [ 39 | "coverage", 40 | "pytest", 41 | "pytest-console-scripts", 42 | "pytest-mock", 43 | ] 44 | 45 | [project.scripts] 46 | scrapyd-deploy = "scrapyd_client.deploy:main" 47 | scrapyd-client = "scrapyd_client.__main__:main" 48 | 49 | [tool.setuptools.packages.find] 50 | exclude = [ 51 | "tests", 52 | "tests.*", 53 | ] 54 | 55 | [tool.ruff] 56 | line-length = 119 57 | target-version = "py39" 58 | 59 | [tool.ruff.lint] 60 | select = ["ALL"] 61 | ignore = [ 62 | "ANN", "C901", "COM812", "D203", "D212", "D415", "EM", "ISC001", "PERF203", "PLR091", "Q000", 63 | "D1", 64 | "PTH", 65 | "FBT002", # positional boolean 66 | "S113", # timeout 67 | "S310", # URL scheme 68 | "S603", # untrusted input 69 | "S607", # executable path 70 | "T201", # print 71 | "TRY003" , # errors 72 | ] 73 | 74 | [tool.ruff.lint.flake8-builtins] 75 | builtins-ignorelist = ["copyright"] 76 | 77 | [tool.ruff.lint.flake8-unused-arguments] 78 | ignore-variadic-names = true 79 | 80 | [tool.ruff.lint.per-file-ignores] 81 | "docs/conf.py" = ["D100", "INP001"] 82 | "tests/*" = [ 83 | "ARG001", "D", "FBT003", "INP001", "PLR2004", "S", "TRY003", 84 | ] 85 | -------------------------------------------------------------------------------- /scrapyd_client/__init__.py: -------------------------------------------------------------------------------- 1 | from scrapyd_client.pyclient import ScrapydClient # noqa: F401 2 | -------------------------------------------------------------------------------- /scrapyd_client/__main__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from argparse import ArgumentParser 3 | from textwrap import indent 4 | from traceback import print_exc 5 | 6 | import requests 7 | from scrapy.utils.project import inside_project 8 | 9 | import scrapyd_client.deploy 10 | from scrapyd_client.exceptions import ErrorResponse, MalformedResponse 11 | from scrapyd_client.pyclient import ScrapydClient 12 | from scrapyd_client.utils import _get_targets, get_config 13 | 14 | ISSUE_TRACKER_URL = "https://github.com/scrapy/scrapyd-client/issues" 15 | 16 | 17 | def _get_client(args): 18 | target = _get_targets()[args.target] 19 | 20 | return ScrapydClient(target.get("url"), target.get("username"), password=target.get("password", "")) 21 | 22 | 23 | def deploy(args): # noqa: ARG001 24 | """Deploy a Scrapy project to a Scrapyd instance. For help, invoke scrapyd-deploy.""" 25 | sys.argv.pop(1) 26 | scrapyd_client.deploy.main() 27 | 28 | 29 | def targets(args): # noqa: ARG001 30 | """List all targets.""" 31 | for name, target in _get_targets().items(): 32 | print("%-20s %s" % (name, target["url"])) 33 | 34 | 35 | def projects(args): 36 | """List all projects deployed on a Scrapyd instance.""" 37 | client = _get_client(args) 38 | 39 | if projects := client.projects(): 40 | print("\n".join(projects)) 41 | 42 | 43 | def schedule(args): 44 | """Schedule the specified spider(s).""" 45 | client = _get_client(args) 46 | job_args = [tuple(job_arg.split("=", 1)) for job_arg in args.arg] 47 | 48 | for project in client.projects(args.project): 49 | for spider in client.spiders(project, args.spider): 50 | job_id = client.schedule(project, spider, job_args) 51 | print(f"{project} / {spider} => {job_id}") 52 | 53 | 54 | def spiders(args): 55 | """List all spiders for the given project(s).""" 56 | client = _get_client(args) 57 | 58 | for project in client.projects(args.project): 59 | spiders = client.spiders(project) 60 | if not args.verbose: 61 | print(f"{project}:") 62 | if spiders: 63 | print(indent("\n".join(spiders), " ")) 64 | else: 65 | print(" No spiders.") 66 | elif spiders: 67 | print("\n".join(f"{project} {spider}" for spider in spiders)) 68 | 69 | 70 | def parse_cli_args(args): 71 | cfg = get_config() 72 | 73 | project_kwargs = { 74 | "metavar": "PROJECT", 75 | "required": True, 76 | "help": "Specifies the project, can be a globbing pattern.", 77 | } 78 | if project_default := cfg.get("deploy", "project", fallback=None): 79 | project_kwargs["default"] = project_default 80 | 81 | description = "A command line interface for Scrapyd." 82 | mainparser = ArgumentParser(description=description) 83 | subparsers = mainparser.add_subparsers() 84 | 85 | parser = subparsers.add_parser("deploy", description=deploy.__doc__) 86 | parser.set_defaults(action=deploy) 87 | 88 | parser = subparsers.add_parser("targets", description=targets.__doc__) 89 | parser.set_defaults(action=targets) 90 | 91 | parser = subparsers.add_parser("projects", description=projects.__doc__) 92 | parser.set_defaults(action=projects) 93 | parser.add_argument("-t", "--target", default="default", help="Specifies the target Scrapyd server by name.") 94 | 95 | parser = subparsers.add_parser("schedule", description=schedule.__doc__) 96 | parser.set_defaults(action=schedule) 97 | parser.add_argument("-t", "--target", default="default", help="Specifies the target Scrapyd server by name.") 98 | parser.add_argument("-p", "--project", **project_kwargs) 99 | parser.add_argument( 100 | "spider", 101 | metavar="SPIDER", 102 | help="Specifies the spider, can be a globbing pattern.", 103 | ) 104 | parser.add_argument( 105 | "--arg", 106 | action="append", 107 | default=[], 108 | help="Additional argument (key=value), can be specified multiple times.", 109 | ) 110 | 111 | parser = subparsers.add_parser("spiders", description=spiders.__doc__) 112 | parser.set_defaults(action=spiders) 113 | parser.add_argument("-t", "--target", default="default", help="Specifies the target Scrapyd server by name.") 114 | parser.add_argument("-p", "--project", **project_kwargs) 115 | parser.add_argument( 116 | "-v", 117 | "--verbose", 118 | action="store_true", 119 | default=False, 120 | help="Prints project's and spider's name in each line, intended for " "processing stdout in scripts.", 121 | ) 122 | 123 | # If 'deploy' is moved to this module, these lines can be removed. (b9ba799) 124 | parsed_args, _ = mainparser.parse_known_args(args) 125 | if getattr(parsed_args, "action", None) is not deploy: 126 | parsed_args = mainparser.parse_args(args) 127 | 128 | if not hasattr(parsed_args, "action"): 129 | mainparser.print_help() 130 | raise SystemExit(0) 131 | 132 | return parsed_args 133 | 134 | 135 | def main(): 136 | if not inside_project(): 137 | print("Error: no Scrapy project found in this location", file=sys.stderr) 138 | sys.exit(1) 139 | 140 | max_response_length = 120 141 | try: 142 | args = parse_cli_args(sys.argv[1:]) 143 | args.action(args) 144 | except KeyboardInterrupt: 145 | print("Aborted due to keyboard interrupt.") 146 | exit_code = 0 147 | except SystemExit as e: 148 | exit_code = e.code 149 | except requests.ConnectionError as e: 150 | print(f"Failed to connect to target ({args.target}):") 151 | print(e) 152 | exit_code = 1 153 | except ErrorResponse as e: 154 | print("Scrapyd responded with an error:") 155 | print(e) 156 | exit_code = 1 157 | except MalformedResponse as e: 158 | text = str(e) 159 | if len(text) > max_response_length: 160 | text = f"{text[:50]} [...] {text[-50:]}" 161 | print("Received a malformed response:") 162 | print(text) 163 | exit_code = 1 164 | except Exception: # noqa: BLE001 165 | print(f"Caught unhandled exception, please report at {ISSUE_TRACKER_URL}") 166 | print_exc() 167 | exit_code = 3 168 | else: 169 | exit_code = 0 170 | finally: 171 | raise SystemExit(exit_code) 172 | 173 | 174 | if __name__ == "__main__": 175 | main() 176 | -------------------------------------------------------------------------------- /scrapyd_client/deploy.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import json 3 | import os 4 | import shutil 5 | import subprocess 6 | import sys 7 | import tempfile 8 | import time 9 | from argparse import ArgumentParser 10 | from urllib.parse import urljoin 11 | 12 | import requests 13 | from requests.auth import HTTPBasicAuth 14 | from scrapy.utils.conf import closest_scrapy_cfg 15 | from scrapy.utils.project import inside_project 16 | 17 | from scrapyd_client.utils import _get_targets, get_auth, get_config 18 | 19 | _SETUP_PY_TEMPLATE = """ 20 | # Automatically created by: scrapyd-deploy 21 | 22 | from setuptools import setup, find_packages 23 | 24 | setup( 25 | name = 'project', 26 | version = '1.0', 27 | packages = find_packages(), 28 | entry_points = {'scrapy': ['settings = %(settings)s']}, 29 | ) 30 | """.lstrip() 31 | 32 | 33 | def parse_args(): 34 | parser = ArgumentParser(description="Deploy Scrapy project to Scrapyd server") 35 | parser.add_argument("target", nargs="?", default="default", metavar="TARGET") 36 | parser.add_argument("-p", "--project", help="the project name in the TARGET") 37 | parser.add_argument("-v", "--version", help="the version to deploy. Defaults to current timestamp") 38 | parser.add_argument("-a", "--deploy-all-targets", action="store_true", help="deploy all targets") 39 | parser.add_argument( 40 | "-d", 41 | "--debug", 42 | action="store_true", 43 | help="debug mode (do not remove build dir)", 44 | ) 45 | parser.add_argument("--egg", metavar="FILE", help="use the given egg, instead of building it") 46 | parser.add_argument("--build-egg", metavar="FILE", help="only build the egg, don't deploy it") 47 | parser.add_argument( 48 | "--include-dependencies", 49 | action="store_true", 50 | help="include dependencies from requirements.txt in the egg", 51 | ) 52 | return parser.parse_args() 53 | 54 | 55 | def main(): 56 | opts = parse_args() 57 | exitcode = 0 58 | if not inside_project(): 59 | print("Error: no Scrapy project found in this location", file=sys.stderr) 60 | sys.exit(1) 61 | 62 | tmpdir = None 63 | 64 | if opts.build_egg: # build egg only 65 | eggpath, tmpdir = _build_egg(opts) 66 | print(f"Writing egg to {opts.build_egg}", file=sys.stderr) 67 | shutil.copyfile(eggpath, opts.build_egg) 68 | elif opts.deploy_all_targets: 69 | version = None 70 | for target in _get_targets().values(): 71 | if version is None: 72 | version = _get_version(target, opts) 73 | _, tmpdir = _build_egg_and_deploy_target(target, version, opts) 74 | _remove_tmpdir(tmpdir, opts) 75 | else: # build egg and deploy 76 | try: 77 | target = _get_targets()[opts.target] 78 | except KeyError: 79 | print(f"Unknown target: {opts.target}", file=sys.stderr) 80 | sys.exit(1) 81 | 82 | version = _get_version(target, opts) 83 | exitcode, tmpdir = _build_egg_and_deploy_target(target, version, opts) 84 | _remove_tmpdir(tmpdir, opts) 85 | 86 | sys.exit(exitcode) 87 | 88 | 89 | def _remove_tmpdir(tmpdir, opts): 90 | if tmpdir: 91 | if opts.debug: 92 | print(f"Output dir not removed: {tmpdir}", file=sys.stderr) 93 | else: 94 | shutil.rmtree(tmpdir) 95 | 96 | 97 | def _build_egg_and_deploy_target(target, version, opts): 98 | exitcode = 0 99 | tmpdir = None 100 | 101 | project = opts.project or target.get("project") 102 | if not project: 103 | print("Error: Missing project", file=sys.stderr) 104 | sys.exit(1) 105 | 106 | if opts.egg: 107 | print(f"Using egg: {opts.egg}", file=sys.stderr) 108 | eggpath = opts.egg 109 | else: 110 | print(f"Packing version {version}", file=sys.stderr) 111 | eggpath, tmpdir = _build_egg(opts) 112 | 113 | url = _url(target, "addversion.json") 114 | print(f'Deploying to project "{project}" in {url}', file=sys.stderr) 115 | 116 | # Upload egg. 117 | kwargs = {} 118 | if auth := get_auth(url=target["url"], username=target.get("username"), password=target.get("password", "")): 119 | kwargs["auth"] = HTTPBasicAuth(auth.username, auth.password) 120 | 121 | try: 122 | with open(eggpath, "rb") as f: 123 | response = requests.post( 124 | _url(target, "addversion.json"), 125 | data={"project": project, "version": version}, 126 | files=[("egg", ("project.egg", f))], 127 | **kwargs, 128 | ) 129 | response.raise_for_status() 130 | print(f"Server response ({response.status_code}):", file=sys.stderr) 131 | print(response.text) 132 | except requests.HTTPError as e: 133 | print(f"Deploy failed ({e.response.status_code}):", file=sys.stderr) 134 | exitcode = 1 135 | try: 136 | data = e.response.json() 137 | except json.decoder.JSONDecodeError: 138 | print(e.response.text) 139 | else: 140 | if "status" in data and "message" in data: 141 | print(f"Status: {data['status']}") 142 | print(f"Message:\n{data['message']}") 143 | else: 144 | print(json.dumps(data, indent=3)) 145 | except requests.RequestException as e: 146 | print(f"Deploy failed: {e}", file=sys.stderr) 147 | exitcode = 1 148 | 149 | return exitcode, tmpdir 150 | 151 | 152 | def _url(target, action): 153 | if "url" in target: 154 | return urljoin(target["url"], action) 155 | print("Error: Missing url for project", file=sys.stderr) 156 | sys.exit(1) 157 | 158 | 159 | def _get_version(target, opts): 160 | version = opts.version or target.get("version") 161 | if version == "HG": 162 | process = subprocess.Popen( 163 | ["hg", "tip", "--template", "{rev}"], stdout=subprocess.PIPE, universal_newlines=True 164 | ) 165 | descriptor = f"r{process.communicate()[0]}" 166 | process = subprocess.Popen(["hg", "branch"], stdout=subprocess.PIPE, universal_newlines=True) 167 | name = process.communicate()[0].strip("\n") 168 | return f"{descriptor}-{name}" 169 | if version == "GIT": 170 | process = subprocess.Popen(["git", "describe"], stdout=subprocess.PIPE, universal_newlines=True) 171 | descriptor = process.communicate()[0].strip("\n") 172 | if process.wait() != 0: 173 | process = subprocess.Popen( 174 | ["git", "rev-list", "--count", "HEAD"], 175 | stdout=subprocess.PIPE, 176 | universal_newlines=True, 177 | ) 178 | descriptor = "r{}".format(process.communicate()[0].strip("\n")) 179 | 180 | process = subprocess.Popen( 181 | ["git", "rev-parse", "--abbrev-ref", "HEAD"], 182 | stdout=subprocess.PIPE, 183 | universal_newlines=True, 184 | ) 185 | name = process.communicate()[0].strip("\n") 186 | return f"{descriptor}-{name}" 187 | if version: 188 | return version 189 | return str(int(time.time())) 190 | 191 | 192 | def _build_egg(opts): 193 | closest = closest_scrapy_cfg() 194 | os.chdir(os.path.dirname(closest)) 195 | if not os.path.exists("setup.py"): 196 | settings = get_config().get("settings", "default") 197 | with open("setup.py", "w") as f: 198 | f.write(_SETUP_PY_TEMPLATE % {"settings": settings}) 199 | tmpdir = tempfile.mkdtemp(prefix="scrapydeploy-") 200 | 201 | if opts.include_dependencies: 202 | print("Including dependencies from requirements.txt", file=sys.stderr) 203 | if not os.path.isfile("requirements.txt"): 204 | print("Error: Missing requirements.txt", file=sys.stderr) 205 | sys.exit(1) 206 | command = "bdist_uberegg" 207 | else: 208 | command = "bdist_egg" 209 | 210 | kwargs = {} if opts.debug else {"stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL} 211 | subprocess.run([sys.executable, "setup.py", "clean", "-a"], check=True, **kwargs) 212 | subprocess.run([sys.executable, "setup.py", command, "-d", tmpdir], check=True, **kwargs) 213 | 214 | eggpath = glob.glob(os.path.join(tmpdir, "*.egg"))[0] 215 | return eggpath, tmpdir 216 | 217 | 218 | if __name__ == "__main__": 219 | main() 220 | -------------------------------------------------------------------------------- /scrapyd_client/exceptions.py: -------------------------------------------------------------------------------- 1 | class ErrorResponse(Exception): # noqa: N818 2 | """Raised when Scrapyd reports an error.""" 3 | 4 | 5 | class MalformedResponse(Exception): # noqa: N818 6 | """Raised when the response can't be decoded.""" 7 | -------------------------------------------------------------------------------- /scrapyd_client/pyclient.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import fnmatch 4 | import json 5 | 6 | import requests 7 | 8 | from scrapyd_client.exceptions import ErrorResponse, MalformedResponse 9 | from scrapyd_client.utils import get_auth 10 | 11 | DEFAULT_TARGET_URL = "http://localhost:6800" 12 | HEADERS = requests.utils.default_headers().copy() 13 | HEADERS["User-Agent"] = "Scrapyd-client/2.0.3" 14 | 15 | 16 | class ScrapydClient: 17 | """ScrapydClient to interact with a Scrapyd instance.""" 18 | 19 | def __init__(self, url: str | None = None, username: str | None = None, password: str | None = None) -> None: 20 | """Initialize ScrapydClient.""" 21 | self.url = DEFAULT_TARGET_URL if url is None else url 22 | self.auth = get_auth(url=self.url, username=username, password=password) 23 | 24 | def projects(self, pattern: str = "*") -> list[str]: 25 | """ 26 | Return the projects matching a pattern (if provided). 27 | 28 | :param pattern: The `pattern `__ for the projects to match 29 | :return: The "projects" value of the API response, filtered by the pattern (if provided). 30 | 31 | .. seealso:: `listprojects.json `__ 32 | """ 33 | return fnmatch.filter(self._get("listprojects")["projects"], pattern) 34 | 35 | def spiders(self, project: str, pattern: str = "*") -> list[str]: 36 | """ 37 | Return the spiders matching a pattern (if provided). 38 | 39 | :param pattern: The `pattern `__ for the spiders to match 40 | :return: The "spiders" value of the API response, filtered by the pattern (if provided). 41 | 42 | .. seealso:: `listspiders.json `__ 43 | """ 44 | return fnmatch.filter(self._get("listspiders", {"project": project})["spiders"], pattern) 45 | 46 | def jobs(self, project: str) -> dict: 47 | """ 48 | :return: The unmodified API response. 49 | 50 | .. seealso:: `listjobs.json `__ 51 | """ 52 | return self._get("listjobs", {"project": project}) 53 | 54 | def daemonstatus(self) -> dict: 55 | """ 56 | :return: The unmodified API response. 57 | 58 | .. seealso:: `daemonstatus.json `__ 59 | """ 60 | return self._get("daemonstatus") 61 | 62 | def versions(self, project: str) -> list[str]: 63 | """ 64 | :return: The "versions" value of the API response. 65 | 66 | .. seealso:: `listversions.json `__ 67 | """ 68 | return self._get("listversions", {"project": project})["versions"] 69 | 70 | def schedule(self, project: str, spider: str, args: list[tuple[str, str]] | None = None) -> str: 71 | """ 72 | :return: The "jobid" value of the API response. 73 | 74 | .. seealso:: `schedule.json `__ 75 | """ 76 | if args is None: 77 | args = [] 78 | 79 | return self._post("schedule", data=[*args, ("project", project), ("spider", spider)])["jobid"] 80 | 81 | def status(self, jobid: str, project: str | None = None) -> dict: 82 | """ 83 | :return: The unmodified API response. 84 | 85 | .. seealso:: `status.json `__ 86 | """ 87 | params = {"job": jobid} 88 | if project is not None: 89 | params["project"] = project 90 | 91 | return self._get("status", params) 92 | 93 | def delproject(self, project: str) -> dict: 94 | """ 95 | :return: The unmodified API response. 96 | 97 | .. seealso:: `delproject.json `__ 98 | """ 99 | return self._post("delproject", data={"project": project}) 100 | 101 | def delversion(self, project: str, version: str) -> dict: 102 | """ 103 | :return: The unmodified API response. 104 | 105 | .. seealso:: `delversion.json `__ 106 | """ 107 | return self._post("delversion", data={"project": project, "version": version}) 108 | 109 | def cancel(self, project: str, jobid: str) -> dict: 110 | """ 111 | :return: The unmodified API response. 112 | 113 | .. seealso:: `cancel.json `__ 114 | """ 115 | return self._post("cancel", data={"project": project, "job": jobid}) 116 | 117 | def _get(self, basename: str, params=None): 118 | if params is None: 119 | params = {} 120 | 121 | return _process_response( 122 | requests.get(f"{self.url}/{basename}.json", params=params, headers=HEADERS, auth=self.auth) 123 | ) 124 | 125 | def _post(self, basename: str, data): 126 | return _process_response( 127 | requests.post(f"{self.url}/{basename}.json", data=data, headers=HEADERS, auth=self.auth) 128 | ) 129 | 130 | 131 | def _process_response(response): 132 | try: 133 | response = response.json() 134 | except json.decoder.JSONDecodeError as e: 135 | raise MalformedResponse(response.text) from e 136 | 137 | status = response["status"] 138 | if status == "ok": 139 | return response 140 | if status == "error": 141 | raise ErrorResponse(response["message"]) 142 | raise RuntimeError(f"Unhandled response status: {status}") 143 | -------------------------------------------------------------------------------- /scrapyd_client/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import netrc 4 | import os 5 | from configparser import BasicInterpolation, ConfigParser 6 | from urllib.parse import urlparse 7 | 8 | from requests.auth import HTTPBasicAuth 9 | from scrapy.utils import conf 10 | 11 | 12 | class EnvInterpolation(BasicInterpolation): 13 | """Interpolation which expands environment variables in values.""" 14 | 15 | def before_get(self, parser, section, option, value, defaults): 16 | value = super().before_get(parser, section, option, value, defaults) 17 | return os.path.expandvars(value) 18 | 19 | 20 | def get_auth(url: str, username: str, password: str) -> HTTPBasicAuth | None: 21 | """Retrieve authentication from arguments or infers from .netrc.""" 22 | if username: 23 | return HTTPBasicAuth(username=username, password=password) 24 | 25 | try: 26 | username, _account, password = netrc.netrc().authenticators(urlparse(url).hostname) 27 | return HTTPBasicAuth(username=username, password=password) 28 | except (OSError, netrc.NetrcParseError, TypeError): 29 | return None 30 | 31 | 32 | def get_config(use_closest=True): 33 | """Get Scrapy config file as a ConfigParser.""" 34 | cfg = ConfigParser(interpolation=EnvInterpolation()) 35 | cfg.read(conf.get_sources(use_closest)) 36 | return cfg 37 | 38 | 39 | def _get_targets(): 40 | cfg = get_config() 41 | baset = dict(cfg.items("deploy")) if cfg.has_section("deploy") else {} 42 | targets = {} 43 | if "url" in baset: 44 | targets["default"] = baset 45 | for section in cfg.sections(): 46 | if section.startswith("deploy:"): 47 | t = baset.copy() 48 | t.update(cfg.items(section)) 49 | targets[section[7:]] = t 50 | return targets 51 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | def assert_lines(actual, expected): 5 | if isinstance(expected, str): 6 | assert actual.splitlines() == expected.splitlines() 7 | else: 8 | lines = actual.splitlines() 9 | assert len(lines) == len(expected) 10 | for i, line in enumerate(lines): 11 | assert re.search(f"^{expected[i]}$", line), f"{line} does not match {expected[i]}" 12 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | from textwrap import dedent 3 | 4 | import pytest 5 | 6 | 7 | def _write_conf_file(content): 8 | """ 9 | Scrapy startproject writes a file like: 10 | 11 | .. code-block:: ini 12 | 13 | # Automatically created by: scrapy startproject 14 | # 15 | # For more information about the [deploy] section see: 16 | # https://scrapyd.readthedocs.io/en/latest/deploy.html 17 | 18 | [settings] 19 | default = ${project_name}.settings 20 | 21 | [deploy] 22 | #url = http://localhost:6800/ 23 | project = ${project_name} 24 | 25 | See scrapy/templates/project/scrapy.cfg 26 | """ 27 | with open("scrapy.cfg", "w") as f: 28 | f.write( 29 | dedent( 30 | """\ 31 | [settings] 32 | default = scrapyproj.settings 33 | """ 34 | ) 35 | + dedent(content) 36 | ) 37 | 38 | 39 | @pytest.fixture 40 | def project(tmpdir, script_runner): 41 | cwd = os.getcwd() 42 | 43 | p = tmpdir.mkdir("myhome") 44 | p.chdir() 45 | ret = script_runner.run(["scrapy", "startproject", "scrapyproj"]) 46 | 47 | assert "New Scrapy project 'scrapyproj'" in ret.stdout 48 | assert ret.stderr == "" 49 | assert ret.success 50 | 51 | os.chdir("scrapyproj") 52 | yield 53 | os.chdir(cwd) 54 | 55 | 56 | @pytest.fixture 57 | def project_with_dependencies(project): 58 | with open("requirements.txt", "w") as f: 59 | f.write("") 60 | 61 | 62 | @pytest.fixture 63 | def conf_empty_section_implicit_target(project): 64 | _write_conf_file("[deploy]") 65 | 66 | 67 | @pytest.fixture 68 | def conf_empty_section_explicit_target(project): 69 | _write_conf_file("[deploy:mytarget]") 70 | 71 | 72 | @pytest.fixture 73 | def conf_no_project(project): 74 | _write_conf_file( 75 | """\ 76 | [deploy] 77 | url = http://localhost:6800/ 78 | """ 79 | ) 80 | 81 | 82 | @pytest.fixture 83 | def conf_no_url(project): 84 | _write_conf_file( 85 | """\ 86 | [deploy:mytarget] 87 | project = scrapydproject 88 | """ 89 | ) 90 | 91 | 92 | @pytest.fixture 93 | def conf_default_target(project): 94 | _write_conf_file( 95 | """\ 96 | [deploy] 97 | url = http://localhost:6800/ 98 | project = scrapydproject 99 | """ 100 | ) 101 | 102 | 103 | @pytest.fixture 104 | def conf_named_targets(project): 105 | # target2 is deliberately before target 1, to test ordering. 106 | _write_conf_file( 107 | """\ 108 | [deploy:target2] 109 | url = http://localhost:6802/ 110 | project = anotherproject 111 | 112 | [deploy:target1] 113 | url = http://localhost:6801/ 114 | project = scrapydproject 115 | """ 116 | ) 117 | 118 | 119 | @pytest.fixture 120 | def conf_mixed_targets(project): 121 | # target2 is deliberately before target 1, to test ordering. 122 | _write_conf_file( 123 | """\ 124 | [deploy] 125 | url = http://localhost:6800/ 126 | project = anotherproject 127 | 128 | [deploy:target1] 129 | url = http://localhost:6801/ 130 | project = scrapydproject 131 | """ 132 | ) 133 | -------------------------------------------------------------------------------- /tests/test_errors.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import requests 4 | 5 | 6 | def test_decode_error(mocker, script_runner, conf_default_target): 7 | mock_response = mocker.Mock() 8 | mock_response.json.side_effect = json.decoder.JSONDecodeError("", "", 0) 9 | mock_get = mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True) 10 | mock_get.return_value = mock_response 11 | result = script_runner.run(["scrapyd-client", "projects"]) 12 | 13 | assert not result.success 14 | assert result.stdout.startswith("Received a malformed response:\n") 15 | 16 | 17 | def test_projects(mocker, script_runner, conf_default_target): 18 | mock_response = mocker.Mock() 19 | mock_response.json.return_value = { 20 | "status": "error", 21 | "message": "Something went wrong.", 22 | } 23 | mock_get = mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True) 24 | mock_get.return_value = mock_response 25 | result = script_runner.run(["scrapyd-client", "projects"]) 26 | 27 | assert not result.success 28 | assert result.stdout == "Scrapyd responded with an error:\nSomething went wrong.\n" 29 | 30 | 31 | def test_connection_error(mocker, script_runner, conf_default_target): 32 | mock_get = mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True) 33 | mock_get.side_effect = requests.ConnectionError() 34 | result = script_runner.run(["scrapyd-client", "projects"]) 35 | 36 | assert not result.success 37 | assert result.stdout == "Failed to connect to target (default):\n\n" 38 | -------------------------------------------------------------------------------- /tests/test_projects.py: -------------------------------------------------------------------------------- 1 | def test_projects(mocker, script_runner, conf_default_target): 2 | projects = ["foo", "bar"] 3 | mock_response = mocker.Mock() 4 | mock_response.json.return_value = {"projects": ["foo", "bar"], "status": "ok"} 5 | mock_get = mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True) 6 | mock_get.return_value = mock_response 7 | result = script_runner.run(["scrapyd-client", "projects"]) 8 | 9 | assert result.success, result.stdout + "\n" + result.stderr 10 | assert not result.stderr, result.stderr 11 | assert result.stdout == "\n".join(projects) + "\n" 12 | -------------------------------------------------------------------------------- /tests/test_pyclient_idempotent.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from scrapyd_client.exceptions import ErrorResponse 4 | from scrapyd_client.pyclient import ScrapydClient 5 | 6 | 7 | def test_daemon_status_returns_valid_response(mocker, conf_default_target): 8 | mock_response = mocker.Mock() 9 | mock_response.json.return_value = {"status": "ok", "running": 5, "pending": 2, "finished": 10} 10 | mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True, return_value=mock_response) 11 | 12 | assert ScrapydClient().daemonstatus() == {"status": "ok", "running": 5, "pending": 2, "finished": 10} 13 | 14 | 15 | def test_daemon_status_handles_error_response(mocker, conf_default_target): 16 | mock_response = mocker.Mock() 17 | mock_response.json.return_value = {"status": "error", "message": "Daemon is not reachable."} 18 | mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True, return_value=mock_response) 19 | 20 | with pytest.raises(ErrorResponse) as excinfo: 21 | ScrapydClient().daemonstatus() 22 | assert "Daemon is not reachable." in str(excinfo.value) 23 | 24 | 25 | def test_versions_returns_versions(mocker, conf_default_target): 26 | mock_response = mocker.Mock() 27 | mock_response.json.return_value = {"status": "ok", "versions": ["v1.0", "v1.1", "v2.0"]} 28 | mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True, return_value=mock_response) 29 | 30 | assert ScrapydClient().versions("my_project") == ["v1.0", "v1.1", "v2.0"] 31 | 32 | 33 | def test_versions_handles_no_versions(mocker, conf_default_target): 34 | mock_response = mocker.Mock() 35 | mock_response.json.return_value = {"status": "ok", "versions": []} 36 | mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True, return_value=mock_response) 37 | 38 | assert ScrapydClient().versions("my_project") == [] 39 | 40 | 41 | def test_versions_handles_error_response(mocker, conf_default_target): 42 | mock_response = mocker.Mock() 43 | mock_response.json.return_value = {"status": "error", "message": "Project not found."} 44 | mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True, return_value=mock_response) 45 | 46 | with pytest.raises(ErrorResponse) as excinfo: 47 | ScrapydClient().versions("nonexistent") 48 | assert "Project not found." in str(excinfo.value) 49 | -------------------------------------------------------------------------------- /tests/test_pyclient_stateful.py: -------------------------------------------------------------------------------- 1 | from scrapyd_client.pyclient import ScrapydClient 2 | 3 | 4 | def test_delproject(mocker, conf_default_target): 5 | client = ScrapydClient() 6 | 7 | mocker.patch.object(client, "projects", return_value=["existing_project"]) 8 | mock_post = mocker.patch.object(client, "_post", return_value={"status": "ok"}) 9 | 10 | response = client.delproject("existing_project") 11 | 12 | assert response["status"] == "ok" 13 | mock_post.assert_called_once_with("delproject", data={"project": "existing_project"}) 14 | 15 | 16 | def test_delversion(mocker, conf_default_target): 17 | client = ScrapydClient() 18 | 19 | mocker.patch.object(client, "projects", return_value=["existing_project"]) 20 | mocker.patch.object(client, "versions", return_value=["v1.0", "v1.1"]) 21 | mock_post = mocker.patch.object(client, "_post", return_value={"status": "ok"}) 22 | 23 | response = client.delversion("existing_project", "v1.0") 24 | 25 | assert response["status"] == "ok" 26 | mock_post.assert_called_once_with("delversion", data={"project": "existing_project", "version": "v1.0"}) 27 | 28 | 29 | def test_cancel(mocker, conf_default_target): 30 | client = ScrapydClient() 31 | 32 | mocker.patch.object(client, "projects", return_value=["existing_project"]) 33 | mocker.patch.object(client, "jobs", return_value={"running": ["jobid1", "jobid2"]}) 34 | mock_post = mocker.patch.object(client, "_post", return_value={"status": "ok"}) 35 | 36 | response = client.cancel("existing_project", "jobid1") 37 | 38 | assert response["status"] == "ok" 39 | mock_post.assert_called_once_with("cancel", data={"project": "existing_project", "job": "jobid1"}) 40 | -------------------------------------------------------------------------------- /tests/test_schedule.py: -------------------------------------------------------------------------------- 1 | from itertools import chain 2 | 3 | 4 | def test_schedule(mocker, script_runner, conf_default_target): 5 | get_responses = [ 6 | {"projects": ["foo"]}, 7 | {"spiders": ["bar"]}, 8 | ] 9 | 10 | post_responses = [{"jobid": "42"}] 11 | for response in chain(get_responses, post_responses): 12 | response["status"] = "ok" 13 | 14 | mock_get_response = mocker.Mock() 15 | mock_get_response.json.side_effect = get_responses 16 | mock_get = mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True) 17 | mock_get.return_value = mock_get_response 18 | 19 | mock_post_response = mocker.Mock() 20 | mock_post_response.json.side_effect = post_responses 21 | mock_post = mocker.patch("scrapyd_client.pyclient.requests.post", autospec=True) 22 | mock_post.return_value = mock_post_response 23 | 24 | result = script_runner.run( 25 | ["scrapyd-client", "schedule", "-p", "foo", "--arg", "setting=HTTPPROXY_ENABLED=True", "bar"] 26 | ) 27 | 28 | assert result.success, result.stdout + "\n" + result.stderr 29 | assert not result.stderr, result.stderr 30 | assert result.stdout == "foo / bar => 42\n" 31 | -------------------------------------------------------------------------------- /tests/test_scrapyd_deploy.py: -------------------------------------------------------------------------------- 1 | import json 2 | from textwrap import dedent 3 | from unittest.mock import MagicMock, patch 4 | 5 | import pytest 6 | import requests 7 | 8 | from tests import assert_lines 9 | 10 | 11 | @pytest.mark.parametrize("args", [[], ["default"]]) 12 | def test_not_in_project(args, script_runner): 13 | ret = script_runner.run(["scrapyd-deploy", *args]) 14 | 15 | assert ret.stdout == "" 16 | assert_lines(ret.stderr, "Error: no Scrapy project found in this location") 17 | assert not ret.success 18 | 19 | 20 | def test_too_many_arguments(script_runner, project): 21 | ret = script_runner.run(["scrapyd-deploy", "mytarget", "extra"]) 22 | 23 | assert ret.stdout == "" 24 | assert_lines( 25 | ret.stderr, 26 | dedent( 27 | """\ 28 | usage: scrapyd-deploy [-h] [-p PROJECT] [-v VERSION] [-a] [-d] [--egg FILE] 29 | [--build-egg FILE] [--include-dependencies] 30 | [TARGET] 31 | scrapyd-deploy: error: unrecognized arguments: extra 32 | """ 33 | ), 34 | ) 35 | assert not ret.success 36 | 37 | 38 | def test_missing_url(script_runner, conf_no_url): 39 | ret = script_runner.run(["scrapyd-deploy", "mytarget"]) 40 | 41 | assert ret.stdout == "" 42 | assert_lines(ret.stderr, [r"Packing version \d+", "Error: Missing url for project"]) 43 | assert not ret.success 44 | 45 | 46 | def test_unknown_target_implicit(script_runner, project): 47 | ret = script_runner.run(["scrapyd-deploy"]) 48 | 49 | assert ret.stdout == "" 50 | assert_lines(ret.stderr, "Unknown target: default") 51 | assert not ret.success 52 | 53 | 54 | def test_unknown_target_explicit(script_runner, project): 55 | ret = script_runner.run(["scrapyd-deploy", "nonexistent"]) 56 | 57 | assert ret.stdout == "" 58 | assert_lines(ret.stderr, "Unknown target: nonexistent") 59 | assert not ret.success 60 | 61 | 62 | def test_empty_section_implicit_target(script_runner, conf_empty_section_implicit_target): 63 | ret = script_runner.run(["scrapyd-deploy"]) 64 | 65 | assert ret.stdout == "" 66 | assert_lines(ret.stderr, "Unknown target: default") 67 | assert not ret.success 68 | 69 | 70 | def test_empty_section_explicit_target(script_runner, conf_empty_section_explicit_target): 71 | ret = script_runner.run(["scrapyd-deploy", "mytarget"]) 72 | 73 | assert ret.stdout == "" 74 | assert_lines(ret.stderr, "Error: Missing project") 75 | assert not ret.success 76 | 77 | 78 | def test_deploy_missing_project(script_runner, conf_no_project): 79 | ret = script_runner.run(["scrapyd-deploy"]) 80 | 81 | assert ret.stdout == "" 82 | assert_lines(ret.stderr, "Error: Missing project") 83 | assert not ret.success 84 | 85 | 86 | def test_deploy_missing_url(script_runner, conf_no_url): 87 | ret = script_runner.run(["scrapyd-deploy", "mytarget"]) 88 | 89 | assert ret.stdout == "" 90 | assert_lines(ret.stderr, [r"Packing version \d+", "Error: Missing url for project"]) 91 | assert not ret.success 92 | 93 | 94 | def test_build_egg(script_runner, project): 95 | ret = script_runner.run(["scrapyd-deploy", "--build-egg", "myegg.egg"]) 96 | 97 | assert ret.stdout == "" 98 | assert_lines(ret.stderr, "Writing egg to myegg.egg") 99 | assert ret.success 100 | 101 | 102 | def test_build_egg_inc_dependencies_no_dep(script_runner, project): 103 | ret = script_runner.run(["scrapyd-deploy", "--include-dependencies", "--build-egg", "myegg-deps.egg"]) 104 | 105 | assert ret.stdout == "" 106 | assert_lines( 107 | ret.stderr, 108 | dedent( 109 | """\ 110 | Including dependencies from requirements.txt 111 | Error: Missing requirements.txt 112 | """ 113 | ), 114 | ) 115 | assert not ret.success 116 | 117 | 118 | def test_build_egg_inc_dependencies_with_dep(script_runner, project_with_dependencies): 119 | ret = script_runner.run(["scrapyd-deploy", "--include-dependencies", "--build-egg", "myegg-deps.egg", "--debug"]) 120 | 121 | assert ret.stdout == "" 122 | assert_lines( 123 | ret.stderr, 124 | dedent( 125 | """\ 126 | Including dependencies from requirements.txt 127 | Writing egg to myegg-deps.egg 128 | """ 129 | ), 130 | ) 131 | assert ret.success 132 | 133 | 134 | def test_deploy_success(script_runner, conf_default_target): 135 | with patch("scrapyd_client.deploy.requests.post") as mocked: 136 | mocked.return_value.status_code = 200 137 | mocked.return_value.text = '{"status": "ok"}' 138 | 139 | ret = script_runner.run(["scrapyd-deploy"]) 140 | 141 | assert_lines(ret.stdout, '{"status": "ok"}') 142 | assert_lines( 143 | ret.stderr, 144 | [ 145 | r"Packing version \d+", 146 | r'Deploying to project "scrapydproject" in http://localhost:6800/addversion\.json', 147 | r"Server response \(200\):", 148 | ], 149 | ) 150 | assert ret.success 151 | 152 | 153 | @pytest.mark.parametrize( 154 | ("content", "expected"), 155 | [ 156 | ("content", "content"), 157 | (["content"], '[\n "content"\n]'), 158 | ( 159 | {"status": "error", "message": "content"}, 160 | "Status: error\nMessage:\ncontent", 161 | ), 162 | ], 163 | ) 164 | def test_deploy_httperror(content, expected, script_runner, conf_default_target): 165 | with patch("scrapyd_client.deploy.requests.post") as mocked: 166 | response = MagicMock(status_code=404, text=content) 167 | if isinstance(content, (dict, list)): 168 | response.json.side_effect = lambda: content 169 | else: 170 | response.json.side_effect = json.decoder.JSONDecodeError("", "", 0) 171 | mocked.side_effect = requests.HTTPError(response=response) 172 | 173 | ret = script_runner.run(["scrapyd-deploy"]) 174 | 175 | assert ret.returncode == 1 176 | assert_lines(ret.stdout, f"{expected}") 177 | assert_lines( 178 | ret.stderr, 179 | [ 180 | r"Packing version \d+", 181 | r'Deploying to project "scrapydproject" in http://localhost:6800/addversion\.json', 182 | r"Deploy failed \(404\):", 183 | ], 184 | ) 185 | 186 | 187 | def test_deploy_urlerror(script_runner, conf_default_target): 188 | with patch("scrapyd_client.deploy.requests.post") as mocked: 189 | mocked.side_effect = requests.RequestException("content") 190 | 191 | ret = script_runner.run(["scrapyd-deploy"]) 192 | 193 | assert ret.returncode == 1 194 | assert ret.stdout == "" 195 | assert_lines( 196 | ret.stderr, 197 | [ 198 | r"Packing version \d+", 199 | r'Deploying to project "scrapydproject" in http://localhost:6800/addversion\.json', 200 | r"Deploy failed: content", 201 | ], 202 | ) 203 | -------------------------------------------------------------------------------- /tests/test_spiders.py: -------------------------------------------------------------------------------- 1 | responses = [ 2 | {"projects": ["foo", "bar", "peng"]}, 3 | {"spiders": ["foo_1"]}, 4 | {"spiders": []}, 5 | {"spiders": ["boing", "boom", "tschak"]}, 6 | ] 7 | for response in responses: 8 | response["status"] = "ok" 9 | 10 | 11 | def test_spiders(mocker, script_runner, conf_default_target): 12 | mock_response = mocker.Mock() 13 | mock_response.json.side_effect = responses 14 | mock_get = mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True) 15 | mock_get.return_value = mock_response 16 | result = script_runner.run(["scrapyd-client", "spiders", "-p", "*"]) 17 | 18 | assert result.success, result.stdout + "\n" + result.stderr 19 | assert not result.stderr, result.stderr 20 | assert ( 21 | result.stdout 22 | == """ 23 | foo: 24 | foo_1 25 | bar: 26 | No spiders. 27 | peng: 28 | boing 29 | boom 30 | tschak 31 | """.strip() 32 | + "\n" 33 | ) 34 | 35 | 36 | def test_spiders_verbose(mocker, script_runner, conf_default_target): 37 | mock_response = mocker.Mock() 38 | mock_response.json.side_effect = responses 39 | mock_get = mocker.patch("scrapyd_client.pyclient.requests.get", autospec=True) 40 | mock_get.return_value = mock_response 41 | result = script_runner.run(["scrapyd-client", "spiders", "-v", "-p", "*"]) 42 | 43 | assert result.success, result.stdout + "\n" + result.stderr 44 | assert not result.stderr, result.stderr 45 | assert ( 46 | result.stdout 47 | == """ 48 | foo foo_1 49 | peng boing 50 | peng boom 51 | peng tschak 52 | """.strip() 53 | + "\n" 54 | ) 55 | -------------------------------------------------------------------------------- /tests/test_targets.py: -------------------------------------------------------------------------------- 1 | from textwrap import dedent 2 | 3 | from tests import assert_lines 4 | 5 | 6 | def test_not_in_project(script_runner): 7 | ret = script_runner.run(["scrapyd-client", "targets"]) 8 | 9 | assert ret.stdout == "" 10 | assert_lines(ret.stderr, "Error: no Scrapy project found in this location") 11 | assert not ret.success 12 | 13 | 14 | def test_too_many_arguments(script_runner, project): 15 | ret = script_runner.run(["scrapyd-client", "targets", "extra"]) 16 | 17 | assert ret.stdout == "" 18 | assert_lines( 19 | ret.stderr, 20 | dedent( 21 | """\ 22 | usage: scrapyd-client [-h] {deploy,targets,projects,schedule,spiders} ... 23 | scrapyd-client: error: unrecognized arguments: extra 24 | """ 25 | ), 26 | ) 27 | assert not ret.success 28 | 29 | 30 | def test_list_targets_with_default(script_runner, conf_mixed_targets): 31 | ret = script_runner.run(["scrapyd-client", "targets"]) 32 | 33 | assert_lines( 34 | ret.stdout, 35 | dedent( 36 | """\ 37 | default http://localhost:6800/ 38 | target1 http://localhost:6801/ 39 | """ 40 | ), 41 | ) 42 | assert ret.stderr == "" 43 | assert ret.success 44 | 45 | 46 | def test_list_targets_without_default(script_runner, conf_named_targets): 47 | ret = script_runner.run(["scrapyd-client", "targets"]) 48 | 49 | assert_lines( 50 | ret.stdout, 51 | dedent( 52 | """\ 53 | target2 http://localhost:6802/ 54 | target1 http://localhost:6801/ 55 | """ 56 | ), 57 | ) 58 | assert ret.stderr == "" 59 | assert ret.success 60 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import netrc 2 | 3 | import pytest 4 | from requests.auth import HTTPBasicAuth 5 | 6 | from scrapyd_client.utils import get_auth 7 | 8 | try: 9 | netrc.netrc() 10 | exists = True 11 | except FileNotFoundError: 12 | exists = False 13 | 14 | 15 | @pytest.mark.skipif(exists, reason="a .netrc file exists") 16 | @pytest.mark.parametrize( 17 | ("url", "username", "password", "expected"), 18 | [ 19 | ("http://localhost:6800", None, None, None), 20 | ( 21 | "http://localhost:6800", 22 | "user", 23 | "pass", 24 | HTTPBasicAuth("user", "pass"), 25 | ), 26 | ], 27 | ) 28 | def test_get_auth(url, username, password, expected): 29 | assert get_auth(url, username, password) == expected 30 | 31 | 32 | def test_get_auth_netrc(mocker): 33 | n = mocker.patch("scrapyd_client.utils.netrc") # mock netrc 34 | n.netrc.return_value.authenticators.return_value = ("user", "", "pass") 35 | assert get_auth("http://localhost:6800", None, None) == HTTPBasicAuth("user", "pass") 36 | --------------------------------------------------------------------------------